@rawdash/connector-firebase-crashlytics 0.21.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/dist/index.d.ts +4 -22
- package/dist/index.js +303 -232
- package/dist/index.js.map +1 -1
- package/package.json +14 -14
package/dist/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// ../gcp-shared/dist/index.js
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { z as z2 } from "zod";
|
|
4
|
+
import { z as z3 } from "zod";
|
|
4
5
|
var serviceAccountKeySchema = z.object({
|
|
5
6
|
client_email: z.string().min(1),
|
|
6
7
|
private_key: z.string().min(1),
|
|
@@ -77,6 +78,15 @@ async function buildServiceAccountJwt(serviceAccountJson, scope) {
|
|
|
77
78
|
body
|
|
78
79
|
};
|
|
79
80
|
}
|
|
81
|
+
function buildRefreshTokenGrant(credentials) {
|
|
82
|
+
const body = new URLSearchParams({
|
|
83
|
+
grant_type: "refresh_token",
|
|
84
|
+
refresh_token: credentials.refreshToken,
|
|
85
|
+
client_id: credentials.clientId,
|
|
86
|
+
client_secret: credentials.clientSecret
|
|
87
|
+
}).toString();
|
|
88
|
+
return { url: "https://oauth2.googleapis.com/token", body };
|
|
89
|
+
}
|
|
80
90
|
var gcpAuthConfigShape = {
|
|
81
91
|
serviceAccountJson: z2.object({ $secret: z2.string().trim().min(1) }).meta({
|
|
82
92
|
label: "Service Account JSON",
|
|
@@ -84,8 +94,6 @@ var gcpAuthConfigShape = {
|
|
|
84
94
|
secret: true
|
|
85
95
|
})
|
|
86
96
|
};
|
|
87
|
-
|
|
88
|
-
// ../../connector-shared/dist/index.js
|
|
89
97
|
var HttpClientError = class extends Error {
|
|
90
98
|
response;
|
|
91
99
|
constructor(message, response) {
|
|
@@ -99,10 +107,227 @@ var AuthError = class extends HttpClientError {
|
|
|
99
107
|
};
|
|
100
108
|
var HTTP_CLIENT_VERSION = "0.0.0";
|
|
101
109
|
var DEFAULT_USER_AGENT = `rawdash-connector/${HTTP_CLIENT_VERSION} (+https://rawdash.dev)`;
|
|
110
|
+
function parseEpoch(value, unit) {
|
|
111
|
+
if (value === null || value === void 0) {
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
if (unit === "iso") {
|
|
115
|
+
if (typeof value !== "string") {
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
const ms = new Date(value).getTime();
|
|
119
|
+
return Number.isFinite(ms) ? ms : null;
|
|
120
|
+
}
|
|
121
|
+
if (typeof value === "string" && value.trim() === "") {
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
const n = typeof value === "number" ? value : Number(value);
|
|
125
|
+
if (!Number.isFinite(n)) {
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
const result = unit === "s" ? n * 1e3 : n;
|
|
129
|
+
return Number.isFinite(result) ? result : null;
|
|
130
|
+
}
|
|
131
|
+
var BQ_IDENT_RE = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
132
|
+
var BQ_DATASET_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
133
|
+
var BQ_API_BASE = "https://bigquery.googleapis.com/bigquery/v2";
|
|
134
|
+
var BQ_READONLY_SCOPE = "https://www.googleapis.com/auth/bigquery.readonly";
|
|
135
|
+
var BQ_PAGE_SIZE = 1e4;
|
|
136
|
+
var BQ_QUERY_TIMEOUT_MS = 3e4;
|
|
137
|
+
var bqQueryResponseSchema = z3.object({
|
|
138
|
+
jobComplete: z3.boolean().optional(),
|
|
139
|
+
schema: z3.object({
|
|
140
|
+
fields: z3.array(z3.object({ name: z3.string(), type: z3.string() }))
|
|
141
|
+
}).optional(),
|
|
142
|
+
rows: z3.array(
|
|
143
|
+
z3.object({
|
|
144
|
+
f: z3.array(z3.object({ v: z3.string().nullable().optional() }))
|
|
145
|
+
})
|
|
146
|
+
).optional(),
|
|
147
|
+
pageToken: z3.string().optional(),
|
|
148
|
+
jobReference: z3.object({
|
|
149
|
+
projectId: z3.string(),
|
|
150
|
+
jobId: z3.string(),
|
|
151
|
+
location: z3.string().optional()
|
|
152
|
+
}).optional()
|
|
153
|
+
});
|
|
154
|
+
function buildBigQueryPageRequest(opts) {
|
|
155
|
+
const pageSize = opts.pageSize ?? BQ_PAGE_SIZE;
|
|
156
|
+
const timeoutMs = opts.timeoutMs ?? BQ_QUERY_TIMEOUT_MS;
|
|
157
|
+
if (opts.pageToken === void 0) {
|
|
158
|
+
const url2 = `${BQ_API_BASE}/projects/${encodeURIComponent(
|
|
159
|
+
opts.projectId
|
|
160
|
+
)}/queries`;
|
|
161
|
+
const body = {
|
|
162
|
+
query: opts.sql,
|
|
163
|
+
useLegacySql: false,
|
|
164
|
+
maxResults: pageSize,
|
|
165
|
+
timeoutMs
|
|
166
|
+
};
|
|
167
|
+
if (opts.location !== void 0) {
|
|
168
|
+
body["location"] = opts.location;
|
|
169
|
+
}
|
|
170
|
+
return { method: "POST", url: url2, body: JSON.stringify(body) };
|
|
171
|
+
}
|
|
172
|
+
if (opts.jobReference === void 0) {
|
|
173
|
+
throw new Error(
|
|
174
|
+
"cannot fetch the next page of BigQuery results without a jobReference"
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
const params = new URLSearchParams({
|
|
178
|
+
pageToken: opts.pageToken,
|
|
179
|
+
maxResults: String(pageSize),
|
|
180
|
+
timeoutMs: String(timeoutMs)
|
|
181
|
+
});
|
|
182
|
+
const location = opts.jobReference.location ?? opts.location;
|
|
183
|
+
if (location !== void 0) {
|
|
184
|
+
params.set("location", location);
|
|
185
|
+
}
|
|
186
|
+
const url = `${BQ_API_BASE}/projects/${encodeURIComponent(
|
|
187
|
+
opts.jobReference.projectId
|
|
188
|
+
)}/queries/${encodeURIComponent(opts.jobReference.jobId)}?${params.toString()}`;
|
|
189
|
+
return { method: "GET", url };
|
|
190
|
+
}
|
|
191
|
+
async function collectBigQueryPages(opts) {
|
|
192
|
+
const rows = [];
|
|
193
|
+
let pageToken;
|
|
194
|
+
let jobReference;
|
|
195
|
+
let page = 0;
|
|
196
|
+
const phaseStart = Date.now();
|
|
197
|
+
do {
|
|
198
|
+
if (opts.signal?.aborted) {
|
|
199
|
+
return { rows, aborted: true };
|
|
200
|
+
}
|
|
201
|
+
const request = buildBigQueryPageRequest({
|
|
202
|
+
projectId: opts.projectId,
|
|
203
|
+
sql: opts.sql,
|
|
204
|
+
pageToken,
|
|
205
|
+
jobReference,
|
|
206
|
+
location: opts.location,
|
|
207
|
+
pageSize: opts.pageSize
|
|
208
|
+
});
|
|
209
|
+
let response;
|
|
210
|
+
try {
|
|
211
|
+
response = await opts.fetchPage(request, opts.signal);
|
|
212
|
+
} catch (err) {
|
|
213
|
+
opts.logger?.warn("fetch page failed", {
|
|
214
|
+
resource: opts.resource,
|
|
215
|
+
page: page + 1,
|
|
216
|
+
error: err instanceof Error ? err.message : String(err)
|
|
217
|
+
});
|
|
218
|
+
throw err;
|
|
219
|
+
}
|
|
220
|
+
if (response.jobComplete === false) {
|
|
221
|
+
throw new Error(opts.jobIncompleteMessage);
|
|
222
|
+
}
|
|
223
|
+
if (response.jobReference !== void 0) {
|
|
224
|
+
jobReference = response.jobReference;
|
|
225
|
+
}
|
|
226
|
+
const pageRows = opts.mapRows(response);
|
|
227
|
+
rows.push(...pageRows);
|
|
228
|
+
pageToken = typeof response.pageToken === "string" && response.pageToken.length > 0 ? response.pageToken : void 0;
|
|
229
|
+
page += 1;
|
|
230
|
+
opts.logger?.info("fetched page", {
|
|
231
|
+
resource: opts.resource,
|
|
232
|
+
page,
|
|
233
|
+
items: pageRows.length,
|
|
234
|
+
next: pageToken ?? null
|
|
235
|
+
});
|
|
236
|
+
} while (pageToken !== void 0);
|
|
237
|
+
opts.logger?.info("resource done", {
|
|
238
|
+
resource: opts.resource,
|
|
239
|
+
pages: page,
|
|
240
|
+
items: rows.length,
|
|
241
|
+
duration_ms: Date.now() - phaseStart
|
|
242
|
+
});
|
|
243
|
+
return { rows, aborted: false };
|
|
244
|
+
}
|
|
245
|
+
function indexBqFields(response) {
|
|
246
|
+
const fieldIndex = {};
|
|
247
|
+
(response.schema?.fields ?? []).forEach((field, idx) => {
|
|
248
|
+
fieldIndex[field.name] = idx;
|
|
249
|
+
});
|
|
250
|
+
return fieldIndex;
|
|
251
|
+
}
|
|
252
|
+
function readBqCell(cells, fieldIndex, name) {
|
|
253
|
+
const idx = fieldIndex[name];
|
|
254
|
+
if (idx === void 0) {
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
const raw = cells[idx]?.v;
|
|
258
|
+
if (raw === void 0 || raw === null) {
|
|
259
|
+
return null;
|
|
260
|
+
}
|
|
261
|
+
return raw;
|
|
262
|
+
}
|
|
263
|
+
function parseBqDateOrEpoch(value) {
|
|
264
|
+
const dateMatch = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
|
|
265
|
+
if (dateMatch) {
|
|
266
|
+
return Date.UTC(
|
|
267
|
+
Number(dateMatch[1]),
|
|
268
|
+
Number(dateMatch[2]) - 1,
|
|
269
|
+
Number(dateMatch[3])
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
return parseEpoch(value, "iso");
|
|
273
|
+
}
|
|
274
|
+
var GcpAccessTokenProvider = class {
|
|
275
|
+
constructor(opts) {
|
|
276
|
+
this.opts = opts;
|
|
277
|
+
}
|
|
278
|
+
opts;
|
|
279
|
+
cached = null;
|
|
280
|
+
async resolveGrant() {
|
|
281
|
+
const serviceAccountJson = this.opts.getServiceAccountJson();
|
|
282
|
+
if (serviceAccountJson) {
|
|
283
|
+
return buildServiceAccountJwt(serviceAccountJson, this.opts.scope);
|
|
284
|
+
}
|
|
285
|
+
const refreshTokenCredentials = this.opts.getRefreshTokenCredentials?.();
|
|
286
|
+
if (refreshTokenCredentials) {
|
|
287
|
+
return buildRefreshTokenGrant(refreshTokenCredentials);
|
|
288
|
+
}
|
|
289
|
+
throw new AuthError(
|
|
290
|
+
`${this.opts.connectorId}: missing serviceAccountJson or refresh-token credentials`
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
async getToken(signal) {
|
|
294
|
+
if (this.cached && Date.now() < this.cached.expiresAt) {
|
|
295
|
+
return this.cached.token;
|
|
296
|
+
}
|
|
297
|
+
const { url, body } = await this.resolveGrant();
|
|
298
|
+
const res = await this.opts.post(url, {
|
|
299
|
+
resource: "oauth_token",
|
|
300
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
301
|
+
body,
|
|
302
|
+
signal
|
|
303
|
+
});
|
|
304
|
+
const expiresIn = res.body.expires_in ?? 3600;
|
|
305
|
+
this.cached = {
|
|
306
|
+
token: res.body.access_token,
|
|
307
|
+
expiresAt: Date.now() + (expiresIn - 60) * 1e3
|
|
308
|
+
};
|
|
309
|
+
return this.cached.token;
|
|
310
|
+
}
|
|
311
|
+
};
|
|
312
|
+
var MS_PER_DAY = 864e5;
|
|
313
|
+
function pad2(n) {
|
|
314
|
+
return String(n).padStart(2, "0");
|
|
315
|
+
}
|
|
316
|
+
function toDateStr(ms) {
|
|
317
|
+
const d = new Date(ms);
|
|
318
|
+
return `${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}-${pad2(d.getUTCDate())}`;
|
|
319
|
+
}
|
|
320
|
+
function startOfUtcDay(ms) {
|
|
321
|
+
return Math.floor(ms / MS_PER_DAY) * MS_PER_DAY;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// ../../connector-shared/dist/index.js
|
|
325
|
+
var HTTP_CLIENT_VERSION2 = "0.0.0";
|
|
326
|
+
var DEFAULT_USER_AGENT2 = `rawdash-connector/${HTTP_CLIENT_VERSION2} (+https://rawdash.dev)`;
|
|
102
327
|
function connectorUserAgent(connectorId) {
|
|
103
|
-
return `rawdash-connector-${connectorId}/${
|
|
328
|
+
return `rawdash-connector-${connectorId}/${HTTP_CLIENT_VERSION2} (+https://rawdash.dev)`;
|
|
104
329
|
}
|
|
105
|
-
function
|
|
330
|
+
function parseEpoch2(value, unit) {
|
|
106
331
|
if (value === null || value === void 0) {
|
|
107
332
|
return null;
|
|
108
333
|
}
|
|
@@ -132,18 +357,16 @@ import {
|
|
|
132
357
|
defineResources,
|
|
133
358
|
schemasFromResources
|
|
134
359
|
} from "@rawdash/core";
|
|
135
|
-
import { z as
|
|
136
|
-
var BQ_IDENT_RE = /^[A-Za-z_][A-Za-z0-9_-]*$/;
|
|
137
|
-
var BQ_DATASET_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
360
|
+
import { z as z4 } from "zod";
|
|
138
361
|
var configFields = defineConfigFields(
|
|
139
|
-
|
|
362
|
+
z4.object({
|
|
140
363
|
...gcpAuthConfigShape,
|
|
141
|
-
projectId:
|
|
364
|
+
projectId: z4.string().regex(BQ_IDENT_RE, "projectId must be a valid GCP project id").meta({
|
|
142
365
|
label: "GCP project ID",
|
|
143
366
|
description: "Project that hosts the Firebase Crashlytics -> BigQuery export (also the project used to bill the BigQuery queries this connector runs).",
|
|
144
367
|
placeholder: "my-firebase-project"
|
|
145
368
|
}),
|
|
146
|
-
bqDataset:
|
|
369
|
+
bqDataset: z4.string().regex(
|
|
147
370
|
BQ_DATASET_RE,
|
|
148
371
|
"bqDataset must be a valid BigQuery dataset id (letters, digits, and underscores; must start with a letter or underscore)"
|
|
149
372
|
).optional().meta({
|
|
@@ -151,17 +374,17 @@ var configFields = defineConfigFields(
|
|
|
151
374
|
description: "BigQuery dataset containing the Crashlytics export tables. Defaults to firebase_crashlytics (the default name Firebase uses when you enable the export).",
|
|
152
375
|
placeholder: "firebase_crashlytics"
|
|
153
376
|
}),
|
|
154
|
-
bqLocation:
|
|
377
|
+
bqLocation: z4.string().min(1).optional().meta({
|
|
155
378
|
label: "BigQuery location",
|
|
156
379
|
description: "Region or multi-region of the Crashlytics dataset (e.g. US, EU, us-central1). Defaults to US.",
|
|
157
380
|
placeholder: "US"
|
|
158
381
|
}),
|
|
159
|
-
lookbackDays:
|
|
382
|
+
lookbackDays: z4.number().int().positive().max(720).optional().meta({
|
|
160
383
|
label: "Backfill window (days)",
|
|
161
384
|
description: "How many days of history to query on a full sync. Defaults to 90.",
|
|
162
385
|
placeholder: "90"
|
|
163
386
|
}),
|
|
164
|
-
topIssuesLimit:
|
|
387
|
+
topIssuesLimit: z4.number().int().positive().max(500).optional().meta({
|
|
165
388
|
label: "Top issues limit",
|
|
166
389
|
description: "How many top issues to retain per sync, ranked by event count over the backfill window. Defaults to 50.",
|
|
167
390
|
placeholder: "50"
|
|
@@ -175,6 +398,7 @@ var doc = defineConnectorDoc({
|
|
|
175
398
|
tagline: "Track mobile app reliability over time from the Firebase Crashlytics -> BigQuery export: daily crashes, crash-free user rate, and top issues by impact.",
|
|
176
399
|
vendor: {
|
|
177
400
|
name: "Firebase",
|
|
401
|
+
domain: "firebase.google.com",
|
|
178
402
|
apiDocs: "https://firebase.google.com/docs/crashlytics/bigquery-export",
|
|
179
403
|
website: "https://firebase.google.com/products/crashlytics"
|
|
180
404
|
},
|
|
@@ -197,39 +421,18 @@ var doc = defineConnectorDoc({
|
|
|
197
421
|
"The Crashlytics BigQuery export is streamed; the trailing 2 days are always refetched on incremental syncs to pick up late-arriving rows."
|
|
198
422
|
]
|
|
199
423
|
});
|
|
200
|
-
var BQ_API_BASE = "https://bigquery.googleapis.com/bigquery/v2";
|
|
201
|
-
var BQ_SCOPE = "https://www.googleapis.com/auth/bigquery.readonly";
|
|
202
424
|
var CRASHES_METRIC_NAME = "crashes_per_day";
|
|
203
425
|
var TOP_ISSUES_ENTITY_TYPE = "firebase_crashlytics_issue";
|
|
204
426
|
var DEFAULT_LOOKBACK_DAYS = 90;
|
|
205
427
|
var DEFAULT_TOP_ISSUES_LIMIT = 50;
|
|
206
428
|
var DEFAULT_BQ_DATASET = "firebase_crashlytics";
|
|
207
429
|
var INCREMENTAL_LOOKBACK_DAYS = 2;
|
|
208
|
-
var MS_PER_DAY = 864e5;
|
|
209
|
-
var PAGE_SIZE = 1e4;
|
|
210
430
|
var firebaseCrashlyticsCredentials = {
|
|
211
431
|
serviceAccountJson: {
|
|
212
432
|
description: "Google service account JSON key (raw JSON or base64)",
|
|
213
433
|
auth: "required"
|
|
214
434
|
}
|
|
215
435
|
};
|
|
216
|
-
var bqQueryResponseSchema = z3.object({
|
|
217
|
-
jobComplete: z3.boolean().optional(),
|
|
218
|
-
schema: z3.object({
|
|
219
|
-
fields: z3.array(z3.object({ name: z3.string(), type: z3.string() }))
|
|
220
|
-
}).optional(),
|
|
221
|
-
rows: z3.array(
|
|
222
|
-
z3.object({
|
|
223
|
-
f: z3.array(z3.object({ v: z3.string().nullable().optional() }))
|
|
224
|
-
})
|
|
225
|
-
).optional(),
|
|
226
|
-
pageToken: z3.string().optional(),
|
|
227
|
-
jobReference: z3.object({
|
|
228
|
-
projectId: z3.string(),
|
|
229
|
-
jobId: z3.string(),
|
|
230
|
-
location: z3.string().optional()
|
|
231
|
-
}).optional()
|
|
232
|
-
});
|
|
233
436
|
var firebaseCrashlyticsResources = defineResources({
|
|
234
437
|
[CRASHES_METRIC_NAME]: {
|
|
235
438
|
shape: "metric",
|
|
@@ -330,56 +533,35 @@ var FirebaseCrashlyticsConnector = class _FirebaseCrashlyticsConnector extends B
|
|
|
330
533
|
}
|
|
331
534
|
id = id;
|
|
332
535
|
credentials = firebaseCrashlyticsCredentials;
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
throw new AuthError(`${this.id}: missing serviceAccountJson credential`);
|
|
341
|
-
}
|
|
342
|
-
const { url, body } = await buildServiceAccountJwt(
|
|
343
|
-
serviceAccountJson,
|
|
344
|
-
BQ_SCOPE
|
|
345
|
-
);
|
|
346
|
-
const res = await this.post(url, {
|
|
347
|
-
resource: "oauth_token",
|
|
348
|
-
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
349
|
-
body,
|
|
350
|
-
signal
|
|
536
|
+
tokenProvider;
|
|
537
|
+
getAccessToken(signal) {
|
|
538
|
+
this.tokenProvider ??= new GcpAccessTokenProvider({
|
|
539
|
+
connectorId: this.id,
|
|
540
|
+
scope: BQ_READONLY_SCOPE,
|
|
541
|
+
getServiceAccountJson: () => this.creds.serviceAccountJson,
|
|
542
|
+
post: (url, opts) => this.post(url, opts)
|
|
351
543
|
});
|
|
352
|
-
|
|
353
|
-
this.cachedToken = {
|
|
354
|
-
token: res.body.access_token,
|
|
355
|
-
expiresAt: Date.now() + (expiresIn - 60) * 1e3
|
|
356
|
-
};
|
|
357
|
-
return this.cachedToken.token;
|
|
544
|
+
return this.tokenProvider.getToken(signal);
|
|
358
545
|
}
|
|
359
|
-
async
|
|
360
|
-
const
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
useLegacySql: false,
|
|
366
|
-
maxResults: PAGE_SIZE,
|
|
367
|
-
timeoutMs: 3e4
|
|
546
|
+
async fetchBigQueryPage(resource, request, signal) {
|
|
547
|
+
const accessToken = await this.getAccessToken(signal);
|
|
548
|
+
const headers = {
|
|
549
|
+
Authorization: `Bearer ${accessToken}`,
|
|
550
|
+
"Content-Type": "application/json",
|
|
551
|
+
"User-Agent": connectorUserAgent(this.id)
|
|
368
552
|
};
|
|
369
|
-
if (
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
553
|
+
if (request.method === "POST") {
|
|
554
|
+
const res2 = await this.post(request.url, {
|
|
555
|
+
resource,
|
|
556
|
+
headers,
|
|
557
|
+
body: request.body,
|
|
558
|
+
signal
|
|
559
|
+
});
|
|
560
|
+
return res2.body;
|
|
374
561
|
}
|
|
375
|
-
const res = await this.
|
|
562
|
+
const res = await this.get(request.url, {
|
|
376
563
|
resource,
|
|
377
|
-
headers
|
|
378
|
-
Authorization: `Bearer ${accessToken}`,
|
|
379
|
-
"Content-Type": "application/json",
|
|
380
|
-
"User-Agent": connectorUserAgent(this.id)
|
|
381
|
-
},
|
|
382
|
-
body: JSON.stringify(body),
|
|
564
|
+
headers,
|
|
383
565
|
signal
|
|
384
566
|
});
|
|
385
567
|
return res.body;
|
|
@@ -432,107 +614,36 @@ var FirebaseCrashlyticsConnector = class _FirebaseCrashlyticsConnector extends B
|
|
|
432
614
|
}
|
|
433
615
|
return { done: true };
|
|
434
616
|
}
|
|
617
|
+
jobIncompleteMessage() {
|
|
618
|
+
return `${this.id}: BigQuery query did not complete within the synchronous timeout (jobComplete=false). Narrow the lookbackDays so the query finishes faster.`;
|
|
619
|
+
}
|
|
435
620
|
async collectSamples(sql, signal) {
|
|
436
|
-
const
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
const phaseStart = Date.now();
|
|
440
|
-
do {
|
|
441
|
-
if (signal?.aborted) {
|
|
442
|
-
break;
|
|
443
|
-
}
|
|
444
|
-
const accessToken = await this.getAccessToken(signal);
|
|
445
|
-
let response;
|
|
446
|
-
try {
|
|
447
|
-
response = await this.runQuery(
|
|
448
|
-
accessToken,
|
|
449
|
-
CRASHES_METRIC_NAME,
|
|
450
|
-
sql,
|
|
451
|
-
pageToken,
|
|
452
|
-
signal
|
|
453
|
-
);
|
|
454
|
-
} catch (err) {
|
|
455
|
-
this.logger.warn("fetch page failed", {
|
|
456
|
-
resource: CRASHES_METRIC_NAME,
|
|
457
|
-
page: page + 1,
|
|
458
|
-
error: err instanceof Error ? err.message : String(err)
|
|
459
|
-
});
|
|
460
|
-
throw err;
|
|
461
|
-
}
|
|
462
|
-
if (response.jobComplete === false) {
|
|
463
|
-
throw new Error(
|
|
464
|
-
`${this.id}: BigQuery query did not complete within the synchronous timeout (jobComplete=false). Narrow the lookbackDays so the query finishes faster.`
|
|
465
|
-
);
|
|
466
|
-
}
|
|
467
|
-
const pageSamples = buildCrashesSamplesFromBqResponse(response);
|
|
468
|
-
samples.push(...pageSamples);
|
|
469
|
-
pageToken = typeof response.pageToken === "string" && response.pageToken.length > 0 ? response.pageToken : void 0;
|
|
470
|
-
page += 1;
|
|
471
|
-
this.logger.info("fetched page", {
|
|
472
|
-
resource: CRASHES_METRIC_NAME,
|
|
473
|
-
page,
|
|
474
|
-
items: pageSamples.length,
|
|
475
|
-
next: pageToken ?? null
|
|
476
|
-
});
|
|
477
|
-
} while (pageToken !== void 0);
|
|
478
|
-
this.logger.info("resource done", {
|
|
621
|
+
const { rows } = await collectBigQueryPages({
|
|
622
|
+
projectId: this.settings.projectId,
|
|
623
|
+
sql,
|
|
479
624
|
resource: CRASHES_METRIC_NAME,
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
625
|
+
location: this.settings.bqLocation,
|
|
626
|
+
signal,
|
|
627
|
+
logger: this.logger,
|
|
628
|
+
mapRows: buildCrashesSamplesFromBqResponse,
|
|
629
|
+
jobIncompleteMessage: this.jobIncompleteMessage(),
|
|
630
|
+
fetchPage: (request, sig) => this.fetchBigQueryPage(CRASHES_METRIC_NAME, request, sig)
|
|
483
631
|
});
|
|
484
|
-
return
|
|
632
|
+
return rows;
|
|
485
633
|
}
|
|
486
634
|
async collectIssues(sql, signal) {
|
|
487
|
-
const
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
const phaseStart = Date.now();
|
|
491
|
-
do {
|
|
492
|
-
if (signal?.aborted) {
|
|
493
|
-
break;
|
|
494
|
-
}
|
|
495
|
-
const accessToken = await this.getAccessToken(signal);
|
|
496
|
-
let response;
|
|
497
|
-
try {
|
|
498
|
-
response = await this.runQuery(
|
|
499
|
-
accessToken,
|
|
500
|
-
"top_issues",
|
|
501
|
-
sql,
|
|
502
|
-
pageToken,
|
|
503
|
-
signal
|
|
504
|
-
);
|
|
505
|
-
} catch (err) {
|
|
506
|
-
this.logger.warn("fetch page failed", {
|
|
507
|
-
resource: "top_issues",
|
|
508
|
-
page: page + 1,
|
|
509
|
-
error: err instanceof Error ? err.message : String(err)
|
|
510
|
-
});
|
|
511
|
-
throw err;
|
|
512
|
-
}
|
|
513
|
-
if (response.jobComplete === false) {
|
|
514
|
-
throw new Error(
|
|
515
|
-
`${this.id}: BigQuery query did not complete within the synchronous timeout (jobComplete=false). Narrow the lookbackDays so the query finishes faster.`
|
|
516
|
-
);
|
|
517
|
-
}
|
|
518
|
-
const pageEntities = buildTopIssuesEntitiesFromBqResponse(response);
|
|
519
|
-
entities.push(...pageEntities);
|
|
520
|
-
pageToken = typeof response.pageToken === "string" && response.pageToken.length > 0 ? response.pageToken : void 0;
|
|
521
|
-
page += 1;
|
|
522
|
-
this.logger.info("fetched page", {
|
|
523
|
-
resource: "top_issues",
|
|
524
|
-
page,
|
|
525
|
-
items: pageEntities.length,
|
|
526
|
-
next: pageToken ?? null
|
|
527
|
-
});
|
|
528
|
-
} while (pageToken !== void 0);
|
|
529
|
-
this.logger.info("resource done", {
|
|
635
|
+
const { rows } = await collectBigQueryPages({
|
|
636
|
+
projectId: this.settings.projectId,
|
|
637
|
+
sql,
|
|
530
638
|
resource: "top_issues",
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
639
|
+
location: this.settings.bqLocation,
|
|
640
|
+
signal,
|
|
641
|
+
logger: this.logger,
|
|
642
|
+
mapRows: buildTopIssuesEntitiesFromBqResponse,
|
|
643
|
+
jobIncompleteMessage: this.jobIncompleteMessage(),
|
|
644
|
+
fetchPage: (request, sig) => this.fetchBigQueryPage("top_issues", request, sig)
|
|
534
645
|
});
|
|
535
|
-
return
|
|
646
|
+
return rows;
|
|
536
647
|
}
|
|
537
648
|
};
|
|
538
649
|
function buildCrashesPerDaySql(args) {
|
|
@@ -580,27 +691,17 @@ function buildTopIssuesSql(args) {
|
|
|
580
691
|
` AND DATE(event_timestamp) < DATE('${args.endDate}')`,
|
|
581
692
|
" AND issue_id IS NOT NULL",
|
|
582
693
|
"GROUP BY issue_id",
|
|
583
|
-
"ORDER BY event_count DESC",
|
|
694
|
+
"ORDER BY event_count DESC, last_seen DESC, issue_id ASC",
|
|
584
695
|
`LIMIT ${args.limit}`
|
|
585
696
|
].join("\n");
|
|
586
697
|
}
|
|
587
|
-
function pad2(n) {
|
|
588
|
-
return String(n).padStart(2, "0");
|
|
589
|
-
}
|
|
590
|
-
function toDateStr(ms) {
|
|
591
|
-
const d = new Date(ms);
|
|
592
|
-
return `${d.getUTCFullYear()}-${pad2(d.getUTCMonth() + 1)}-${pad2(d.getUTCDate())}`;
|
|
593
|
-
}
|
|
594
|
-
function startOfUtcDay(ms) {
|
|
595
|
-
return Math.floor(ms / MS_PER_DAY) * MS_PER_DAY;
|
|
596
|
-
}
|
|
597
698
|
function getCrashlyticsWindow(options, lookbackDays, now = Date.now()) {
|
|
598
699
|
const endMs = startOfUtcDay(now) + MS_PER_DAY;
|
|
599
700
|
let days = lookbackDays;
|
|
600
701
|
if (options.mode === "latest") {
|
|
601
702
|
days = INCREMENTAL_LOOKBACK_DAYS;
|
|
602
703
|
} else if (options.since !== void 0) {
|
|
603
|
-
const sinceMs =
|
|
704
|
+
const sinceMs = parseEpoch2(options.since, "iso");
|
|
604
705
|
if (sinceMs !== null) {
|
|
605
706
|
const elapsed = Math.ceil((now - sinceMs) / MS_PER_DAY);
|
|
606
707
|
days = Math.min(
|
|
@@ -615,14 +716,10 @@ function getCrashlyticsWindow(options, lookbackDays, now = Date.now()) {
|
|
|
615
716
|
};
|
|
616
717
|
}
|
|
617
718
|
function buildCrashesSamplesFromBqResponse(response) {
|
|
618
|
-
const
|
|
619
|
-
const fieldIndex = {};
|
|
620
|
-
schema.forEach((field, idx) => {
|
|
621
|
-
fieldIndex[field.name] = idx;
|
|
622
|
-
});
|
|
719
|
+
const fieldIndex = indexBqFields(response);
|
|
623
720
|
const samples = [];
|
|
624
721
|
for (const row of response.rows ?? []) {
|
|
625
|
-
const dateValue =
|
|
722
|
+
const dateValue = readBqCell(row.f, fieldIndex, "date");
|
|
626
723
|
if (dateValue === null) {
|
|
627
724
|
continue;
|
|
628
725
|
}
|
|
@@ -630,7 +727,7 @@ function buildCrashesSamplesFromBqResponse(response) {
|
|
|
630
727
|
if (ts === null) {
|
|
631
728
|
continue;
|
|
632
729
|
}
|
|
633
|
-
const crashesRaw =
|
|
730
|
+
const crashesRaw = readBqCell(row.f, fieldIndex, "crashes");
|
|
634
731
|
if (crashesRaw === null) {
|
|
635
732
|
continue;
|
|
636
733
|
}
|
|
@@ -638,8 +735,8 @@ function buildCrashesSamplesFromBqResponse(response) {
|
|
|
638
735
|
if (!Number.isFinite(crashes)) {
|
|
639
736
|
continue;
|
|
640
737
|
}
|
|
641
|
-
const crashingUsersRaw =
|
|
642
|
-
const totalUsersRaw =
|
|
738
|
+
const crashingUsersRaw = readBqCell(row.f, fieldIndex, "crashing_users");
|
|
739
|
+
const totalUsersRaw = readBqCell(row.f, fieldIndex, "total_users");
|
|
643
740
|
const crashingUsers = crashingUsersRaw !== null ? Number.parseFloat(crashingUsersRaw) : NaN;
|
|
644
741
|
const totalUsers = totalUsersRaw !== null ? Number.parseFloat(totalUsersRaw) : NaN;
|
|
645
742
|
let crashFreeRate = null;
|
|
@@ -648,9 +745,9 @@ function buildCrashesSamplesFromBqResponse(response) {
|
|
|
648
745
|
crashFreeRate = Math.max(0, Math.min(1, rate));
|
|
649
746
|
}
|
|
650
747
|
const attributes = {};
|
|
651
|
-
const appId =
|
|
652
|
-
const platform =
|
|
653
|
-
const version =
|
|
748
|
+
const appId = readBqCell(row.f, fieldIndex, "app_id");
|
|
749
|
+
const platform = readBqCell(row.f, fieldIndex, "platform");
|
|
750
|
+
const version = readBqCell(row.f, fieldIndex, "version");
|
|
654
751
|
attributes["app_id"] = appId;
|
|
655
752
|
attributes["platform"] = platform;
|
|
656
753
|
attributes["version"] = version;
|
|
@@ -666,30 +763,26 @@ function buildCrashesSamplesFromBqResponse(response) {
|
|
|
666
763
|
return samples;
|
|
667
764
|
}
|
|
668
765
|
function buildTopIssuesEntitiesFromBqResponse(response) {
|
|
669
|
-
const
|
|
670
|
-
const fieldIndex = {};
|
|
671
|
-
schema.forEach((field, idx) => {
|
|
672
|
-
fieldIndex[field.name] = idx;
|
|
673
|
-
});
|
|
766
|
+
const fieldIndex = indexBqFields(response);
|
|
674
767
|
const entities = [];
|
|
675
768
|
for (const row of response.rows ?? []) {
|
|
676
|
-
const issueId =
|
|
769
|
+
const issueId = readBqCell(row.f, fieldIndex, "issue_id");
|
|
677
770
|
if (issueId === null || issueId.length === 0) {
|
|
678
771
|
continue;
|
|
679
772
|
}
|
|
680
|
-
const eventCountRaw =
|
|
681
|
-
const userCountRaw =
|
|
773
|
+
const eventCountRaw = readBqCell(row.f, fieldIndex, "event_count");
|
|
774
|
+
const userCountRaw = readBqCell(row.f, fieldIndex, "user_count");
|
|
682
775
|
const eventCount = eventCountRaw !== null ? Number.parseFloat(eventCountRaw) : NaN;
|
|
683
776
|
const userCount = userCountRaw !== null ? Number.parseFloat(userCountRaw) : NaN;
|
|
684
|
-
const lastSeenRaw =
|
|
777
|
+
const lastSeenRaw = readBqCell(row.f, fieldIndex, "last_seen");
|
|
685
778
|
const lastSeenMs = lastSeenRaw !== null ? parseBqDateOrEpoch(lastSeenRaw) : null;
|
|
686
779
|
const updatedAt = lastSeenMs ?? Date.now();
|
|
687
780
|
const attributes = {
|
|
688
781
|
issue_id: issueId,
|
|
689
|
-
title:
|
|
690
|
-
subtitle:
|
|
691
|
-
app_id:
|
|
692
|
-
platform:
|
|
782
|
+
title: readBqCell(row.f, fieldIndex, "title"),
|
|
783
|
+
subtitle: readBqCell(row.f, fieldIndex, "subtitle"),
|
|
784
|
+
app_id: readBqCell(row.f, fieldIndex, "app_id"),
|
|
785
|
+
platform: readBqCell(row.f, fieldIndex, "platform"),
|
|
693
786
|
event_count: Number.isFinite(eventCount) ? eventCount : 0,
|
|
694
787
|
user_count: Number.isFinite(userCount) ? userCount : 0,
|
|
695
788
|
last_seen: lastSeenMs !== null ? new Date(lastSeenMs).toISOString() : null
|
|
@@ -703,28 +796,6 @@ function buildTopIssuesEntitiesFromBqResponse(response) {
|
|
|
703
796
|
}
|
|
704
797
|
return entities;
|
|
705
798
|
}
|
|
706
|
-
function readCell(cells, fieldIndex, name) {
|
|
707
|
-
const idx = fieldIndex[name];
|
|
708
|
-
if (idx === void 0) {
|
|
709
|
-
return null;
|
|
710
|
-
}
|
|
711
|
-
const raw = cells[idx]?.v;
|
|
712
|
-
if (raw === void 0 || raw === null) {
|
|
713
|
-
return null;
|
|
714
|
-
}
|
|
715
|
-
return raw;
|
|
716
|
-
}
|
|
717
|
-
function parseBqDateOrEpoch(value) {
|
|
718
|
-
const dateMatch = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value);
|
|
719
|
-
if (dateMatch) {
|
|
720
|
-
return Date.UTC(
|
|
721
|
-
Number(dateMatch[1]),
|
|
722
|
-
Number(dateMatch[2]) - 1,
|
|
723
|
-
Number(dateMatch[3])
|
|
724
|
-
);
|
|
725
|
-
}
|
|
726
|
-
return parseEpoch(value, "iso");
|
|
727
|
-
}
|
|
728
799
|
|
|
729
800
|
// src/index.ts
|
|
730
801
|
var index_default = FirebaseCrashlyticsConnector;
|