@hot-updater/plugin-insights 1.0.0-rc.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +33 -0
- package/README.md +24 -0
- package/client/package.json +7 -0
- package/dist/client/eventId.cjs +181 -0
- package/dist/client/eventId.mjs +179 -0
- package/dist/client/index.cjs +234 -0
- package/dist/client/index.d.cts +42 -0
- package/dist/client/index.d.mts +42 -0
- package/dist/client/index.mjs +233 -0
- package/dist/client/sender.cjs +103 -0
- package/dist/client/sender.mjs +103 -0
- package/dist/client/state.cjs +121 -0
- package/dist/client/state.mjs +119 -0
- package/dist/internal.cjs +9 -0
- package/dist/internal.d.cts +5 -0
- package/dist/internal.d.mts +5 -0
- package/dist/internal.mjs +5 -0
- package/dist/server/contract.cjs +193 -0
- package/dist/server/contract.mjs +189 -0
- package/dist/server/domain.d.cts +149 -0
- package/dist/server/domain.d.mts +149 -0
- package/dist/server/errors.cjs +15 -0
- package/dist/server/errors.d.cts +6 -0
- package/dist/server/errors.d.mts +6 -0
- package/dist/server/errors.mjs +14 -0
- package/dist/server/eventInput.cjs +284 -0
- package/dist/server/eventInput.d.cts +6 -0
- package/dist/server/eventInput.d.mts +6 -0
- package/dist/server/eventInput.mjs +282 -0
- package/dist/server/eventRow.cjs +49 -0
- package/dist/server/eventRow.d.cts +64 -0
- package/dist/server/eventRow.d.mts +64 -0
- package/dist/server/eventRow.mjs +47 -0
- package/dist/server/index.cjs +128 -0
- package/dist/server/index.d.cts +394 -0
- package/dist/server/index.d.mts +394 -0
- package/dist/server/index.mjs +121 -0
- package/dist/server/model.cjs +17 -0
- package/dist/server/model.d.cts +10 -0
- package/dist/server/model.d.mts +10 -0
- package/dist/server/model.mjs +17 -0
- package/dist/server/modelTypes.d.cts +176 -0
- package/dist/server/modelTypes.d.mts +176 -0
- package/dist/server/overview.cjs +109 -0
- package/dist/server/overview.d.cts +15 -0
- package/dist/server/overview.d.mts +15 -0
- package/dist/server/overview.mjs +107 -0
- package/dist/server/provider.cjs +375 -0
- package/dist/server/provider.d.cts +6 -0
- package/dist/server/provider.d.mts +6 -0
- package/dist/server/provider.mjs +374 -0
- package/dist/server/queryInput.cjs +93 -0
- package/dist/server/queryInput.mjs +91 -0
- package/dist/server/reads.cjs +681 -0
- package/dist/server/reads.d.cts +60 -0
- package/dist/server/reads.d.mts +60 -0
- package/dist/server/reads.mjs +674 -0
- package/dist/server/recordEvent.cjs +319 -0
- package/dist/server/recordEvent.d.cts +17 -0
- package/dist/server/recordEvent.d.mts +17 -0
- package/dist/server/recordEvent.mjs +316 -0
- package/dist/server/routes.cjs +96 -0
- package/dist/server/routes.d.cts +36 -0
- package/dist/server/routes.d.mts +36 -0
- package/dist/server/routes.mjs +95 -0
- package/dist/server/schema.cjs +489 -0
- package/dist/server/schema.d.cts +706 -0
- package/dist/server/schema.d.mts +706 -0
- package/dist/server/schema.mjs +483 -0
- package/dist/server/testing/eventRow.d.mts +64 -0
- package/dist/server/testing/expectInsightsIndex.mjs +18 -0
- package/dist/server/testing/fixtures.d.mts +8 -0
- package/dist/server/testing/fixtures.mjs +25 -0
- package/dist/server/testing/index.d.mts +24 -0
- package/dist/server/testing/index.mjs +25 -0
- package/dist/server/testing/insightsModelTests.mjs +714 -0
- package/dist/server/testing/modelTypes.d.mts +176 -0
- package/dist/server/testing/setupInsightsHttpTestSuite.d.mts +11 -0
- package/dist/server/testing/setupInsightsHttpTestSuite.mjs +107 -0
- package/dist/server/testing/setupInsightsModelTestSuite.d.mts +11 -0
- package/dist/server/testing/setupInsightsModelTestSuite.mjs +31 -0
- package/dist/server/testing/types.d.mts +26 -0
- package/dist/server/types.d.cts +33 -0
- package/dist/server/types.d.mts +33 -0
- package/package.json +68 -0
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
const require_contract = require("./contract.cjs");
|
|
2
|
+
const require_overview = require("./overview.cjs");
|
|
3
|
+
const require_schema = require("./schema.cjs");
|
|
4
|
+
let _hot_updater_plugin_core_internal = require("@hot-updater/plugin-core/internal");
|
|
5
|
+
//#region src/server/recordEvent.ts
|
|
6
|
+
const hourOf = (ms) => ms - ms % require_schema.HOUR_MS;
|
|
7
|
+
const dayOf = (ms) => ms - ms % require_schema.DAY_MS;
|
|
8
|
+
/** The `identity` of an overview or sketch row: its scope and period, hashed. */
|
|
9
|
+
const insightsIdentity = (identity) => require_overview.insightsOverviewId({
|
|
10
|
+
...identity,
|
|
11
|
+
scopeKind: identity.scopeKind,
|
|
12
|
+
periodKind: identity.periodKind,
|
|
13
|
+
bucketStartMs: 0
|
|
14
|
+
});
|
|
15
|
+
/**
|
|
16
|
+
* The bucket a head's gauges count it in, the UTC day of its event, and the
|
|
17
|
+
* release its distribution row names: the one it runs, which a download
|
|
18
|
+
* leaves at its source.
|
|
19
|
+
*/
|
|
20
|
+
const gaugeSlot = (head) => ({
|
|
21
|
+
bucket: dayOf(head.received_at_ms),
|
|
22
|
+
releaseId: head.type === "UPDATE_DOWNLOADED" ? head.from_release_id : head.to_release_id
|
|
23
|
+
});
|
|
24
|
+
/**
|
|
25
|
+
* The `insights_latest_by_bundle` field of a head's (from, to) pair. A head
|
|
26
|
+
* matches a `from` and a `to` predicate of one type at once only through its
|
|
27
|
+
* pair, so this gauge is what a count subtracts to count it once.
|
|
28
|
+
*/
|
|
29
|
+
const PAIR_FIELD = "from_to";
|
|
30
|
+
/** The pair gauge's `bundle_id`: a hash of the two bundle ids, which fit no single 36-character column. */
|
|
31
|
+
const bundlePairKey = (from, to) => require_overview.insightsKey(`${from.length}:${from}${to.length}:${to}`);
|
|
32
|
+
/**
|
|
33
|
+
* Moves a head's gauges: the distribution row, one row per bundle it
|
|
34
|
+
* references, and its pair; -1 takes back the head an event replaces.
|
|
35
|
+
*/
|
|
36
|
+
const countHead = (tx, head, delta) => {
|
|
37
|
+
const { bucket, releaseId } = gaugeSlot(head);
|
|
38
|
+
const shardBy = head.install_id;
|
|
39
|
+
tx.aggregate("insights_distribution", {
|
|
40
|
+
channel: head.channel,
|
|
41
|
+
platform: head.platform,
|
|
42
|
+
app_version: head.app_version,
|
|
43
|
+
release_id: releaseId ?? "",
|
|
44
|
+
bucket_start_ms: bucket
|
|
45
|
+
}, { latest_installations: delta }, { shardBy });
|
|
46
|
+
for (const field of ["from_bundle_id", "to_bundle_id"]) {
|
|
47
|
+
const bundleId = head[field];
|
|
48
|
+
if (bundleId === null) continue;
|
|
49
|
+
tx.aggregate("insights_latest_by_bundle", {
|
|
50
|
+
platform: head.platform,
|
|
51
|
+
channel: head.channel,
|
|
52
|
+
bundle_field: field,
|
|
53
|
+
bundle_id: bundleId,
|
|
54
|
+
type: head.type,
|
|
55
|
+
bucket_start_ms: bucket
|
|
56
|
+
}, { installations: delta }, { shardBy });
|
|
57
|
+
}
|
|
58
|
+
if (head.from_bundle_id !== null) tx.aggregate("insights_latest_by_bundle", {
|
|
59
|
+
platform: head.platform,
|
|
60
|
+
channel: head.channel,
|
|
61
|
+
bundle_field: PAIR_FIELD,
|
|
62
|
+
bundle_id: bundlePairKey(head.from_bundle_id, head.to_bundle_id),
|
|
63
|
+
type: head.type,
|
|
64
|
+
bucket_start_ms: bucket
|
|
65
|
+
}, { installations: delta }, { shardBy });
|
|
66
|
+
};
|
|
67
|
+
/**
|
|
68
|
+
* Where each period's rows go, each with its retention: hours for 90 days,
|
|
69
|
+
* days for 13 months, and a release's lifetime counters kept.
|
|
70
|
+
*/
|
|
71
|
+
const PERIOD_MODELS = {
|
|
72
|
+
hour: {
|
|
73
|
+
counters: "insights_overview",
|
|
74
|
+
sketches: "insights_sketches"
|
|
75
|
+
},
|
|
76
|
+
day: {
|
|
77
|
+
counters: "insights_overview_daily",
|
|
78
|
+
sketches: "insights_sketches_daily"
|
|
79
|
+
},
|
|
80
|
+
lifetime: {
|
|
81
|
+
counters: "insights_overview_lifetime",
|
|
82
|
+
sketches: void 0
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
/** A download's patch counters: a patch delivered it, or it fell back from one. */
|
|
86
|
+
const patchCounters = (event) => event.type !== "UPDATE_DOWNLOADED" ? {} : {
|
|
87
|
+
...event.metadata.delivery === "patch" ? { patch_downloads: 1 } : {},
|
|
88
|
+
...event.metadata.patch_fallback === true ? { patch_fallbacks: 1 } : {}
|
|
89
|
+
};
|
|
90
|
+
/**
|
|
91
|
+
* Counters and sketches for one event: release, channel, and usage rows, with
|
|
92
|
+
* day rollups for channel and usage. Usage rows are written for the event's
|
|
93
|
+
* platform only, since a read for every platform merges the ios and android
|
|
94
|
+
* sketches; a channel's active installations come from its usage rows, so
|
|
95
|
+
* channel rows keep no sketch of their own. A download's patch counters go
|
|
96
|
+
* in the rows that count it.
|
|
97
|
+
*/
|
|
98
|
+
const countEvent = (tx, event) => {
|
|
99
|
+
const shardBy = event.install_id;
|
|
100
|
+
for (const delta of require_overview.insightsOverviewDeltas(event)) {
|
|
101
|
+
if (delta.identity.platform === "all") continue;
|
|
102
|
+
const { bucketStartMs, ...parts } = delta.identity;
|
|
103
|
+
const periods = parts.scopeKind === "channel" || parts.scopeKind === "usage" ? [["hour", bucketStartMs], ["day", dayOf(event.received_at_ms)]] : [[parts.periodKind, bucketStartMs]];
|
|
104
|
+
for (const [periodKind, bucket] of periods) {
|
|
105
|
+
const models = PERIOD_MODELS[periodKind];
|
|
106
|
+
const key = {
|
|
107
|
+
identity: insightsIdentity({
|
|
108
|
+
...parts,
|
|
109
|
+
periodKind
|
|
110
|
+
}),
|
|
111
|
+
bucket_start_ms: bucket
|
|
112
|
+
};
|
|
113
|
+
const counters = {
|
|
114
|
+
...Object.fromEntries([
|
|
115
|
+
["downloads", delta.downloads],
|
|
116
|
+
["launches", delta.launches],
|
|
117
|
+
["failed_launches", delta.failedLaunches]
|
|
118
|
+
].filter(([, value]) => value !== 0)),
|
|
119
|
+
...delta.downloads === 0 ? {} : patchCounters(event)
|
|
120
|
+
};
|
|
121
|
+
if (Object.keys(counters).length > 0) tx.aggregate(models.counters, key, counters, { shardBy });
|
|
122
|
+
const sketches = {
|
|
123
|
+
...delta.launchIdentity === void 0 || parts.scopeKind === "channel" ? {} : { launch_users: (0, _hot_updater_plugin_core_internal.addDistinct)(null, delta.launchIdentity) },
|
|
124
|
+
...delta.activityIdentity === void 0 ? {} : { activity_users: (0, _hot_updater_plugin_core_internal.addDistinct)(null, delta.activityIdentity) }
|
|
125
|
+
};
|
|
126
|
+
if (models.sketches !== void 0 && Object.keys(sketches).length > 0) tx.aggregate(models.sketches, key, sketches, { shardBy });
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* A stored event's outcome rows: its bundle filter's hour, which a failed
|
|
132
|
+
* check has none of, and every stored event's UTC day.
|
|
133
|
+
*/
|
|
134
|
+
const countOutcome = (tx, event) => {
|
|
135
|
+
const shardBy = event.install_id;
|
|
136
|
+
if (!require_schema.isFailedCheck(event)) tx.aggregate("insights_outcomes", {
|
|
137
|
+
platform: event.platform,
|
|
138
|
+
channel: event.channel,
|
|
139
|
+
type: event.type,
|
|
140
|
+
bundle_ref: event.type === "RECOVERED" ? `from:${event.from_bundle_id}` : `to:${event.to_bundle_id}`,
|
|
141
|
+
bucket_start_ms: hourOf(event.received_at_ms)
|
|
142
|
+
}, { events: 1 }, { shardBy });
|
|
143
|
+
tx.aggregate("insights_outcomes", {
|
|
144
|
+
...require_schema.DAILY_EVENTS,
|
|
145
|
+
bucket_start_ms: dayOf(event.received_at_ms)
|
|
146
|
+
}, { events: 1 }, { shardBy });
|
|
147
|
+
};
|
|
148
|
+
/** The row check requires an UPDATE_FAILED row's failure; its type leaves it optional. */
|
|
149
|
+
const UNKNOWN_FAILURE = {
|
|
150
|
+
stage: "unknown",
|
|
151
|
+
reason: "unknown"
|
|
152
|
+
};
|
|
153
|
+
/** A failure's `detail`: what else the client knew, as a JSON array. */
|
|
154
|
+
const failureDetail = (failure) => JSON.stringify([
|
|
155
|
+
failure.resource ?? null,
|
|
156
|
+
failure.http_status ?? null,
|
|
157
|
+
failure.origin_code ?? null,
|
|
158
|
+
failure.transport ?? null
|
|
159
|
+
]);
|
|
160
|
+
/**
|
|
161
|
+
* An update failure's sketches, breakdown row, and lifetime counter. Its
|
|
162
|
+
* hourly breakdown row is what windowed reads sum, so no hourly or daily
|
|
163
|
+
* counter repeats it. A failed check counts for its channel, in the `check`
|
|
164
|
+
* sketch of installations. Any other failure counts in the channel's
|
|
165
|
+
* `failure` sketch, and for its target release, when it names one, in the
|
|
166
|
+
* release's hourly and lifetime `failure` sketches and its lifetime
|
|
167
|
+
* `failed_updates`, which the bundle detail reads. A failure is no launch
|
|
168
|
+
* and no activity: the installation reports its launch that day.
|
|
169
|
+
*/
|
|
170
|
+
const countFailure = (tx, event) => {
|
|
171
|
+
const shardBy = event.install_id;
|
|
172
|
+
const failure = event.metadata.failure ?? UNKNOWN_FAILURE;
|
|
173
|
+
const check = failure.stage === "check";
|
|
174
|
+
const hour = hourOf(event.received_at_ms);
|
|
175
|
+
const scope = {
|
|
176
|
+
channel: event.channel,
|
|
177
|
+
platform: event.platform,
|
|
178
|
+
appVersionKind: "all",
|
|
179
|
+
appVersion: ""
|
|
180
|
+
};
|
|
181
|
+
const channel = {
|
|
182
|
+
...scope,
|
|
183
|
+
releaseKind: "all",
|
|
184
|
+
releaseId: ""
|
|
185
|
+
};
|
|
186
|
+
const users = { failed_users: (0, _hot_updater_plugin_core_internal.addDistinct)(null, event.install_id) };
|
|
187
|
+
for (const [periodKind, bucket] of [["hour", hour], ["day", dayOf(event.received_at_ms)]]) {
|
|
188
|
+
const identity = insightsIdentity({
|
|
189
|
+
...channel,
|
|
190
|
+
scopeKind: check ? "check" : "failure",
|
|
191
|
+
periodKind
|
|
192
|
+
});
|
|
193
|
+
tx.aggregate(PERIOD_MODELS[periodKind].sketches, {
|
|
194
|
+
identity,
|
|
195
|
+
bucket_start_ms: bucket
|
|
196
|
+
}, users, { shardBy });
|
|
197
|
+
}
|
|
198
|
+
const releaseId = check ? null : event.to_release_id;
|
|
199
|
+
if (releaseId !== null) {
|
|
200
|
+
const release = {
|
|
201
|
+
...scope,
|
|
202
|
+
releaseKind: "specific",
|
|
203
|
+
releaseId
|
|
204
|
+
};
|
|
205
|
+
for (const [periodKind, bucket, sketches] of [[
|
|
206
|
+
"hour",
|
|
207
|
+
hour,
|
|
208
|
+
"insights_sketches"
|
|
209
|
+
], [
|
|
210
|
+
"lifetime",
|
|
211
|
+
0,
|
|
212
|
+
"insights_sketches_lifetime"
|
|
213
|
+
]]) {
|
|
214
|
+
const key = (scopeKind) => ({
|
|
215
|
+
identity: insightsIdentity({
|
|
216
|
+
...release,
|
|
217
|
+
scopeKind,
|
|
218
|
+
periodKind
|
|
219
|
+
}),
|
|
220
|
+
bucket_start_ms: bucket
|
|
221
|
+
});
|
|
222
|
+
tx.aggregate(sketches, key("failure"), users, { shardBy });
|
|
223
|
+
}
|
|
224
|
+
tx.aggregate("insights_overview_lifetime", {
|
|
225
|
+
identity: insightsIdentity({
|
|
226
|
+
...release,
|
|
227
|
+
scopeKind: "release",
|
|
228
|
+
periodKind: "lifetime"
|
|
229
|
+
}),
|
|
230
|
+
bucket_start_ms: 0
|
|
231
|
+
}, { failed_updates: 1 }, { shardBy });
|
|
232
|
+
}
|
|
233
|
+
tx.aggregate("insights_failures", {
|
|
234
|
+
platform: event.platform,
|
|
235
|
+
channel: event.channel,
|
|
236
|
+
bucket_start_ms: hour,
|
|
237
|
+
release_id: releaseId ?? "",
|
|
238
|
+
stage: failure.stage,
|
|
239
|
+
reason: failure.reason,
|
|
240
|
+
detail: failureDetail(failure)
|
|
241
|
+
}, { events: 1 }, { shardBy });
|
|
242
|
+
};
|
|
243
|
+
/**
|
|
244
|
+
* A recovery's exit reason, which Android 11+ reports, in the breakdown of
|
|
245
|
+
* the release whose launch failed: stage `launch`, the reason as its reason.
|
|
246
|
+
*/
|
|
247
|
+
const countExit = (tx, event) => {
|
|
248
|
+
const exit = event.metadata.previous_process_exit;
|
|
249
|
+
if (event.type !== "RECOVERED" || exit === void 0) return;
|
|
250
|
+
tx.aggregate("insights_failures", {
|
|
251
|
+
platform: event.platform,
|
|
252
|
+
channel: event.channel,
|
|
253
|
+
bucket_start_ms: hourOf(event.received_at_ms),
|
|
254
|
+
release_id: event.from_release_id ?? "",
|
|
255
|
+
stage: "launch",
|
|
256
|
+
reason: exit,
|
|
257
|
+
detail: ""
|
|
258
|
+
}, { events: 1 }, { shardBy: event.install_id });
|
|
259
|
+
};
|
|
260
|
+
const isNewer = (event, head) => event.received_at_ms !== head.received_at_ms ? event.received_at_ms > head.received_at_ms : (0, _hot_updater_plugin_core_internal.compareUtf8)(event.id, head.id) > 0;
|
|
261
|
+
/** What an UNCHANGED report must share with its installation's head to repeat it. */
|
|
262
|
+
const REPEATED_FIELDS = [
|
|
263
|
+
"channel",
|
|
264
|
+
"platform",
|
|
265
|
+
"app_version",
|
|
266
|
+
"to_bundle_id",
|
|
267
|
+
"to_release_id",
|
|
268
|
+
"user_id"
|
|
269
|
+
];
|
|
270
|
+
/**
|
|
271
|
+
* An UNCHANGED report that repeats its installation's head from the same UTC
|
|
272
|
+
* day: the installation already counts as active that day, on that bundle,
|
|
273
|
+
* so the report records nothing. A download's head does not count, because
|
|
274
|
+
* its installation still ran the bundle it downloaded from.
|
|
275
|
+
*/
|
|
276
|
+
const repeatsHead = (event, head) => event.type === "UNCHANGED" && head.type !== "UPDATE_DOWNLOADED" && dayOf(event.received_at_ms) === dayOf(head.received_at_ms) && REPEATED_FIELDS.every((field) => event[field] === head[field]);
|
|
277
|
+
/**
|
|
278
|
+
* Records one event in one transaction: one batch read of the event and its
|
|
279
|
+
* installation's head, one of the gauge and sketch rows it changes, then one
|
|
280
|
+
* write. On a database that batches aggregates (DynamoDB, Firestore), the
|
|
281
|
+
* write holds the event's rows and one log row instead, and a compaction
|
|
282
|
+
* reads and writes the aggregate rows. A stored event's id changes nothing
|
|
283
|
+
* when repeated, whichever installation sends it, and neither does the id
|
|
284
|
+
* of the installation's head. An UNCHANGED report that repeats its head on
|
|
285
|
+
* the same UTC day writes nothing at all. An older event still counts in its
|
|
286
|
+
* own hour but never replaces the head. An update failure is stored and
|
|
287
|
+
* counted, but changes what no installation runs, so it moves no head.
|
|
288
|
+
*/
|
|
289
|
+
const recordEvent = (db, event) => {
|
|
290
|
+
require_contract.assertBundleEventRow(event);
|
|
291
|
+
return db.transaction(async (tx) => {
|
|
292
|
+
const [existing, previous] = await Promise.all([tx.findOne("bundle_events", { id: event.id }), tx.findOne("bundle_event_heads", { install_id: event.install_id })]);
|
|
293
|
+
if (existing !== null || previous?.id === event.id) return;
|
|
294
|
+
if (previous !== null && repeatsHead(event, previous)) return;
|
|
295
|
+
if (event.type !== "UNCHANGED") {
|
|
296
|
+
tx.create("bundle_events", event);
|
|
297
|
+
countOutcome(tx, event);
|
|
298
|
+
}
|
|
299
|
+
if (event.type === "UPDATE_FAILED") {
|
|
300
|
+
countFailure(tx, event);
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
countEvent(tx, event);
|
|
304
|
+
countExit(tx, event);
|
|
305
|
+
if (previous !== null && !isNewer(event, previous)) return;
|
|
306
|
+
if (previous === null) tx.create("bundle_event_heads", event);
|
|
307
|
+
else {
|
|
308
|
+
const { install_id: _, ...fields } = event;
|
|
309
|
+
countHead(tx, previous, -1);
|
|
310
|
+
tx.update("bundle_event_heads", previous, fields);
|
|
311
|
+
}
|
|
312
|
+
countHead(tx, event, 1);
|
|
313
|
+
});
|
|
314
|
+
};
|
|
315
|
+
//#endregion
|
|
316
|
+
exports.PAIR_FIELD = PAIR_FIELD;
|
|
317
|
+
exports.bundlePairKey = bundlePairKey;
|
|
318
|
+
exports.insightsIdentity = insightsIdentity;
|
|
319
|
+
exports.recordEvent = recordEvent;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import "./eventRow.cjs";
|
|
2
|
+
import { InsightsOverviewIdentity } from "./overview.cjs";
|
|
3
|
+
import { HotUpdaterDatabase } from "@hot-updater/plugin-core/server-plugin";
|
|
4
|
+
//#region src/server/recordEvent.d.ts
|
|
5
|
+
/**
|
|
6
|
+
* An overview or sketch row's scope and period; day periods roll up channel
|
|
7
|
+
* and usage rows. The `failure` and `check` scopes key the sketches of
|
|
8
|
+
* installations whose update, or whose update check, failed.
|
|
9
|
+
*/
|
|
10
|
+
type InsightsIdentityParts = Omit<InsightsOverviewIdentity, "bucketStartMs" | "periodKind" | "scopeKind"> & {
|
|
11
|
+
readonly scopeKind: InsightsOverviewIdentity["scopeKind"] | "failure" | "check";
|
|
12
|
+
readonly periodKind: InsightsOverviewIdentity["periodKind"] | "day";
|
|
13
|
+
};
|
|
14
|
+
/** The `identity` of an overview or sketch row: its scope and period, hashed. */
|
|
15
|
+
declare const insightsIdentity: (identity: InsightsIdentityParts) => string;
|
|
16
|
+
//#endregion
|
|
17
|
+
export { InsightsIdentityParts, insightsIdentity };
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import "./eventRow.mjs";
|
|
2
|
+
import { InsightsOverviewIdentity } from "./overview.mjs";
|
|
3
|
+
import { HotUpdaterDatabase } from "@hot-updater/plugin-core/server-plugin";
|
|
4
|
+
//#region src/server/recordEvent.d.ts
|
|
5
|
+
/**
|
|
6
|
+
* An overview or sketch row's scope and period; day periods roll up channel
|
|
7
|
+
* and usage rows. The `failure` and `check` scopes key the sketches of
|
|
8
|
+
* installations whose update, or whose update check, failed.
|
|
9
|
+
*/
|
|
10
|
+
type InsightsIdentityParts = Omit<InsightsOverviewIdentity, "bucketStartMs" | "periodKind" | "scopeKind"> & {
|
|
11
|
+
readonly scopeKind: InsightsOverviewIdentity["scopeKind"] | "failure" | "check";
|
|
12
|
+
readonly periodKind: InsightsOverviewIdentity["periodKind"] | "day";
|
|
13
|
+
};
|
|
14
|
+
/** The `identity` of an overview or sketch row: its scope and period, hashed. */
|
|
15
|
+
declare const insightsIdentity: (identity: InsightsIdentityParts) => string;
|
|
16
|
+
//#endregion
|
|
17
|
+
export { InsightsIdentityParts, insightsIdentity };
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
import { assertBundleEventRow } from "./contract.mjs";
|
|
2
|
+
import { insightsKey, insightsOverviewDeltas, insightsOverviewId } from "./overview.mjs";
|
|
3
|
+
import { DAILY_EVENTS, DAY_MS, HOUR_MS, isFailedCheck } from "./schema.mjs";
|
|
4
|
+
import { addDistinct, compareUtf8 } from "@hot-updater/plugin-core/internal";
|
|
5
|
+
//#region src/server/recordEvent.ts
|
|
6
|
+
const hourOf = (ms) => ms - ms % HOUR_MS;
|
|
7
|
+
const dayOf = (ms) => ms - ms % DAY_MS;
|
|
8
|
+
/** The `identity` of an overview or sketch row: its scope and period, hashed. */
|
|
9
|
+
const insightsIdentity = (identity) => insightsOverviewId({
|
|
10
|
+
...identity,
|
|
11
|
+
scopeKind: identity.scopeKind,
|
|
12
|
+
periodKind: identity.periodKind,
|
|
13
|
+
bucketStartMs: 0
|
|
14
|
+
});
|
|
15
|
+
/**
|
|
16
|
+
* The bucket a head's gauges count it in, the UTC day of its event, and the
|
|
17
|
+
* release its distribution row names: the one it runs, which a download
|
|
18
|
+
* leaves at its source.
|
|
19
|
+
*/
|
|
20
|
+
const gaugeSlot = (head) => ({
|
|
21
|
+
bucket: dayOf(head.received_at_ms),
|
|
22
|
+
releaseId: head.type === "UPDATE_DOWNLOADED" ? head.from_release_id : head.to_release_id
|
|
23
|
+
});
|
|
24
|
+
/**
|
|
25
|
+
* The `insights_latest_by_bundle` field of a head's (from, to) pair. A head
|
|
26
|
+
* matches a `from` and a `to` predicate of one type at once only through its
|
|
27
|
+
* pair, so this gauge is what a count subtracts to count it once.
|
|
28
|
+
*/
|
|
29
|
+
const PAIR_FIELD = "from_to";
|
|
30
|
+
/** The pair gauge's `bundle_id`: a hash of the two bundle ids, which fit no single 36-character column. */
|
|
31
|
+
const bundlePairKey = (from, to) => insightsKey(`${from.length}:${from}${to.length}:${to}`);
|
|
32
|
+
/**
|
|
33
|
+
* Moves a head's gauges: the distribution row, one row per bundle it
|
|
34
|
+
* references, and its pair; -1 takes back the head an event replaces.
|
|
35
|
+
*/
|
|
36
|
+
const countHead = (tx, head, delta) => {
|
|
37
|
+
const { bucket, releaseId } = gaugeSlot(head);
|
|
38
|
+
const shardBy = head.install_id;
|
|
39
|
+
tx.aggregate("insights_distribution", {
|
|
40
|
+
channel: head.channel,
|
|
41
|
+
platform: head.platform,
|
|
42
|
+
app_version: head.app_version,
|
|
43
|
+
release_id: releaseId ?? "",
|
|
44
|
+
bucket_start_ms: bucket
|
|
45
|
+
}, { latest_installations: delta }, { shardBy });
|
|
46
|
+
for (const field of ["from_bundle_id", "to_bundle_id"]) {
|
|
47
|
+
const bundleId = head[field];
|
|
48
|
+
if (bundleId === null) continue;
|
|
49
|
+
tx.aggregate("insights_latest_by_bundle", {
|
|
50
|
+
platform: head.platform,
|
|
51
|
+
channel: head.channel,
|
|
52
|
+
bundle_field: field,
|
|
53
|
+
bundle_id: bundleId,
|
|
54
|
+
type: head.type,
|
|
55
|
+
bucket_start_ms: bucket
|
|
56
|
+
}, { installations: delta }, { shardBy });
|
|
57
|
+
}
|
|
58
|
+
if (head.from_bundle_id !== null) tx.aggregate("insights_latest_by_bundle", {
|
|
59
|
+
platform: head.platform,
|
|
60
|
+
channel: head.channel,
|
|
61
|
+
bundle_field: PAIR_FIELD,
|
|
62
|
+
bundle_id: bundlePairKey(head.from_bundle_id, head.to_bundle_id),
|
|
63
|
+
type: head.type,
|
|
64
|
+
bucket_start_ms: bucket
|
|
65
|
+
}, { installations: delta }, { shardBy });
|
|
66
|
+
};
|
|
67
|
+
/**
|
|
68
|
+
* Where each period's rows go, each with its retention: hours for 90 days,
|
|
69
|
+
* days for 13 months, and a release's lifetime counters kept.
|
|
70
|
+
*/
|
|
71
|
+
const PERIOD_MODELS = {
|
|
72
|
+
hour: {
|
|
73
|
+
counters: "insights_overview",
|
|
74
|
+
sketches: "insights_sketches"
|
|
75
|
+
},
|
|
76
|
+
day: {
|
|
77
|
+
counters: "insights_overview_daily",
|
|
78
|
+
sketches: "insights_sketches_daily"
|
|
79
|
+
},
|
|
80
|
+
lifetime: {
|
|
81
|
+
counters: "insights_overview_lifetime",
|
|
82
|
+
sketches: void 0
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
/** A download's patch counters: a patch delivered it, or it fell back from one. */
|
|
86
|
+
const patchCounters = (event) => event.type !== "UPDATE_DOWNLOADED" ? {} : {
|
|
87
|
+
...event.metadata.delivery === "patch" ? { patch_downloads: 1 } : {},
|
|
88
|
+
...event.metadata.patch_fallback === true ? { patch_fallbacks: 1 } : {}
|
|
89
|
+
};
|
|
90
|
+
/**
|
|
91
|
+
* Counters and sketches for one event: release, channel, and usage rows, with
|
|
92
|
+
* day rollups for channel and usage. Usage rows are written for the event's
|
|
93
|
+
* platform only, since a read for every platform merges the ios and android
|
|
94
|
+
* sketches; a channel's active installations come from its usage rows, so
|
|
95
|
+
* channel rows keep no sketch of their own. A download's patch counters go
|
|
96
|
+
* in the rows that count it.
|
|
97
|
+
*/
|
|
98
|
+
const countEvent = (tx, event) => {
|
|
99
|
+
const shardBy = event.install_id;
|
|
100
|
+
for (const delta of insightsOverviewDeltas(event)) {
|
|
101
|
+
if (delta.identity.platform === "all") continue;
|
|
102
|
+
const { bucketStartMs, ...parts } = delta.identity;
|
|
103
|
+
const periods = parts.scopeKind === "channel" || parts.scopeKind === "usage" ? [["hour", bucketStartMs], ["day", dayOf(event.received_at_ms)]] : [[parts.periodKind, bucketStartMs]];
|
|
104
|
+
for (const [periodKind, bucket] of periods) {
|
|
105
|
+
const models = PERIOD_MODELS[periodKind];
|
|
106
|
+
const key = {
|
|
107
|
+
identity: insightsIdentity({
|
|
108
|
+
...parts,
|
|
109
|
+
periodKind
|
|
110
|
+
}),
|
|
111
|
+
bucket_start_ms: bucket
|
|
112
|
+
};
|
|
113
|
+
const counters = {
|
|
114
|
+
...Object.fromEntries([
|
|
115
|
+
["downloads", delta.downloads],
|
|
116
|
+
["launches", delta.launches],
|
|
117
|
+
["failed_launches", delta.failedLaunches]
|
|
118
|
+
].filter(([, value]) => value !== 0)),
|
|
119
|
+
...delta.downloads === 0 ? {} : patchCounters(event)
|
|
120
|
+
};
|
|
121
|
+
if (Object.keys(counters).length > 0) tx.aggregate(models.counters, key, counters, { shardBy });
|
|
122
|
+
const sketches = {
|
|
123
|
+
...delta.launchIdentity === void 0 || parts.scopeKind === "channel" ? {} : { launch_users: addDistinct(null, delta.launchIdentity) },
|
|
124
|
+
...delta.activityIdentity === void 0 ? {} : { activity_users: addDistinct(null, delta.activityIdentity) }
|
|
125
|
+
};
|
|
126
|
+
if (models.sketches !== void 0 && Object.keys(sketches).length > 0) tx.aggregate(models.sketches, key, sketches, { shardBy });
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* A stored event's outcome rows: its bundle filter's hour, which a failed
|
|
132
|
+
* check has none of, and every stored event's UTC day.
|
|
133
|
+
*/
|
|
134
|
+
const countOutcome = (tx, event) => {
|
|
135
|
+
const shardBy = event.install_id;
|
|
136
|
+
if (!isFailedCheck(event)) tx.aggregate("insights_outcomes", {
|
|
137
|
+
platform: event.platform,
|
|
138
|
+
channel: event.channel,
|
|
139
|
+
type: event.type,
|
|
140
|
+
bundle_ref: event.type === "RECOVERED" ? `from:${event.from_bundle_id}` : `to:${event.to_bundle_id}`,
|
|
141
|
+
bucket_start_ms: hourOf(event.received_at_ms)
|
|
142
|
+
}, { events: 1 }, { shardBy });
|
|
143
|
+
tx.aggregate("insights_outcomes", {
|
|
144
|
+
...DAILY_EVENTS,
|
|
145
|
+
bucket_start_ms: dayOf(event.received_at_ms)
|
|
146
|
+
}, { events: 1 }, { shardBy });
|
|
147
|
+
};
|
|
148
|
+
/** The row check requires an UPDATE_FAILED row's failure; its type leaves it optional. */
|
|
149
|
+
const UNKNOWN_FAILURE = {
|
|
150
|
+
stage: "unknown",
|
|
151
|
+
reason: "unknown"
|
|
152
|
+
};
|
|
153
|
+
/** A failure's `detail`: what else the client knew, as a JSON array. */
|
|
154
|
+
const failureDetail = (failure) => JSON.stringify([
|
|
155
|
+
failure.resource ?? null,
|
|
156
|
+
failure.http_status ?? null,
|
|
157
|
+
failure.origin_code ?? null,
|
|
158
|
+
failure.transport ?? null
|
|
159
|
+
]);
|
|
160
|
+
/**
|
|
161
|
+
* An update failure's sketches, breakdown row, and lifetime counter. Its
|
|
162
|
+
* hourly breakdown row is what windowed reads sum, so no hourly or daily
|
|
163
|
+
* counter repeats it. A failed check counts for its channel, in the `check`
|
|
164
|
+
* sketch of installations. Any other failure counts in the channel's
|
|
165
|
+
* `failure` sketch, and for its target release, when it names one, in the
|
|
166
|
+
* release's hourly and lifetime `failure` sketches and its lifetime
|
|
167
|
+
* `failed_updates`, which the bundle detail reads. A failure is no launch
|
|
168
|
+
* and no activity: the installation reports its launch that day.
|
|
169
|
+
*/
|
|
170
|
+
const countFailure = (tx, event) => {
|
|
171
|
+
const shardBy = event.install_id;
|
|
172
|
+
const failure = event.metadata.failure ?? UNKNOWN_FAILURE;
|
|
173
|
+
const check = failure.stage === "check";
|
|
174
|
+
const hour = hourOf(event.received_at_ms);
|
|
175
|
+
const scope = {
|
|
176
|
+
channel: event.channel,
|
|
177
|
+
platform: event.platform,
|
|
178
|
+
appVersionKind: "all",
|
|
179
|
+
appVersion: ""
|
|
180
|
+
};
|
|
181
|
+
const channel = {
|
|
182
|
+
...scope,
|
|
183
|
+
releaseKind: "all",
|
|
184
|
+
releaseId: ""
|
|
185
|
+
};
|
|
186
|
+
const users = { failed_users: addDistinct(null, event.install_id) };
|
|
187
|
+
for (const [periodKind, bucket] of [["hour", hour], ["day", dayOf(event.received_at_ms)]]) {
|
|
188
|
+
const identity = insightsIdentity({
|
|
189
|
+
...channel,
|
|
190
|
+
scopeKind: check ? "check" : "failure",
|
|
191
|
+
periodKind
|
|
192
|
+
});
|
|
193
|
+
tx.aggregate(PERIOD_MODELS[periodKind].sketches, {
|
|
194
|
+
identity,
|
|
195
|
+
bucket_start_ms: bucket
|
|
196
|
+
}, users, { shardBy });
|
|
197
|
+
}
|
|
198
|
+
const releaseId = check ? null : event.to_release_id;
|
|
199
|
+
if (releaseId !== null) {
|
|
200
|
+
const release = {
|
|
201
|
+
...scope,
|
|
202
|
+
releaseKind: "specific",
|
|
203
|
+
releaseId
|
|
204
|
+
};
|
|
205
|
+
for (const [periodKind, bucket, sketches] of [[
|
|
206
|
+
"hour",
|
|
207
|
+
hour,
|
|
208
|
+
"insights_sketches"
|
|
209
|
+
], [
|
|
210
|
+
"lifetime",
|
|
211
|
+
0,
|
|
212
|
+
"insights_sketches_lifetime"
|
|
213
|
+
]]) {
|
|
214
|
+
const key = (scopeKind) => ({
|
|
215
|
+
identity: insightsIdentity({
|
|
216
|
+
...release,
|
|
217
|
+
scopeKind,
|
|
218
|
+
periodKind
|
|
219
|
+
}),
|
|
220
|
+
bucket_start_ms: bucket
|
|
221
|
+
});
|
|
222
|
+
tx.aggregate(sketches, key("failure"), users, { shardBy });
|
|
223
|
+
}
|
|
224
|
+
tx.aggregate("insights_overview_lifetime", {
|
|
225
|
+
identity: insightsIdentity({
|
|
226
|
+
...release,
|
|
227
|
+
scopeKind: "release",
|
|
228
|
+
periodKind: "lifetime"
|
|
229
|
+
}),
|
|
230
|
+
bucket_start_ms: 0
|
|
231
|
+
}, { failed_updates: 1 }, { shardBy });
|
|
232
|
+
}
|
|
233
|
+
tx.aggregate("insights_failures", {
|
|
234
|
+
platform: event.platform,
|
|
235
|
+
channel: event.channel,
|
|
236
|
+
bucket_start_ms: hour,
|
|
237
|
+
release_id: releaseId ?? "",
|
|
238
|
+
stage: failure.stage,
|
|
239
|
+
reason: failure.reason,
|
|
240
|
+
detail: failureDetail(failure)
|
|
241
|
+
}, { events: 1 }, { shardBy });
|
|
242
|
+
};
|
|
243
|
+
/**
|
|
244
|
+
* A recovery's exit reason, which Android 11+ reports, in the breakdown of
|
|
245
|
+
* the release whose launch failed: stage `launch`, the reason as its reason.
|
|
246
|
+
*/
|
|
247
|
+
const countExit = (tx, event) => {
|
|
248
|
+
const exit = event.metadata.previous_process_exit;
|
|
249
|
+
if (event.type !== "RECOVERED" || exit === void 0) return;
|
|
250
|
+
tx.aggregate("insights_failures", {
|
|
251
|
+
platform: event.platform,
|
|
252
|
+
channel: event.channel,
|
|
253
|
+
bucket_start_ms: hourOf(event.received_at_ms),
|
|
254
|
+
release_id: event.from_release_id ?? "",
|
|
255
|
+
stage: "launch",
|
|
256
|
+
reason: exit,
|
|
257
|
+
detail: ""
|
|
258
|
+
}, { events: 1 }, { shardBy: event.install_id });
|
|
259
|
+
};
|
|
260
|
+
const isNewer = (event, head) => event.received_at_ms !== head.received_at_ms ? event.received_at_ms > head.received_at_ms : compareUtf8(event.id, head.id) > 0;
|
|
261
|
+
/** What an UNCHANGED report must share with its installation's head to repeat it. */
|
|
262
|
+
const REPEATED_FIELDS = [
|
|
263
|
+
"channel",
|
|
264
|
+
"platform",
|
|
265
|
+
"app_version",
|
|
266
|
+
"to_bundle_id",
|
|
267
|
+
"to_release_id",
|
|
268
|
+
"user_id"
|
|
269
|
+
];
|
|
270
|
+
/**
|
|
271
|
+
* An UNCHANGED report that repeats its installation's head from the same UTC
|
|
272
|
+
* day: the installation already counts as active that day, on that bundle,
|
|
273
|
+
* so the report records nothing. A download's head does not count, because
|
|
274
|
+
* its installation still ran the bundle it downloaded from.
|
|
275
|
+
*/
|
|
276
|
+
const repeatsHead = (event, head) => event.type === "UNCHANGED" && head.type !== "UPDATE_DOWNLOADED" && dayOf(event.received_at_ms) === dayOf(head.received_at_ms) && REPEATED_FIELDS.every((field) => event[field] === head[field]);
|
|
277
|
+
/**
|
|
278
|
+
* Records one event in one transaction: one batch read of the event and its
|
|
279
|
+
* installation's head, one of the gauge and sketch rows it changes, then one
|
|
280
|
+
* write. On a database that batches aggregates (DynamoDB, Firestore), the
|
|
281
|
+
* write holds the event's rows and one log row instead, and a compaction
|
|
282
|
+
* reads and writes the aggregate rows. A stored event's id changes nothing
|
|
283
|
+
* when repeated, whichever installation sends it, and neither does the id
|
|
284
|
+
* of the installation's head. An UNCHANGED report that repeats its head on
|
|
285
|
+
* the same UTC day writes nothing at all. An older event still counts in its
|
|
286
|
+
* own hour but never replaces the head. An update failure is stored and
|
|
287
|
+
* counted, but changes what no installation runs, so it moves no head.
|
|
288
|
+
*/
|
|
289
|
+
const recordEvent = (db, event) => {
|
|
290
|
+
assertBundleEventRow(event);
|
|
291
|
+
return db.transaction(async (tx) => {
|
|
292
|
+
const [existing, previous] = await Promise.all([tx.findOne("bundle_events", { id: event.id }), tx.findOne("bundle_event_heads", { install_id: event.install_id })]);
|
|
293
|
+
if (existing !== null || previous?.id === event.id) return;
|
|
294
|
+
if (previous !== null && repeatsHead(event, previous)) return;
|
|
295
|
+
if (event.type !== "UNCHANGED") {
|
|
296
|
+
tx.create("bundle_events", event);
|
|
297
|
+
countOutcome(tx, event);
|
|
298
|
+
}
|
|
299
|
+
if (event.type === "UPDATE_FAILED") {
|
|
300
|
+
countFailure(tx, event);
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
countEvent(tx, event);
|
|
304
|
+
countExit(tx, event);
|
|
305
|
+
if (previous !== null && !isNewer(event, previous)) return;
|
|
306
|
+
if (previous === null) tx.create("bundle_event_heads", event);
|
|
307
|
+
else {
|
|
308
|
+
const { install_id: _, ...fields } = event;
|
|
309
|
+
countHead(tx, previous, -1);
|
|
310
|
+
tx.update("bundle_event_heads", previous, fields);
|
|
311
|
+
}
|
|
312
|
+
countHead(tx, event, 1);
|
|
313
|
+
});
|
|
314
|
+
};
|
|
315
|
+
//#endregion
|
|
316
|
+
export { PAIR_FIELD, bundlePairKey, insightsIdentity, recordEvent };
|