blogwright-analytics 0.4.0-beta.2 → 0.4.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -10
- package/dist/adapters/transform-diagnostics.d.ts +4 -0
- package/dist/adapters/transform-diagnostics.js +4 -0
- package/dist/app/_app/immutable/assets/0.BfwUqNJ-.css +1 -0
- package/dist/app/_app/immutable/assets/2.lkfjnM1s.css +1 -0
- package/dist/app/_app/immutable/assets/DefaultTooltip.CXdtNw-m.css +1 -0
- package/dist/app/_app/immutable/assets/Tooltip.BsTJVNOF.css +1 -0
- package/dist/app/_app/immutable/chunks/4DYKm5b2.js +1 -0
- package/dist/app/_app/immutable/chunks/B6Dt35mA.js +1 -0
- package/dist/app/_app/immutable/chunks/BAneOF1f.js +1 -0
- package/dist/app/_app/immutable/chunks/BQHoVMm0.js +1 -0
- package/dist/app/_app/immutable/chunks/{Doz7YX1W.js → BUku_rSO.js} +1 -1
- package/dist/app/_app/immutable/chunks/BoX17NrM.js +1 -0
- package/dist/app/_app/immutable/chunks/{vDgBJUjM.js → BpoiI2_d.js} +1 -1
- package/dist/app/_app/immutable/chunks/Bsj_2e-_.js +1 -0
- package/dist/app/_app/immutable/chunks/{-aNGNaBT.js → BumHYZmq.js} +1 -1
- package/dist/app/_app/immutable/chunks/{BJngAqxk.js → C0tbX_qQ.js} +1 -1
- package/dist/app/_app/immutable/chunks/C7qhOD3U.js +1 -0
- package/dist/app/_app/immutable/chunks/C8TrOru6.js +1 -0
- package/dist/app/_app/immutable/chunks/{DJc8C0NK.js → CAgm0Ruv.js} +1 -1
- package/dist/app/_app/immutable/chunks/CSrT-PUe.js +1 -0
- package/dist/app/_app/immutable/chunks/CU11V7_a.js +1 -0
- package/dist/app/_app/immutable/chunks/CdRe9NtS.js +1 -0
- package/dist/app/_app/immutable/chunks/Ch6MextH.js +1 -0
- package/dist/app/_app/immutable/chunks/{DCkHk5n_.js → Cr8mHXFf.js} +1 -1
- package/dist/app/_app/immutable/chunks/CuOP1eLc.js +2 -0
- package/dist/app/_app/immutable/chunks/{Bc-53cml.js → CwPKxnpy.js} +1 -1
- package/dist/app/_app/immutable/chunks/{FH_3TS2H.js → Cz3xtWLM.js} +1 -1
- package/dist/app/_app/immutable/chunks/D26njVw7.js +5 -0
- package/dist/app/_app/immutable/chunks/D6rPBWFd.js +1 -0
- package/dist/app/_app/immutable/chunks/D8dq3MNd.js +1 -0
- package/dist/app/_app/immutable/chunks/DOqi3Wfa.js +1 -0
- package/dist/app/_app/immutable/chunks/DRQZHPoe.js +1 -0
- package/dist/app/_app/immutable/chunks/DUJFqVVs.js +1 -0
- package/dist/app/_app/immutable/chunks/DVCZGvXq.js +3 -0
- package/dist/app/_app/immutable/chunks/Dco3y1p7.js +1 -0
- package/dist/app/_app/immutable/chunks/DmKBckYv.js +1 -0
- package/dist/app/_app/immutable/chunks/DnQEoGed.js +1 -0
- package/dist/app/_app/immutable/chunks/DoNmKjPy.js +1 -0
- package/dist/app/_app/immutable/chunks/DodcabfQ.js +1 -0
- package/dist/app/_app/immutable/chunks/DugTr35P.js +1 -0
- package/dist/app/_app/immutable/chunks/{xIq_fFFM.js → RBkNRIpl.js} +1 -1
- package/dist/app/_app/immutable/chunks/{DKMlMI4a.js → Sb-KFJGv.js} +1 -1
- package/dist/app/_app/immutable/chunks/XoU76ysy.js +1 -0
- package/dist/app/_app/immutable/chunks/imr7FUlI.js +15 -0
- package/dist/app/_app/immutable/chunks/{Dh958be7.js → xefVn_UF.js} +1 -1
- package/dist/app/_app/immutable/entry/app.w1HyN1Pe.js +2 -0
- package/dist/app/_app/immutable/entry/start.Iz8smxW0.js +1 -0
- package/dist/app/_app/immutable/nodes/0.BjQeslHk.js +1 -0
- package/dist/app/_app/immutable/nodes/1.DfqY5csg.js +1 -0
- package/dist/app/_app/immutable/nodes/2.CByBWAMF.js +10 -0
- package/dist/app/_app/version.json +1 -1
- package/dist/app/index.html +40 -36
- package/dist/aws/firehose.d.ts +32 -1
- package/dist/aws/firehose.js +11 -0
- package/dist/commands.js +1 -1
- package/dist/nodes.d.ts +65 -18
- package/dist/nodes.js +378 -58
- package/dist/path-filter.d.ts +4 -0
- package/dist/path-filter.js +34 -0
- package/dist/plugin.d.ts +2 -2
- package/dist/plugin.js +2 -2
- package/dist/queries.d.ts +16 -7
- package/dist/queries.js +82 -12
- package/dist/reporting-range.d.ts +53 -0
- package/dist/reporting-range.js +57 -0
- package/dist/server.js +47 -5
- package/dist/traffic-breakdown.d.ts +2 -0
- package/dist/traffic-breakdown.js +15 -0
- package/dist/transform/diagnostics.d.ts +17 -0
- package/dist/transform/diagnostics.js +1 -0
- package/dist/transform/handler.d.ts +6 -5
- package/dist/transform/handler.js +34 -11
- package/dist/transform-bundle/index.mjs +53 -13
- package/dist/transform-bundle/transform-manifest.json +2 -2
- package/dist/view-granularity.d.ts +27 -0
- package/dist/view-granularity.js +18 -0
- package/package.json +12 -3
- package/dist/app/_app/immutable/assets/0.BTQrrh5B.css +0 -1
- package/dist/app/_app/immutable/assets/2.DoCaRy3T.css +0 -1
- package/dist/app/_app/immutable/assets/DefaultTooltip.BXd66MzC.css +0 -1
- package/dist/app/_app/immutable/chunks/1aEdY044.js +0 -15
- package/dist/app/_app/immutable/chunks/B3vZDoek.js +0 -1
- package/dist/app/_app/immutable/chunks/B5KRA4hC.js +0 -1
- package/dist/app/_app/immutable/chunks/BC0iYSWx.js +0 -2
- package/dist/app/_app/immutable/chunks/BZrG8lR6.js +0 -5
- package/dist/app/_app/immutable/chunks/Bd1gDe3Y.js +0 -1
- package/dist/app/_app/immutable/chunks/Bkszif4s.js +0 -1
- package/dist/app/_app/immutable/chunks/BnKf2MvA.js +0 -1
- package/dist/app/_app/immutable/chunks/CKcppAud.js +0 -1
- package/dist/app/_app/immutable/chunks/CRTcXoMo.js +0 -1
- package/dist/app/_app/immutable/chunks/CvyVA_jC.js +0 -1
- package/dist/app/_app/immutable/chunks/CxBzrV8P.js +0 -1
- package/dist/app/_app/immutable/chunks/CyCGwyzf.js +0 -1
- package/dist/app/_app/immutable/chunks/D-mRKj3n.js +0 -1
- package/dist/app/_app/immutable/chunks/D0Ty6LN0.js +0 -1
- package/dist/app/_app/immutable/chunks/D2BnX0Uk.js +0 -3
- package/dist/app/_app/immutable/chunks/DBwYG1VY.js +0 -1
- package/dist/app/_app/immutable/chunks/DMoYijy0.js +0 -1
- package/dist/app/_app/immutable/chunks/DRmJpyQa.js +0 -1
- package/dist/app/_app/immutable/chunks/DVXZkpbf.js +0 -1
- package/dist/app/_app/immutable/chunks/DorUkkYD.js +0 -1
- package/dist/app/_app/immutable/chunks/DtuTIrAM.js +0 -1
- package/dist/app/_app/immutable/chunks/V9ZjaxiY.js +0 -1
- package/dist/app/_app/immutable/chunks/ZvRY-WVy.js +0 -1
- package/dist/app/_app/immutable/chunks/caXkbKD3.js +0 -1
- package/dist/app/_app/immutable/chunks/oNmA_rTI.js +0 -1
- package/dist/app/_app/immutable/chunks/yT96EWBw.js +0 -1
- package/dist/app/_app/immutable/entry/app.cy9eKSTf.js +0 -2
- package/dist/app/_app/immutable/entry/start.B2ycr8Ji.js +0 -1
- package/dist/app/_app/immutable/nodes/0.UTMEigHJ.js +0 -1
- package/dist/app/_app/immutable/nodes/1.CMX4YrAG.js +0 -1
- package/dist/app/_app/immutable/nodes/2.CQMIFodk.js +0 -6
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Shared validation for literal, segment-bounded path filters. */
|
|
2
|
+
export declare function normalizePathFilter(value: unknown): string | undefined;
|
|
3
|
+
/** User-facing validation shares the exact rules used by query preparation. */
|
|
4
|
+
export declare function pathProblem(value: string): string | undefined;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/** Shared validation for literal, segment-bounded path filters. */
|
|
2
|
+
export function normalizePathFilter(value) {
|
|
3
|
+
if (value === undefined)
|
|
4
|
+
return undefined;
|
|
5
|
+
if (typeof value !== 'string')
|
|
6
|
+
throw new Error('Path must be a string.');
|
|
7
|
+
const path = value.trim();
|
|
8
|
+
if (path === '')
|
|
9
|
+
return undefined;
|
|
10
|
+
// Reject ASCII control characters without embedding them in a regular expression.
|
|
11
|
+
const hasControlCharacter = Array.from(path).some((character) => {
|
|
12
|
+
const code = character.charCodeAt(0);
|
|
13
|
+
return code < 0x20 || code === 0x7f;
|
|
14
|
+
});
|
|
15
|
+
if (!path.startsWith('/') ||
|
|
16
|
+
path.startsWith('//') ||
|
|
17
|
+
/[\s?#\\]/.test(path) ||
|
|
18
|
+
hasControlCharacter) {
|
|
19
|
+
throw new Error('Enter a path starting with /, without a query string, fragment, or spaces.');
|
|
20
|
+
}
|
|
21
|
+
return path.replace(/\/+$/, '') || '/';
|
|
22
|
+
}
|
|
23
|
+
/** User-facing validation shares the exact rules used by query preparation. */
|
|
24
|
+
export function pathProblem(value) {
|
|
25
|
+
try {
|
|
26
|
+
normalizePathFilter(value);
|
|
27
|
+
return undefined;
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (error instanceof Error)
|
|
31
|
+
return error.message;
|
|
32
|
+
throw error;
|
|
33
|
+
}
|
|
34
|
+
}
|
package/dist/plugin.d.ts
CHANGED
|
@@ -51,11 +51,11 @@
|
|
|
51
51
|
* They are both now, and `analytics status` still runs the declared command
|
|
52
52
|
* above rather than the generic verb, because a declared action wins.
|
|
53
53
|
*
|
|
54
|
-
* The property `nodes` carries is not just "
|
|
54
|
+
* The property `nodes` carries is not just "fourteen nodes": it is that the
|
|
55
55
|
* CLI's engine reconciles them against the store `toPluginContext` scoped to
|
|
56
56
|
* this plugin's name, `state/<env>.analytics.json` - never the site's
|
|
57
57
|
* `state/<env>.json`. `blogwright bootstrap` therefore provisions none of
|
|
58
|
-
* these
|
|
58
|
+
* these fourteen (the site's `buildNodes` does not consult discovery), and
|
|
59
59
|
* `blogwright destroy --yes` removes none of them, because
|
|
60
60
|
* `assertNoScopedState` (`packages/cli/src/commands.ts`) refuses the site
|
|
61
61
|
* teardown for as long as that scoped object exists and names
|
package/dist/plugin.js
CHANGED
|
@@ -51,11 +51,11 @@
|
|
|
51
51
|
* They are both now, and `analytics status` still runs the declared command
|
|
52
52
|
* above rather than the generic verb, because a declared action wins.
|
|
53
53
|
*
|
|
54
|
-
* The property `nodes` carries is not just "
|
|
54
|
+
* The property `nodes` carries is not just "fourteen nodes": it is that the
|
|
55
55
|
* CLI's engine reconciles them against the store `toPluginContext` scoped to
|
|
56
56
|
* this plugin's name, `state/<env>.analytics.json` - never the site's
|
|
57
57
|
* `state/<env>.json`. `blogwright bootstrap` therefore provisions none of
|
|
58
|
-
* these
|
|
58
|
+
* these fourteen (the site's `buildNodes` does not consult discovery), and
|
|
59
59
|
* `blogwright destroy --yes` removes none of them, because
|
|
60
60
|
* `assertNoScopedState` (`packages/cli/src/commands.ts`) refuses the site
|
|
61
61
|
* teardown for as long as that scoped object exists and names
|
package/dist/queries.d.ts
CHANGED
|
@@ -48,6 +48,7 @@
|
|
|
48
48
|
* Pure data and pure functions only: no `node:` builtin, no vendor SDK, no
|
|
49
49
|
* `fetch`. Nothing here runs a statement - that is `AnalyticsQuery`'s adapter.
|
|
50
50
|
*/
|
|
51
|
+
import { type ViewGranularity } from './view-granularity.js';
|
|
51
52
|
import type { AnalyticsConfig } from './config.js';
|
|
52
53
|
import type { PageViewColumnName } from './schema.js';
|
|
53
54
|
/**
|
|
@@ -74,16 +75,16 @@ export declare const PAGE_VIEWS_RELATION = "page_views";
|
|
|
74
75
|
* all of them; the list is per-definition so a statement that forgot one is a
|
|
75
76
|
* test failure naming that query rather than a filter that quietly never ran.
|
|
76
77
|
*/
|
|
77
|
-
declare const QUERY_PARAM_NAMES: readonly ["from", "to", "include_bots"];
|
|
78
|
+
declare const QUERY_PARAM_NAMES: readonly ["from", "to", "include_bots", "bucket_minutes", "from_time", "to_time"];
|
|
78
79
|
/** One of {@link QUERY_PARAM_NAMES}. */
|
|
79
80
|
type QueryParamName = (typeof QUERY_PARAM_NAMES)[number];
|
|
80
81
|
/** A value bound to a placeholder. Days are bound as text and cast in the SQL. */
|
|
81
82
|
type BindValue = string | boolean;
|
|
82
|
-
/**
|
|
83
|
+
/** UTC calendar days (inclusive) or minute timestamps (start inclusive, end exclusive). */
|
|
83
84
|
interface DateRange {
|
|
84
|
-
/**
|
|
85
|
+
/** Inclusive start day or UTC minute. */
|
|
85
86
|
readonly from: string;
|
|
86
|
-
/**
|
|
87
|
+
/** Inclusive end day, or exclusive end UTC minute. */
|
|
87
88
|
readonly to: string;
|
|
88
89
|
}
|
|
89
90
|
/**
|
|
@@ -92,7 +93,13 @@ interface DateRange {
|
|
|
92
93
|
* `config.analytics.bots` (task 44) through {@link BOTS_INCLUDED_BY_DEFAULT}.
|
|
93
94
|
*/
|
|
94
95
|
export interface QueryParams {
|
|
95
|
-
/**
|
|
96
|
+
/** Optional uppercase ISO country code. */
|
|
97
|
+
readonly country?: string | undefined;
|
|
98
|
+
/** Exact path and descendants at a slash boundary; blank means every path. */
|
|
99
|
+
readonly path?: string | undefined;
|
|
100
|
+
/** Include disjoint bot/non-bot contributions; requires bots to be included. */
|
|
101
|
+
readonly splitBots?: boolean | undefined;
|
|
102
|
+
/** The UTC reporting window. */
|
|
96
103
|
readonly range: DateRange;
|
|
97
104
|
/**
|
|
98
105
|
* Whether bot-flagged rows are counted. Absent means "whatever
|
|
@@ -101,6 +108,8 @@ export interface QueryParams {
|
|
|
101
108
|
* spec's Decision *Flag bots, do not drop them*.
|
|
102
109
|
*/
|
|
103
110
|
readonly includeBots?: boolean | undefined;
|
|
111
|
+
/** UTC bucket width; accepted only by views-over-time. */
|
|
112
|
+
readonly granularity?: ViewGranularity | undefined;
|
|
104
113
|
}
|
|
105
114
|
/** One named query: its statement, what it reads, what it binds, what it returns. */
|
|
106
115
|
interface QueryDefinition {
|
|
@@ -130,7 +139,7 @@ interface QueryDefinition {
|
|
|
130
139
|
* The name of the row-count query. Not one of the seven the spec's §Local
|
|
131
140
|
* server lists - those answer the dashboard's panels; this one answers
|
|
132
141
|
* `analytics status`, which reports the table's current row count beside the
|
|
133
|
-
* plugin's
|
|
142
|
+
* plugin's fourteen nodes. It lives in this set rather than in the command
|
|
134
143
|
* because the command may not write SQL: every statement this package runs is
|
|
135
144
|
* one of these definitions, reached through the `AnalyticsQuery` port.
|
|
136
145
|
*/
|
|
@@ -279,7 +288,7 @@ export declare function queryDefinition(name: string): QueryDefinition;
|
|
|
279
288
|
export interface PreparedQuery {
|
|
280
289
|
/** The name that resolved. */
|
|
281
290
|
readonly name: QueryName;
|
|
282
|
-
/** The statement to execute,
|
|
291
|
+
/** The fixed statement to execute, including an intraday variant when requested. */
|
|
283
292
|
readonly sql: string;
|
|
284
293
|
/** The columns a row of the result carries. */
|
|
285
294
|
readonly resultColumns: readonly string[];
|
package/dist/queries.js
CHANGED
|
@@ -48,6 +48,9 @@
|
|
|
48
48
|
* Pure data and pure functions only: no `node:` builtin, no vendor SDK, no
|
|
49
49
|
* `fetch`. Nothing here runs a statement - that is `AnalyticsQuery`'s adapter.
|
|
50
50
|
*/
|
|
51
|
+
import { normalizePathFilter } from './path-filter.js';
|
|
52
|
+
import { withTrafficBreakdown } from './traffic-breakdown.js';
|
|
53
|
+
import { parseViewGranularity, VIEW_GRANULARITIES, } from './view-granularity.js';
|
|
51
54
|
/**
|
|
52
55
|
* The relation every named query reads. Not "the configured table name" - the
|
|
53
56
|
* adapter binds this name to the configured `<namespace>.<table>` inside the
|
|
@@ -74,7 +77,14 @@ function sql(fragments, ...relations) {
|
|
|
74
77
|
* all of them; the list is per-definition so a statement that forgot one is a
|
|
75
78
|
* test failure naming that query rather than a filter that quietly never ran.
|
|
76
79
|
*/
|
|
77
|
-
const QUERY_PARAM_NAMES = [
|
|
80
|
+
const QUERY_PARAM_NAMES = [
|
|
81
|
+
'from',
|
|
82
|
+
'to',
|
|
83
|
+
'include_bots',
|
|
84
|
+
'bucket_minutes',
|
|
85
|
+
'from_time',
|
|
86
|
+
'to_time',
|
|
87
|
+
];
|
|
78
88
|
/**
|
|
79
89
|
* The `x-edge-result-type` values CloudFront reports for a cache hit. AWS
|
|
80
90
|
* documents that field's values as `Hit`, `RefreshHit`, `Miss`,
|
|
@@ -89,7 +99,7 @@ const CACHE_HIT_RESULT_TYPES = ['Hit', 'RefreshHit'];
|
|
|
89
99
|
* The name of the row-count query. Not one of the seven the spec's §Local
|
|
90
100
|
* server lists - those answer the dashboard's panels; this one answers
|
|
91
101
|
* `analytics status`, which reports the table's current row count beside the
|
|
92
|
-
* plugin's
|
|
102
|
+
* plugin's fourteen nodes. It lives in this set rather than in the command
|
|
93
103
|
* because the command may not write SQL: every statement this package runs is
|
|
94
104
|
* one of these definitions, reached through the `AnalyticsQuery` port.
|
|
95
105
|
*/
|
|
@@ -333,6 +343,13 @@ function isCalendarDay(day) {
|
|
|
333
343
|
return false;
|
|
334
344
|
return new Date(time).toISOString().slice(0, DAY_LENGTH) === day;
|
|
335
345
|
}
|
|
346
|
+
/** Minute precision only; explicit UTC prevents host-timezone interpretation. */
|
|
347
|
+
function isUtcMinute(value) {
|
|
348
|
+
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z$/.test(value))
|
|
349
|
+
return false;
|
|
350
|
+
const instant = Date.parse(value);
|
|
351
|
+
return Number.isFinite(instant) && new Date(instant).toISOString().slice(0, 16) + 'Z' === value;
|
|
352
|
+
}
|
|
336
353
|
/**
|
|
337
354
|
* Resolve a name to its definition, raising with the available names when it
|
|
338
355
|
* is not one of them. Takes a `string` rather than a {@link QueryName} because
|
|
@@ -368,18 +385,31 @@ export function queryDefinition(name) {
|
|
|
368
385
|
function validateRange(name, range) {
|
|
369
386
|
const from = range?.from;
|
|
370
387
|
const to = range?.to;
|
|
371
|
-
if (typeof from !== 'string' || !isCalendarDay(from)) {
|
|
372
|
-
throw new Error(`analytics query ${describeValue(name)}: range.from must be a YYYY-MM-DD calendar day, got ${describeValue(from)}`);
|
|
388
|
+
if (typeof from !== 'string' || !(isCalendarDay(from) || isUtcMinute(from))) {
|
|
389
|
+
throw new Error(`analytics query ${describeValue(name)}: range.from must be a YYYY-MM-DD calendar day or YYYY-MM-DDTHH:mmZ UTC minute, got ${describeValue(from)}`);
|
|
390
|
+
}
|
|
391
|
+
if (typeof to !== 'string' || !(isCalendarDay(to) || isUtcMinute(to))) {
|
|
392
|
+
throw new Error(`analytics query ${describeValue(name)}: range.to must be a YYYY-MM-DD calendar day or YYYY-MM-DDTHH:mmZ UTC minute, got ${describeValue(to)}`);
|
|
373
393
|
}
|
|
374
|
-
if (
|
|
375
|
-
throw new Error(
|
|
394
|
+
if (isUtcMinute(from) !== isUtcMinute(to)) {
|
|
395
|
+
throw new Error('range endpoints must both be calendar days or both UTC minute timestamps');
|
|
376
396
|
}
|
|
377
|
-
//
|
|
378
|
-
if (from > to) {
|
|
397
|
+
// Canonical UTC strings sort chronologically.
|
|
398
|
+
if (from > to || (isUtcMinute(from) && from === to)) {
|
|
379
399
|
throw new Error(`analytics query ${describeValue(name)}: range is inverted - from ${describeValue(from)} is after to ${describeValue(to)}`);
|
|
380
400
|
}
|
|
381
401
|
return { from, to };
|
|
382
402
|
}
|
|
403
|
+
/** Intraday alternative keeps the day partition filter for Iceberg pruning. */
|
|
404
|
+
const INTRADAY_VIEWS_SQL = sql `
|
|
405
|
+
SELECT strftime(time_bucket(CAST($bucket_minutes AS BIGINT) * INTERVAL '1 minute', event_time), '%Y-%m-%dT%H:%M:%SZ') AS day,
|
|
406
|
+
count(*) AS views
|
|
407
|
+
FROM ${PAGE_VIEWS}
|
|
408
|
+
WHERE day BETWEEN CAST($from AS DATE) AND CAST($to AS DATE)
|
|
409
|
+
AND ($include_bots OR NOT coalesce(is_bot, false))
|
|
410
|
+
GROUP BY 1
|
|
411
|
+
ORDER BY 1
|
|
412
|
+
`;
|
|
383
413
|
/**
|
|
384
414
|
* Resolve a name and its parameters into the statement and the bindings an
|
|
385
415
|
* adapter runs - the one boundary where an unknown name and a bad range are
|
|
@@ -394,21 +424,61 @@ function validateRange(name, range) {
|
|
|
394
424
|
export function prepareQuery(name, params, config) {
|
|
395
425
|
const definition = queryDefinition(name);
|
|
396
426
|
const range = validateRange(name, params.range);
|
|
427
|
+
const path = normalizePathFilter(params.path);
|
|
428
|
+
if (params.country !== undefined && !/^[A-Z]{2}$/.test(params.country)) {
|
|
429
|
+
throw new Error('country must be an uppercase two-letter country code');
|
|
430
|
+
}
|
|
431
|
+
if (params.granularity !== undefined && name !== 'views-over-time') {
|
|
432
|
+
throw new Error('granularity is only supported by views-over-time');
|
|
433
|
+
}
|
|
434
|
+
const granularity = parseViewGranularity(params.granularity);
|
|
435
|
+
const intraday = name === 'views-over-time' && granularity !== '24h';
|
|
397
436
|
const includeBots = params.includeBots ?? BOTS_INCLUDED_BY_DEFAULT[config.bots];
|
|
437
|
+
if (params.splitBots !== undefined && typeof params.splitBots !== 'boolean') {
|
|
438
|
+
throw new Error('splitBots must be a boolean');
|
|
439
|
+
}
|
|
440
|
+
if (params.splitBots && !includeBots)
|
|
441
|
+
throw new Error('splitBots requires includeBots=true');
|
|
398
442
|
const available = {
|
|
399
|
-
from: range.from,
|
|
400
|
-
to: range.to,
|
|
443
|
+
from: range.from.slice(0, 10),
|
|
444
|
+
to: range.to.slice(0, 10),
|
|
445
|
+
from_time: range.from,
|
|
446
|
+
to_time: range.to,
|
|
401
447
|
include_bots: includeBots,
|
|
448
|
+
bucket_minutes: String(VIEW_GRANULARITIES[granularity].minutes),
|
|
402
449
|
};
|
|
403
450
|
const bindings = {};
|
|
404
451
|
for (const bind of definition.binds)
|
|
405
452
|
bindings[bind] = available[bind];
|
|
453
|
+
if (intraday)
|
|
454
|
+
bindings['bucket_minutes'] = available.bucket_minutes;
|
|
455
|
+
let statement = intraday ? INTRADAY_VIEWS_SQL : definition.sql;
|
|
456
|
+
if (isUtcMinute(range.from)) {
|
|
457
|
+
bindings['from_time'] = `${range.from.slice(0, 16)}:00Z`;
|
|
458
|
+
bindings['to_time'] = `${range.to.slice(0, 16)}:00Z`;
|
|
459
|
+
// Add a fixed predicate to the shared partition filter, including CTE queries.
|
|
460
|
+
statement = statement.replace('day BETWEEN CAST($from AS DATE) AND CAST($to AS DATE)', 'day BETWEEN CAST($from AS DATE) AND CAST($to AS DATE) AND event_time >= CAST($from_time AS TIMESTAMP) AND event_time < CAST($to_time AS TIMESTAMP)');
|
|
461
|
+
}
|
|
462
|
+
if (path !== undefined) {
|
|
463
|
+
bindings['path'] = path;
|
|
464
|
+
bindings['path_prefix'] = path === '/' ? '/' : `${path}/`;
|
|
465
|
+
// starts_with treats SQL wildcard characters as literal path characters.
|
|
466
|
+
statement = statement.replace('day BETWEEN CAST($from AS DATE) AND CAST($to AS DATE)', 'day BETWEEN CAST($from AS DATE) AND CAST($to AS DATE) AND (uri = $path OR starts_with(uri, $path_prefix))');
|
|
467
|
+
}
|
|
468
|
+
if (params.country !== undefined) {
|
|
469
|
+
bindings['country'] = params.country;
|
|
470
|
+
statement = statement.replace('day BETWEEN CAST($from AS DATE) AND CAST($to AS DATE)', 'day BETWEEN CAST($from AS DATE) AND CAST($to AS DATE) AND country = $country');
|
|
471
|
+
}
|
|
472
|
+
if (params.splitBots)
|
|
473
|
+
statement = withTrafficBreakdown(name, statement);
|
|
406
474
|
return {
|
|
407
475
|
// Justified by `queryDefinition` above: it raised unless `name` is one of
|
|
408
476
|
// the table's own keys, which is exactly what `QueryName` enumerates.
|
|
409
477
|
name: name,
|
|
410
|
-
sql:
|
|
411
|
-
resultColumns:
|
|
478
|
+
sql: statement,
|
|
479
|
+
resultColumns: params.splitBots
|
|
480
|
+
? [...definition.resultColumns, 'non_bot', 'bot']
|
|
481
|
+
: definition.resultColumns,
|
|
412
482
|
bindings,
|
|
413
483
|
};
|
|
414
484
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/** UTC minute inputs and rolling reporting windows. */
|
|
2
|
+
interface DateRange {
|
|
3
|
+
readonly from: string;
|
|
4
|
+
readonly to: string;
|
|
5
|
+
}
|
|
6
|
+
export declare const PERIODS: readonly [{
|
|
7
|
+
readonly value: "3h";
|
|
8
|
+
readonly label: "3H";
|
|
9
|
+
readonly hours: 3;
|
|
10
|
+
}, {
|
|
11
|
+
readonly value: "6h";
|
|
12
|
+
readonly label: "6H";
|
|
13
|
+
readonly hours: 6;
|
|
14
|
+
}, {
|
|
15
|
+
readonly value: "12h";
|
|
16
|
+
readonly label: "12H";
|
|
17
|
+
readonly hours: 12;
|
|
18
|
+
}, {
|
|
19
|
+
readonly value: "24h";
|
|
20
|
+
readonly label: "24H";
|
|
21
|
+
readonly hours: 24;
|
|
22
|
+
}, {
|
|
23
|
+
readonly value: "5d";
|
|
24
|
+
readonly label: "5D";
|
|
25
|
+
readonly hours: 120;
|
|
26
|
+
}, {
|
|
27
|
+
readonly value: "1w";
|
|
28
|
+
readonly label: "1W";
|
|
29
|
+
readonly hours: 168;
|
|
30
|
+
}, {
|
|
31
|
+
readonly value: "4w";
|
|
32
|
+
readonly label: "4W";
|
|
33
|
+
readonly hours: 672;
|
|
34
|
+
}, {
|
|
35
|
+
readonly value: "3mo";
|
|
36
|
+
readonly label: "3M";
|
|
37
|
+
readonly months: 3;
|
|
38
|
+
}, {
|
|
39
|
+
readonly value: "6mo";
|
|
40
|
+
readonly label: "6M";
|
|
41
|
+
readonly months: 6;
|
|
42
|
+
}, {
|
|
43
|
+
readonly value: "1y";
|
|
44
|
+
readonly label: "1Y";
|
|
45
|
+
readonly months: 12;
|
|
46
|
+
}];
|
|
47
|
+
export type Period = (typeof PERIODS)[number]['value'];
|
|
48
|
+
/** Input values intentionally omit a zone; the UI explicitly labels them UTC. */
|
|
49
|
+
export declare function presetRange(period: Period, now?: Date): DateRange;
|
|
50
|
+
/** Preserve the initial thirty-day window while allowing minute precision. */
|
|
51
|
+
export declare function defaultRange(now?: Date): DateRange;
|
|
52
|
+
export declare function rangeProblem(range: DateRange): string | undefined;
|
|
53
|
+
export {};
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
const MINUTE_MS = 60_000;
|
|
2
|
+
const DAY_MS = 86_400_000;
|
|
3
|
+
export const PERIODS = [
|
|
4
|
+
{ value: '3h', label: '3H', hours: 3 },
|
|
5
|
+
{ value: '6h', label: '6H', hours: 6 },
|
|
6
|
+
{ value: '12h', label: '12H', hours: 12 },
|
|
7
|
+
{ value: '24h', label: '24H', hours: 24 },
|
|
8
|
+
{ value: '5d', label: '5D', hours: 120 },
|
|
9
|
+
{ value: '1w', label: '1W', hours: 168 },
|
|
10
|
+
{ value: '4w', label: '4W', hours: 672 },
|
|
11
|
+
{ value: '3mo', label: '3M', months: 3 },
|
|
12
|
+
{ value: '6mo', label: '6M', months: 6 },
|
|
13
|
+
{ value: '1y', label: '1Y', months: 12 },
|
|
14
|
+
];
|
|
15
|
+
/** Calendar subtraction clamps month-end and leap-day dates without rolling forward. */
|
|
16
|
+
function monthsBefore(end, months) {
|
|
17
|
+
const start = new Date(end);
|
|
18
|
+
start.setUTCDate(1);
|
|
19
|
+
start.setUTCMonth(start.getUTCMonth() - months);
|
|
20
|
+
const lastDay = new Date(start);
|
|
21
|
+
lastDay.setUTCMonth(lastDay.getUTCMonth() + 1);
|
|
22
|
+
lastDay.setUTCDate(0);
|
|
23
|
+
start.setUTCDate(Math.min(end.getUTCDate(), lastDay.getUTCDate()));
|
|
24
|
+
return start;
|
|
25
|
+
}
|
|
26
|
+
/** Input values intentionally omit a zone; the UI explicitly labels them UTC. */
|
|
27
|
+
export function presetRange(period, now = new Date()) {
|
|
28
|
+
const option = PERIODS.find((item) => item.value === period);
|
|
29
|
+
if (option === undefined)
|
|
30
|
+
throw new Error(`Unknown reporting period: ${period}`);
|
|
31
|
+
const end = new Date(Math.floor(now.getTime() / MINUTE_MS) * MINUTE_MS);
|
|
32
|
+
const start = 'hours' in option
|
|
33
|
+
? new Date(end.getTime() - option.hours * 60 * MINUTE_MS)
|
|
34
|
+
: monthsBefore(end, option.months);
|
|
35
|
+
return { from: start.toISOString().slice(0, 16), to: end.toISOString().slice(0, 16) };
|
|
36
|
+
}
|
|
37
|
+
/** Preserve the initial thirty-day window while allowing minute precision. */
|
|
38
|
+
export function defaultRange(now = new Date()) {
|
|
39
|
+
const end = new Date(Math.floor(now.getTime() / MINUTE_MS) * MINUTE_MS);
|
|
40
|
+
return {
|
|
41
|
+
from: new Date(end.getTime() - 30 * DAY_MS).toISOString().slice(0, 16),
|
|
42
|
+
to: end.toISOString().slice(0, 16),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export function rangeProblem(range) {
|
|
46
|
+
for (const value of [range.from, range.to]) {
|
|
47
|
+
const instant = Date.parse(`${value}:00Z`);
|
|
48
|
+
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$/.test(value) ||
|
|
49
|
+
!Number.isFinite(instant) ||
|
|
50
|
+
new Date(instant).toISOString().slice(0, 16) !== value) {
|
|
51
|
+
return 'Choose a valid start and end date and time (UTC).';
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
if (range.from >= range.to)
|
|
55
|
+
return 'The start must be before the end date and time.';
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
package/dist/server.js
CHANGED
|
@@ -75,6 +75,7 @@
|
|
|
75
75
|
* does not know is served as {@link DEFAULT_CONTENT_TYPE}, which a sniffing
|
|
76
76
|
* browser is otherwise free to re-read as something it will execute.
|
|
77
77
|
*/
|
|
78
|
+
import { parseViewGranularity, VIEW_GRANULARITIES } from './view-granularity.js';
|
|
78
79
|
import { createServer } from 'node:http';
|
|
79
80
|
import { join, sep } from 'node:path';
|
|
80
81
|
import { FileNotFoundError } from 'blogwright-core';
|
|
@@ -121,7 +122,15 @@ const QUERY_ROUTE_PREFIX = '/api/queries/';
|
|
|
121
122
|
* because its default is `config.analytics.bots`, applied inside the port -
|
|
122
123
|
* this module must not restate it.
|
|
123
124
|
*/
|
|
124
|
-
const QUERY_STRING_PARAMS = [
|
|
125
|
+
const QUERY_STRING_PARAMS = [
|
|
126
|
+
'from',
|
|
127
|
+
'to',
|
|
128
|
+
'includeBots',
|
|
129
|
+
'granularity',
|
|
130
|
+
'splitBots',
|
|
131
|
+
'path',
|
|
132
|
+
'country',
|
|
133
|
+
];
|
|
125
134
|
/** The two spellings the bot-inclusion flag takes, mapped to what it means. */
|
|
126
135
|
const INCLUDE_BOTS_VALUES = { true: true, false: false };
|
|
127
136
|
/**
|
|
@@ -291,15 +300,44 @@ function parseQueryParams(search) {
|
|
|
291
300
|
}
|
|
292
301
|
}
|
|
293
302
|
const range = { from: requiredParam(search, 'from'), to: requiredParam(search, 'to') };
|
|
303
|
+
const interval = search.get('granularity');
|
|
304
|
+
let granularity;
|
|
305
|
+
if (interval !== null) {
|
|
306
|
+
if (search.getAll('granularity').length !== 1)
|
|
307
|
+
throw new RequestRejected(400, 'granularity must be provided once');
|
|
308
|
+
try {
|
|
309
|
+
granularity = parseViewGranularity(interval);
|
|
310
|
+
}
|
|
311
|
+
catch (err) {
|
|
312
|
+
throw new RequestRejected(400, describeError(err));
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
const country = search.get('country');
|
|
316
|
+
if (search.getAll('country').length > 1)
|
|
317
|
+
throw new RequestRejected(400, 'country must be provided once');
|
|
318
|
+
const path = search.get('path');
|
|
319
|
+
if (search.getAll('path').length > 1)
|
|
320
|
+
throw new RequestRejected(400, 'path must be provided once');
|
|
321
|
+
const split = search.get('splitBots');
|
|
322
|
+
if (split !== null &&
|
|
323
|
+
(search.getAll('splitBots').length !== 1 || !Object.hasOwn(INCLUDE_BOTS_VALUES, split))) {
|
|
324
|
+
throw new RequestRejected(400, 'splitBots must be provided once as true or false');
|
|
325
|
+
}
|
|
326
|
+
const extra = {
|
|
327
|
+
...(path === null ? {} : { path }),
|
|
328
|
+
...(country === null ? {} : { country }),
|
|
329
|
+
...(granularity === undefined ? {} : { granularity }),
|
|
330
|
+
...(split === null ? {} : { splitBots: INCLUDE_BOTS_VALUES[split] }),
|
|
331
|
+
};
|
|
294
332
|
const flag = search.get('includeBots');
|
|
295
333
|
if (flag === null) {
|
|
296
334
|
// Absent on purpose: `config.analytics.bots` decides, inside the port.
|
|
297
|
-
return { range };
|
|
335
|
+
return { range, ...extra };
|
|
298
336
|
}
|
|
299
337
|
if (!Object.hasOwn(INCLUDE_BOTS_VALUES, flag)) {
|
|
300
338
|
throw new RequestRejected(400, `query parameter "includeBots" must be ${Object.keys(INCLUDE_BOTS_VALUES).join(' or ')}, got "${flag}"`);
|
|
301
339
|
}
|
|
302
|
-
return { range, includeBots: INCLUDE_BOTS_VALUES[flag] };
|
|
340
|
+
return { range, ...extra, includeBots: INCLUDE_BOTS_VALUES[flag] };
|
|
303
341
|
}
|
|
304
342
|
/**
|
|
305
343
|
* Start the dashboard's HTTP listener on {@link LOOPBACK_ADDRESS}. Resolves
|
|
@@ -391,8 +429,12 @@ export async function createDashboardServer(opts) {
|
|
|
391
429
|
const rows = await opts.query.run(resolved.name, params);
|
|
392
430
|
return {
|
|
393
431
|
name: resolved.name,
|
|
394
|
-
rowMeaning:
|
|
395
|
-
|
|
432
|
+
rowMeaning: resolved.name === 'views-over-time' &&
|
|
433
|
+
params.granularity !== undefined &&
|
|
434
|
+
params.granularity !== '24h'
|
|
435
|
+
? `one UTC bucket of ${VIEW_GRANULARITIES[params.granularity].label}, labelled by its start, and the number of requests served in it`
|
|
436
|
+
: definition.rowMeaning,
|
|
437
|
+
resultColumns: resolved.resultColumns,
|
|
396
438
|
rows,
|
|
397
439
|
};
|
|
398
440
|
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Fixed SQL projections for disjoint bot/non-bot contributions to report totals. */
|
|
2
|
+
export function withTrafficBreakdown(name, statement) {
|
|
3
|
+
if (name === 'unique-visitors') {
|
|
4
|
+
return statement
|
|
5
|
+
.replace('count(DISTINCT visitor_key) AS daily_unique_visitors', 'count(DISTINCT visitor_key) AS daily_unique_visitors, count(DISTINCT visitor_key) FILTER (WHERE NOT coalesce(is_bot, false)) AS non_bot')
|
|
6
|
+
.replace('SELECT day,\n daily_unique_visitors,', 'SELECT day,\n daily_unique_visitors, non_bot, daily_unique_visitors - non_bot AS bot,');
|
|
7
|
+
}
|
|
8
|
+
if (name === 'cache-hit-ratio') {
|
|
9
|
+
return statement
|
|
10
|
+
.replace('AS cache_hits\n', "AS cache_hits, count(*) FILTER (WHERE result_type IN ('Hit', 'RefreshHit') AND coalesce(is_bot, false)) AS bot_hits\n")
|
|
11
|
+
.replace('AS cache_hit_ratio\n', 'AS cache_hit_ratio, CAST(bot_hits AS DOUBLE) / requests AS bot, CAST(cache_hits - bot_hits AS DOUBLE) / requests AS non_bot\n');
|
|
12
|
+
}
|
|
13
|
+
const projection = name === 'row-count' ? 'count(*) AS row_count' : 'count(*) AS views';
|
|
14
|
+
return statement.replace(projection, `${projection}, count(*) FILTER (WHERE coalesce(is_bot, false)) AS bot, count(*) FILTER (WHERE NOT coalesce(is_bot, false)) AS non_bot`);
|
|
15
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** Bounded transform events: fixed categories only, never input or error details. */
|
|
2
|
+
export interface TransformBatchDiagnostic {
|
|
3
|
+
readonly event: 'transform_batch';
|
|
4
|
+
mapped: number;
|
|
5
|
+
processingFailed: number;
|
|
6
|
+
reasons: {
|
|
7
|
+
invalid_payload: number;
|
|
8
|
+
schema_rejected: number;
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
/** One event per completed batch and per uncached secret read, including retries. */
|
|
12
|
+
export type TransformDiagnostic = TransformBatchDiagnostic | {
|
|
13
|
+
readonly event: 'salt_read';
|
|
14
|
+
readonly outcome: 'success' | 'failure';
|
|
15
|
+
};
|
|
16
|
+
/** Diagnostic sinks must return normally so observation cannot change delivery. */
|
|
17
|
+
export type TransformDiagnostics = (event: TransformDiagnostic) => void;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -6,9 +6,10 @@
|
|
|
6
6
|
* prefix rather than failing the batch" - plus the cold-start secret read step
|
|
7
7
|
* 3 calls for.
|
|
8
8
|
*
|
|
9
|
-
* It owns
|
|
9
|
+
* It owns the envelope types, the base64 and
|
|
10
10
|
* JSON boundary, the cold-start read of the long-lived salt secret, and the
|
|
11
|
-
* translation of one `MapRecordResult` into one Firehose response entry
|
|
11
|
+
* translation of one `MapRecordResult` into one Firehose response entry, plus
|
|
12
|
+
* bounded diagnostics through an injected port. Every
|
|
12
13
|
* decision about *what a row is* - which field fills which column, which
|
|
13
14
|
* values are unusable, how `visitor_key` is derived and under which day's salt
|
|
14
15
|
* - belongs to `map-record.ts` and is forwarded here, never re-decided.
|
|
@@ -42,8 +43,7 @@
|
|
|
42
43
|
* over a 2^32 space. A blanket catch around `mapRecord` would convert those
|
|
43
44
|
* throws into `ProcessingFailed` for every record - the whole batch to the
|
|
44
45
|
* error prefix, no error anywhere, an empty dashboard, and nothing to say why.
|
|
45
|
-
*
|
|
46
|
-
* this module's own boundary. A throw from the mapping propagates, the
|
|
46
|
+
* The mapping remains outside the JSON boundary catch. A throw from the mapping propagates, the
|
|
47
47
|
* invocation fails, and Firehose retries the batch and raises its own error
|
|
48
48
|
* metric. Loudly is the point.
|
|
49
49
|
*
|
|
@@ -69,6 +69,7 @@
|
|
|
69
69
|
* caller that hands the constructed handler to Lambda.
|
|
70
70
|
*/
|
|
71
71
|
import type { SecretsManagerClient } from 'blogwright-core';
|
|
72
|
+
import type { TransformDiagnostics } from './diagnostics.js';
|
|
72
73
|
/**
|
|
73
74
|
* One record as Firehose sends it. The event carries more - `invocationId`,
|
|
74
75
|
* `deliveryStreamArn`, `region`, and per-record source metadata - and this
|
|
@@ -131,5 +132,5 @@ type Environment = Readonly<Record<string, string | undefined>>;
|
|
|
131
132
|
* call disable an execution environment for its whole life, failing every
|
|
132
133
|
* batch it ever sees; leaving it uncached means the next invocation retries.
|
|
133
134
|
*/
|
|
134
|
-
export declare function createTransformHandler(secrets: SaltSecretStore, env?:
|
|
135
|
+
export declare function createTransformHandler(secrets: SaltSecretStore, env: Environment, diagnostics?: TransformDiagnostics): (request: FirehoseTransformRequest) => Promise<FirehoseTransformResponse>;
|
|
135
136
|
export {};
|
|
@@ -6,9 +6,10 @@
|
|
|
6
6
|
* prefix rather than failing the batch" - plus the cold-start secret read step
|
|
7
7
|
* 3 calls for.
|
|
8
8
|
*
|
|
9
|
-
* It owns
|
|
9
|
+
* It owns the envelope types, the base64 and
|
|
10
10
|
* JSON boundary, the cold-start read of the long-lived salt secret, and the
|
|
11
|
-
* translation of one `MapRecordResult` into one Firehose response entry
|
|
11
|
+
* translation of one `MapRecordResult` into one Firehose response entry, plus
|
|
12
|
+
* bounded diagnostics through an injected port. Every
|
|
12
13
|
* decision about *what a row is* - which field fills which column, which
|
|
13
14
|
* values are unusable, how `visitor_key` is derived and under which day's salt
|
|
14
15
|
* - belongs to `map-record.ts` and is forwarded here, never re-decided.
|
|
@@ -42,8 +43,7 @@
|
|
|
42
43
|
* over a 2^32 space. A blanket catch around `mapRecord` would convert those
|
|
43
44
|
* throws into `ProcessingFailed` for every record - the whole batch to the
|
|
44
45
|
* error prefix, no error anywhere, an empty dashboard, and nothing to say why.
|
|
45
|
-
*
|
|
46
|
-
* this module's own boundary. A throw from the mapping propagates, the
|
|
46
|
+
* The mapping remains outside the JSON boundary catch. A throw from the mapping propagates, the
|
|
47
47
|
* invocation fails, and Firehose retries the batch and raises its own error
|
|
48
48
|
* metric. Loudly is the point.
|
|
49
49
|
*
|
|
@@ -135,13 +135,20 @@ function decodePayload(data) {
|
|
|
135
135
|
* carries records of two days, and one salt chosen per invocation would be the
|
|
136
136
|
* wrong day's for every record on the far side of it.
|
|
137
137
|
*/
|
|
138
|
-
function transformRecord(record, saltSecret) {
|
|
138
|
+
function transformRecord(record, saltSecret, counts) {
|
|
139
139
|
const payload = decodePayload(record.data);
|
|
140
|
-
if (payload === undefined)
|
|
140
|
+
if (payload === undefined) {
|
|
141
|
+
counts.processingFailed += 1;
|
|
142
|
+
counts.reasons.invalid_payload += 1;
|
|
141
143
|
return { recordId: record.recordId, result: PROCESSING_FAILED };
|
|
144
|
+
}
|
|
142
145
|
const mapped = mapRecord(payload, saltSecret);
|
|
143
|
-
if (!mapped.mapped)
|
|
146
|
+
if (!mapped.mapped) {
|
|
147
|
+
counts.processingFailed += 1;
|
|
148
|
+
counts.reasons.schema_rejected += 1;
|
|
144
149
|
return { recordId: record.recordId, result: PROCESSING_FAILED };
|
|
150
|
+
}
|
|
151
|
+
counts.mapped += 1;
|
|
145
152
|
return {
|
|
146
153
|
recordId: record.recordId,
|
|
147
154
|
result: OK,
|
|
@@ -162,16 +169,32 @@ function transformRecord(record, saltSecret) {
|
|
|
162
169
|
* call disable an execution environment for its whole life, failing every
|
|
163
170
|
* batch it ever sees; leaving it uncached means the next invocation retries.
|
|
164
171
|
*/
|
|
165
|
-
export function createTransformHandler(secrets, env =
|
|
172
|
+
export function createTransformHandler(secrets, env, diagnostics = () => { }) {
|
|
166
173
|
const secretName = requireSaltSecretName(env);
|
|
167
174
|
let cached;
|
|
168
175
|
async function readSaltSecret() {
|
|
169
|
-
if (cached === undefined)
|
|
170
|
-
|
|
176
|
+
if (cached === undefined) {
|
|
177
|
+
try {
|
|
178
|
+
cached = await loadSaltSecret(secrets, secretName);
|
|
179
|
+
}
|
|
180
|
+
catch (error) {
|
|
181
|
+
diagnostics({ event: 'salt_read', outcome: 'failure' });
|
|
182
|
+
throw error;
|
|
183
|
+
}
|
|
184
|
+
diagnostics({ event: 'salt_read', outcome: 'success' });
|
|
185
|
+
}
|
|
171
186
|
return cached;
|
|
172
187
|
}
|
|
173
188
|
return async function transformFirehoseRecords(request) {
|
|
174
189
|
const saltSecret = await readSaltSecret();
|
|
175
|
-
|
|
190
|
+
const counts = {
|
|
191
|
+
event: 'transform_batch',
|
|
192
|
+
mapped: 0,
|
|
193
|
+
processingFailed: 0,
|
|
194
|
+
reasons: { invalid_payload: 0, schema_rejected: 0 },
|
|
195
|
+
};
|
|
196
|
+
const records = request.records.map((record) => transformRecord(record, saltSecret, counts));
|
|
197
|
+
diagnostics(counts);
|
|
198
|
+
return { records };
|
|
176
199
|
};
|
|
177
200
|
}
|