@byline/analytics 4.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +373 -0
- package/README.md +41 -0
- package/dist/analytics.d.ts +40 -0
- package/dist/analytics.js +221 -0
- package/dist/analytics.test.node.d.ts +8 -0
- package/dist/analytics.test.node.js +216 -0
- package/dist/browser-boundary.test.node.d.ts +8 -0
- package/dist/browser-boundary.test.node.js +26 -0
- package/dist/config.d.ts +52 -0
- package/dist/config.js +92 -0
- package/dist/config.test.node.d.ts +8 -0
- package/dist/config.test.node.js +30 -0
- package/dist/crawler-user-agents.d.ts +8 -0
- package/dist/crawler-user-agents.js +11 -0
- package/dist/date.d.ts +13 -0
- package/dist/date.js +31 -0
- package/dist/dedupe.d.ts +18 -0
- package/dist/dedupe.js +37 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +15 -0
- package/dist/maintenance.d.ts +29 -0
- package/dist/maintenance.js +68 -0
- package/dist/maintenance.test.node.d.ts +8 -0
- package/dist/maintenance.test.node.js +46 -0
- package/dist/metrics.d.ts +18 -0
- package/dist/metrics.js +67 -0
- package/dist/normalize.d.ts +21 -0
- package/dist/normalize.js +90 -0
- package/dist/privacy-statement.d.ts +29 -0
- package/dist/privacy-statement.js +43 -0
- package/dist/privacy-statement.test.node.d.ts +8 -0
- package/dist/privacy-statement.test.node.js +32 -0
- package/dist/runtime.d.ts +15 -0
- package/dist/runtime.js +33 -0
- package/dist/types.d.ts +166 -0
- package/dist/types.js +8 -0
- package/dist/visitor.d.ts +10 -0
- package/dist/visitor.js +24 -0
- package/dist/visitor.test.node.d.ts +8 -0
- package/dist/visitor.test.node.js +19 -0
- package/package.json +74 -0
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { describe, expect, it } from 'vitest';
|
|
9
|
+
import { createAnalytics } from './analytics.js';
|
|
10
|
+
const TEST_BROWSER_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ' +
|
|
11
|
+
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
|
|
12
|
+
describe('analytics ingest', () => {
|
|
13
|
+
it('accepts an exact beacon payload and stores only normalized, pseudonymous data', async () => {
|
|
14
|
+
const store = new MemoryStore();
|
|
15
|
+
const analytics = fixtureAnalytics(store);
|
|
16
|
+
await expect(analytics.ingest({
|
|
17
|
+
method: 'POST',
|
|
18
|
+
body: JSON.stringify({
|
|
19
|
+
v: 1,
|
|
20
|
+
kind: 'page',
|
|
21
|
+
path: '/news//today?preview=1#lead',
|
|
22
|
+
ref: 'https://search.example/results',
|
|
23
|
+
}),
|
|
24
|
+
origin: 'https://example.com',
|
|
25
|
+
userAgent: TEST_BROWSER_USER_AGENT,
|
|
26
|
+
clientIp: '203.0.113.5',
|
|
27
|
+
country: 'th',
|
|
28
|
+
})).resolves.toEqual({ status: 202, accepted: true });
|
|
29
|
+
expect(store.events).toHaveLength(1);
|
|
30
|
+
expect(store.events[0]).toMatchObject({
|
|
31
|
+
path: '/news/today',
|
|
32
|
+
kind: 'page',
|
|
33
|
+
source: 'beacon',
|
|
34
|
+
referrerHost: 'search.example',
|
|
35
|
+
country: 'TH',
|
|
36
|
+
});
|
|
37
|
+
expect(store.events[0]?.visitorHash).toMatch(/^[a-f0-9]{64}$/u);
|
|
38
|
+
expect(JSON.stringify(store.events[0])).not.toContain('203.0.113.5');
|
|
39
|
+
});
|
|
40
|
+
it('never stores the reserved aggregate sentinel as a referrer', async () => {
|
|
41
|
+
const store = new MemoryStore();
|
|
42
|
+
const analytics = fixtureAnalytics(store);
|
|
43
|
+
await expect(analytics.ingest({
|
|
44
|
+
...ingestRequest(),
|
|
45
|
+
body: payload({ ref: 'https://__OTHER__./crafted' }),
|
|
46
|
+
})).resolves.toEqual({ status: 202, accepted: true });
|
|
47
|
+
expect(store.events[0]?.referrerHost).toBeNull();
|
|
48
|
+
});
|
|
49
|
+
it('memoizes the daily salt and rotates it at the next UTC day', async () => {
|
|
50
|
+
const store = new MemoryStore();
|
|
51
|
+
let now = new Date('2026-08-23T23:59:59.999Z');
|
|
52
|
+
const analytics = createAnalytics({
|
|
53
|
+
store,
|
|
54
|
+
publicDomains: ['example.com'],
|
|
55
|
+
now: () => now,
|
|
56
|
+
createSalt: () => new Uint8Array(32).fill(7),
|
|
57
|
+
});
|
|
58
|
+
await analytics.ingest(ingestRequest());
|
|
59
|
+
await analytics.ingest({ ...ingestRequest(), body: payload({ path: '/another' }) });
|
|
60
|
+
expect(store.saltCalls).toBe(1);
|
|
61
|
+
now = new Date('2026-08-24T00:00:00.000Z');
|
|
62
|
+
await analytics.ingest({ ...ingestRequest(), body: payload({ path: '/next-day' }) });
|
|
63
|
+
expect(store.saltCalls).toBe(2);
|
|
64
|
+
});
|
|
65
|
+
it('silently separates origin, admin, bot, prefetch, and replay drops', async () => {
|
|
66
|
+
const store = new MemoryStore();
|
|
67
|
+
const analytics = fixtureAnalytics(store);
|
|
68
|
+
const request = ingestRequest();
|
|
69
|
+
await expect(analytics.ingest({ ...request, origin: 'https://wrong.example' })).resolves.toEqual({ status: 204, accepted: false, reason: 'origin' });
|
|
70
|
+
await expect(analytics.ingest({ ...request, body: payload({ path: '/_byline/users' }) })).resolves.toEqual({ status: 204, accepted: false, reason: 'admin-path' });
|
|
71
|
+
await expect(analytics.ingest({ ...request, userAgent: 'Googlebot/2.1' })).resolves.toEqual({
|
|
72
|
+
status: 204,
|
|
73
|
+
accepted: false,
|
|
74
|
+
reason: 'bot',
|
|
75
|
+
});
|
|
76
|
+
await expect(analytics.ingest({ ...request, secPurpose: 'prefetch' })).resolves.toEqual({
|
|
77
|
+
status: 204,
|
|
78
|
+
accepted: false,
|
|
79
|
+
reason: 'prefetch',
|
|
80
|
+
});
|
|
81
|
+
await expect(analytics.ingest({ ...request, clientIp: null })).resolves.toEqual({
|
|
82
|
+
status: 204,
|
|
83
|
+
accepted: false,
|
|
84
|
+
reason: 'client-ip',
|
|
85
|
+
});
|
|
86
|
+
await expect(analytics.ingest(request)).resolves.toEqual({ status: 202, accepted: true });
|
|
87
|
+
await expect(analytics.ingest(request)).resolves.toEqual({
|
|
88
|
+
status: 204,
|
|
89
|
+
accepted: false,
|
|
90
|
+
reason: 'dedupe',
|
|
91
|
+
});
|
|
92
|
+
expect(store.events).toHaveLength(1);
|
|
93
|
+
});
|
|
94
|
+
it('rejects oversized, malformed, and extended payloads before storage', async () => {
|
|
95
|
+
const store = new MemoryStore();
|
|
96
|
+
const analytics = fixtureAnalytics(store);
|
|
97
|
+
await expect(analytics.ingest({ ...ingestRequest(), body: 'x'.repeat(1_025) })).resolves.toEqual({ status: 400, accepted: false, reason: 'body-size' });
|
|
98
|
+
await expect(analytics.ingest({ ...ingestRequest(), body: '{' })).resolves.toEqual({
|
|
99
|
+
status: 400,
|
|
100
|
+
accepted: false,
|
|
101
|
+
reason: 'malformed',
|
|
102
|
+
});
|
|
103
|
+
await expect(analytics.ingest({
|
|
104
|
+
...ingestRequest(),
|
|
105
|
+
body: JSON.stringify({
|
|
106
|
+
v: 1,
|
|
107
|
+
kind: 'page',
|
|
108
|
+
path: '/',
|
|
109
|
+
ref: '',
|
|
110
|
+
siteId: 'forged',
|
|
111
|
+
}),
|
|
112
|
+
})).resolves.toEqual({ status: 400, accepted: false, reason: 'malformed' });
|
|
113
|
+
expect(store.saltCalls).toBe(0);
|
|
114
|
+
expect(store.events).toEqual([]);
|
|
115
|
+
});
|
|
116
|
+
it('does not poison replay dedupe when persistence fails', async () => {
|
|
117
|
+
const store = new MemoryStore();
|
|
118
|
+
store.failNextInsert = true;
|
|
119
|
+
const analytics = fixtureAnalytics(store);
|
|
120
|
+
await expect(analytics.ingest(ingestRequest())).rejects.toThrow('insert failed');
|
|
121
|
+
await expect(analytics.ingest(ingestRequest())).resolves.toEqual({
|
|
122
|
+
status: 202,
|
|
123
|
+
accepted: true,
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
describe('analytics reporting coverage', () => {
|
|
128
|
+
it('reports independent finite-retention boundaries without shortening headline history', async () => {
|
|
129
|
+
const store = new MemoryStore();
|
|
130
|
+
store.earliestReportDay = '2025-01-01';
|
|
131
|
+
const analytics = createAnalytics({
|
|
132
|
+
store,
|
|
133
|
+
publicDomains: ['example.com'],
|
|
134
|
+
pathRetentionDays: 90,
|
|
135
|
+
referrerRetentionDays: 180,
|
|
136
|
+
now: () => new Date('2026-08-24T12:00:00.000Z'),
|
|
137
|
+
});
|
|
138
|
+
await expect(analytics.getReportCoverage()).resolves.toEqual({
|
|
139
|
+
summaryFrom: '2025-01-01',
|
|
140
|
+
pathsFrom: '2026-05-26',
|
|
141
|
+
referrersFrom: '2026-02-25',
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
it('reports no coverage before any raw or aggregate day exists', async () => {
|
|
145
|
+
await expect(fixtureAnalytics(new MemoryStore()).getReportCoverage()).resolves.toEqual({
|
|
146
|
+
summaryFrom: null,
|
|
147
|
+
pathsFrom: null,
|
|
148
|
+
referrersFrom: null,
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
});
|
|
152
|
+
function fixtureAnalytics(store) {
|
|
153
|
+
return createAnalytics({
|
|
154
|
+
store,
|
|
155
|
+
publicDomains: ['example.com'],
|
|
156
|
+
now: () => new Date('2026-08-23T08:00:00.000Z'),
|
|
157
|
+
createSalt: () => new Uint8Array(32).fill(7),
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
function payload(overrides = {}) {
|
|
161
|
+
return JSON.stringify({ v: 1, kind: 'page', path: '/news', ref: '', ...overrides });
|
|
162
|
+
}
|
|
163
|
+
function ingestRequest() {
|
|
164
|
+
return {
|
|
165
|
+
method: 'POST',
|
|
166
|
+
body: payload(),
|
|
167
|
+
origin: 'https://example.com',
|
|
168
|
+
userAgent: TEST_BROWSER_USER_AGENT,
|
|
169
|
+
clientIp: '203.0.113.5',
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
class MemoryStore {
|
|
173
|
+
events = [];
|
|
174
|
+
saltCalls = 0;
|
|
175
|
+
failNextInsert = false;
|
|
176
|
+
earliestReportDay = null;
|
|
177
|
+
async getOrCreateDailySalt() {
|
|
178
|
+
this.saltCalls += 1;
|
|
179
|
+
return new Uint8Array(32).fill(7);
|
|
180
|
+
}
|
|
181
|
+
async insertEvent(event) {
|
|
182
|
+
if (this.failNextInsert) {
|
|
183
|
+
this.failNextInsert = false;
|
|
184
|
+
throw new Error('insert failed');
|
|
185
|
+
}
|
|
186
|
+
this.events.push(event);
|
|
187
|
+
}
|
|
188
|
+
async getRollupCursor() {
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
async getEarliestEventDay() {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
async getEarliestReportDay() {
|
|
195
|
+
return this.earliestReportDay;
|
|
196
|
+
}
|
|
197
|
+
async rebuildDay(_options) { }
|
|
198
|
+
async prune(_options) {
|
|
199
|
+
return { events: 0, salts: 0, pathAggregates: 0, referrerAggregates: 0 };
|
|
200
|
+
}
|
|
201
|
+
async getSummary(_range) {
|
|
202
|
+
return [];
|
|
203
|
+
}
|
|
204
|
+
async getTopPaths(_query) {
|
|
205
|
+
return { rows: [], total: 0 };
|
|
206
|
+
}
|
|
207
|
+
async getReferrers(_query) {
|
|
208
|
+
return { rows: [], total: 0 };
|
|
209
|
+
}
|
|
210
|
+
async getCountries(_range) {
|
|
211
|
+
return [];
|
|
212
|
+
}
|
|
213
|
+
async deleteEvents() {
|
|
214
|
+
return 0;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { readFileSync } from 'node:fs';
|
|
9
|
+
import { dirname, join } from 'node:path';
|
|
10
|
+
import { fileURLToPath } from 'node:url';
|
|
11
|
+
import { describe, expect, it } from 'vitest';
|
|
12
|
+
const packageDir = join(dirname(fileURLToPath(import.meta.url)), '..');
|
|
13
|
+
describe('@byline/analytics browser boundary', () => {
|
|
14
|
+
it('exports config directly without loading the Node-only root barrel', () => {
|
|
15
|
+
const manifest = JSON.parse(readFileSync(join(packageDir, 'package.json'), 'utf8'));
|
|
16
|
+
expect(manifest.exports['./config']).toEqual({
|
|
17
|
+
types: './dist/config.d.ts',
|
|
18
|
+
import: './dist/config.js',
|
|
19
|
+
require: './dist/config.js',
|
|
20
|
+
});
|
|
21
|
+
});
|
|
22
|
+
it('keeps runtime Node built-ins out of the browser-safe config module', () => {
|
|
23
|
+
const source = readFileSync(join(packageDir, 'src/config.ts'), 'utf8');
|
|
24
|
+
expect(source).not.toMatch(/(?:from|import\()\s*['"]node:/u);
|
|
25
|
+
});
|
|
26
|
+
});
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import type { BylineLogger } from '@byline/core';
|
|
9
|
+
/**
|
|
10
|
+
* Reserved `path` / `referrer_host` value holding each day's overflow beyond
|
|
11
|
+
* the cardinality cap. Ingest normalization must never be able to produce it,
|
|
12
|
+
* and both SQL drivers enforce that with a CHECK constraint.
|
|
13
|
+
*/
|
|
14
|
+
export declare const ANALYTICS_OVERFLOW_KEY = "__other__";
|
|
15
|
+
export declare const ANALYTICS_EVENT_RETENTION_DAYS = 90;
|
|
16
|
+
export declare const ANALYTICS_FIXED_DASHBOARD_PERIODS: readonly [7, 30, 90];
|
|
17
|
+
export declare const ANALYTICS_DASHBOARD_PERIODS: readonly [7, 30, 90, "ytd", "all"];
|
|
18
|
+
export type AnalyticsDashboardPeriod = (typeof ANALYTICS_DASHBOARD_PERIODS)[number];
|
|
19
|
+
export declare const ANALYTICS_LONGEST_DASHBOARD_PERIOD_DAYS: number;
|
|
20
|
+
export declare const ANALYTICS_MIN_CARDINALITY_CAP = 20;
|
|
21
|
+
export declare const ANALYTICS_DEFAULT_CARDINALITY_CAP = 1000;
|
|
22
|
+
export declare const ANALYTICS_MAX_TOP_LIMIT = 100;
|
|
23
|
+
export declare const ANALYTICS_MAX_BODY_BYTES = 1024;
|
|
24
|
+
export declare const ANALYTICS_MAX_PATH_LENGTH = 512;
|
|
25
|
+
export declare const ANALYTICS_DEDUPE_WINDOW_MS = 10000;
|
|
26
|
+
export interface AnalyticsConfig {
|
|
27
|
+
/** Hosts accepted by the Origin/Referer filter, with optional development ports. */
|
|
28
|
+
publicDomains: readonly string[];
|
|
29
|
+
/** Admin/internal path prefixes silently discarded before storage. */
|
|
30
|
+
ignoredPathPrefixes?: readonly string[];
|
|
31
|
+
pathCardinalityCap?: number;
|
|
32
|
+
referrerCardinalityCap?: number;
|
|
33
|
+
/** `null` retains aggregate strings indefinitely. */
|
|
34
|
+
pathRetentionDays?: number | null;
|
|
35
|
+
/** `null` retains aggregate strings indefinitely. */
|
|
36
|
+
referrerRetentionDays?: number | null;
|
|
37
|
+
/** Per-instance replay cache bound. Defaults to 10,000 entries. */
|
|
38
|
+
dedupeMaxEntries?: number;
|
|
39
|
+
logger?: BylineLogger;
|
|
40
|
+
}
|
|
41
|
+
export interface ResolvedAnalyticsConfig {
|
|
42
|
+
publicDomains: ReadonlySet<string>;
|
|
43
|
+
ignoredPathPrefixes: readonly string[];
|
|
44
|
+
pathCardinalityCap: number;
|
|
45
|
+
referrerCardinalityCap: number;
|
|
46
|
+
pathRetentionDays: number | null;
|
|
47
|
+
referrerRetentionDays: number | null;
|
|
48
|
+
dedupeMaxEntries: number;
|
|
49
|
+
logger?: BylineLogger;
|
|
50
|
+
}
|
|
51
|
+
export declare function isAnalyticsDashboardPeriod(value: unknown): value is AnalyticsDashboardPeriod;
|
|
52
|
+
export declare function resolveAnalyticsConfig(config: AnalyticsConfig): ResolvedAnalyticsConfig;
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
// Browser-safe public subpath: @byline/analytics/config.
|
|
9
|
+
// Keep every runtime import in this module browser-compatible.
|
|
10
|
+
/**
|
|
11
|
+
* Reserved `path` / `referrer_host` value holding each day's overflow beyond
|
|
12
|
+
* the cardinality cap. Ingest normalization must never be able to produce it,
|
|
13
|
+
* and both SQL drivers enforce that with a CHECK constraint.
|
|
14
|
+
*/
|
|
15
|
+
export const ANALYTICS_OVERFLOW_KEY = '__other__';
|
|
16
|
+
export const ANALYTICS_EVENT_RETENTION_DAYS = 90;
|
|
17
|
+
export const ANALYTICS_FIXED_DASHBOARD_PERIODS = [7, 30, 90];
|
|
18
|
+
export const ANALYTICS_DASHBOARD_PERIODS = [
|
|
19
|
+
...ANALYTICS_FIXED_DASHBOARD_PERIODS,
|
|
20
|
+
'ytd',
|
|
21
|
+
'all',
|
|
22
|
+
];
|
|
23
|
+
export const ANALYTICS_LONGEST_DASHBOARD_PERIOD_DAYS = Math.max(...ANALYTICS_FIXED_DASHBOARD_PERIODS);
|
|
24
|
+
export const ANALYTICS_MIN_CARDINALITY_CAP = 20;
|
|
25
|
+
export const ANALYTICS_DEFAULT_CARDINALITY_CAP = 1_000;
|
|
26
|
+
export const ANALYTICS_MAX_TOP_LIMIT = 100;
|
|
27
|
+
export const ANALYTICS_MAX_BODY_BYTES = 1_024;
|
|
28
|
+
export const ANALYTICS_MAX_PATH_LENGTH = 512;
|
|
29
|
+
export const ANALYTICS_DEDUPE_WINDOW_MS = 10_000;
|
|
30
|
+
export function isAnalyticsDashboardPeriod(value) {
|
|
31
|
+
return ANALYTICS_DASHBOARD_PERIODS.some((period) => period === value);
|
|
32
|
+
}
|
|
33
|
+
export function resolveAnalyticsConfig(config) {
|
|
34
|
+
if (config.publicDomains.length === 0) {
|
|
35
|
+
throw new Error('analytics.publicDomains must contain at least one host');
|
|
36
|
+
}
|
|
37
|
+
const publicDomains = new Set(config.publicDomains.map(normalizeConfiguredHost));
|
|
38
|
+
const ignoredPathPrefixes = (config.ignoredPathPrefixes ?? ['/_byline', '/api']).map(normalizeIgnoredPrefix);
|
|
39
|
+
return {
|
|
40
|
+
publicDomains,
|
|
41
|
+
ignoredPathPrefixes: [...new Set(ignoredPathPrefixes)],
|
|
42
|
+
pathCardinalityCap: validateCap(config.pathCardinalityCap ?? ANALYTICS_DEFAULT_CARDINALITY_CAP, 'pathCardinalityCap'),
|
|
43
|
+
referrerCardinalityCap: validateCap(config.referrerCardinalityCap ?? ANALYTICS_DEFAULT_CARDINALITY_CAP, 'referrerCardinalityCap'),
|
|
44
|
+
pathRetentionDays: validateRetention(config.pathRetentionDays, 'pathRetentionDays'),
|
|
45
|
+
referrerRetentionDays: validateRetention(config.referrerRetentionDays, 'referrerRetentionDays'),
|
|
46
|
+
dedupeMaxEntries: validatePositiveInteger(config.dedupeMaxEntries ?? 10_000, 'dedupeMaxEntries'),
|
|
47
|
+
...(config.logger == null ? {} : { logger: config.logger }),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
function normalizeConfiguredHost(value) {
|
|
51
|
+
const trimmed = value.trim();
|
|
52
|
+
if (trimmed.length === 0)
|
|
53
|
+
throw new Error('analytics.publicDomains cannot contain a blank host');
|
|
54
|
+
try {
|
|
55
|
+
const url = new URL(trimmed.includes('://') ? trimmed : `https://${trimmed}`);
|
|
56
|
+
if (url.username || url.password || url.pathname !== '/' || url.search || url.hash) {
|
|
57
|
+
throw new Error('host must not include credentials, a path, a query, or a fragment');
|
|
58
|
+
}
|
|
59
|
+
return url.host.toLowerCase().replace(/\.$/, '');
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
throw new Error(`Invalid analytics public domain "${value}": ${error.message}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
function normalizeIgnoredPrefix(value) {
|
|
66
|
+
const trimmed = value.trim();
|
|
67
|
+
if (!trimmed.startsWith('/')) {
|
|
68
|
+
throw new Error(`Analytics ignored path prefix "${value}" must start with /`);
|
|
69
|
+
}
|
|
70
|
+
return trimmed.length > 1 ? trimmed.replace(/\/+$/, '') : trimmed;
|
|
71
|
+
}
|
|
72
|
+
function validateCap(value, label) {
|
|
73
|
+
if (!Number.isSafeInteger(value) || value < ANALYTICS_MIN_CARDINALITY_CAP) {
|
|
74
|
+
throw new Error(`analytics.${label} must be an integer at least ${ANALYTICS_MIN_CARDINALITY_CAP}`);
|
|
75
|
+
}
|
|
76
|
+
return value;
|
|
77
|
+
}
|
|
78
|
+
function validateRetention(value, label) {
|
|
79
|
+
if (value == null)
|
|
80
|
+
return null;
|
|
81
|
+
if (!Number.isSafeInteger(value) || value < ANALYTICS_LONGEST_DASHBOARD_PERIOD_DAYS) {
|
|
82
|
+
throw new Error(`analytics.${label} must be null or an integer at least ` +
|
|
83
|
+
`${ANALYTICS_LONGEST_DASHBOARD_PERIOD_DAYS} days (the longest dashboard period)`);
|
|
84
|
+
}
|
|
85
|
+
return value;
|
|
86
|
+
}
|
|
87
|
+
function validatePositiveInteger(value, label) {
|
|
88
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
89
|
+
throw new Error(`analytics.${label} must be a positive integer`);
|
|
90
|
+
}
|
|
91
|
+
return value;
|
|
92
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { describe, expect, it } from 'vitest';
|
|
9
|
+
import { ANALYTICS_DASHBOARD_PERIODS, ANALYTICS_FIXED_DASHBOARD_PERIODS, ANALYTICS_LONGEST_DASHBOARD_PERIOD_DAYS, isAnalyticsDashboardPeriod, resolveAnalyticsConfig, } from './config.js';
|
|
10
|
+
describe('resolveAnalyticsConfig', () => {
|
|
11
|
+
it('normalizes hosts and supplies bounded defaults', () => {
|
|
12
|
+
const config = resolveAnalyticsConfig({ publicDomains: ['HTTPS://Example.COM:443'] });
|
|
13
|
+
expect([...config.publicDomains]).toEqual(['example.com']);
|
|
14
|
+
expect(config.pathCardinalityCap).toBe(1_000);
|
|
15
|
+
expect(config.referrerCardinalityCap).toBe(1_000);
|
|
16
|
+
expect(config.pathRetentionDays).toBeNull();
|
|
17
|
+
expect(config.referrerRetentionDays).toBeNull();
|
|
18
|
+
});
|
|
19
|
+
it('rejects a cap below the dashboard limit and retention below its longest period', () => {
|
|
20
|
+
expect(() => resolveAnalyticsConfig({ publicDomains: ['example.com'], pathCardinalityCap: 19 })).toThrow(/at least 20/u);
|
|
21
|
+
expect(() => resolveAnalyticsConfig({ publicDomains: ['example.com'], pathRetentionDays: 89 })).toThrow(/at least 90 days/u);
|
|
22
|
+
});
|
|
23
|
+
it('derives the finite retention floor from only the fixed-day periods', () => {
|
|
24
|
+
expect(ANALYTICS_DASHBOARD_PERIODS).toEqual([7, 30, 90, 'ytd', 'all']);
|
|
25
|
+
expect(ANALYTICS_LONGEST_DASHBOARD_PERIOD_DAYS).toBe(Math.max(...ANALYTICS_FIXED_DASHBOARD_PERIODS));
|
|
26
|
+
expect(isAnalyticsDashboardPeriod('ytd')).toBe(true);
|
|
27
|
+
expect(isAnalyticsDashboardPeriod('all')).toBe(true);
|
|
28
|
+
expect(isAnalyticsDashboardPeriod('30')).toBe(false);
|
|
29
|
+
});
|
|
30
|
+
});
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export declare function isCrawlerUserAgent(userAgent: string): boolean;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { isbot } from 'isbot';
|
|
9
|
+
export function isCrawlerUserAgent(userAgent) {
|
|
10
|
+
return isbot(userAgent);
|
|
11
|
+
}
|
package/dist/date.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import type { AnalyticsDay } from './types.js';
|
|
9
|
+
export declare function analyticsDay(date: Date): AnalyticsDay;
|
|
10
|
+
export declare function assertAnalyticsDay(value: string, label?: string): AnalyticsDay;
|
|
11
|
+
export declare function addAnalyticsDays(day: AnalyticsDay, amount: number): AnalyticsDay;
|
|
12
|
+
export declare function startOfAnalyticsDay(day: AnalyticsDay): Date;
|
|
13
|
+
export declare function compareAnalyticsDays(left: AnalyticsDay, right: AnalyticsDay): number;
|
package/dist/date.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
const DAY_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
|
|
9
|
+
export function analyticsDay(date) {
|
|
10
|
+
return date.toISOString().slice(0, 10);
|
|
11
|
+
}
|
|
12
|
+
export function assertAnalyticsDay(value, label = 'day') {
|
|
13
|
+
if (!DAY_PATTERN.test(value))
|
|
14
|
+
throw new Error(`${label} must use YYYY-MM-DD`);
|
|
15
|
+
const date = new Date(`${value}T00:00:00.000Z`);
|
|
16
|
+
if (!Number.isFinite(date.valueOf()) || analyticsDay(date) !== value) {
|
|
17
|
+
throw new Error(`${label} must be a real UTC calendar day`);
|
|
18
|
+
}
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
export function addAnalyticsDays(day, amount) {
|
|
22
|
+
const date = new Date(`${assertAnalyticsDay(day)}T00:00:00.000Z`);
|
|
23
|
+
date.setUTCDate(date.getUTCDate() + amount);
|
|
24
|
+
return analyticsDay(date);
|
|
25
|
+
}
|
|
26
|
+
export function startOfAnalyticsDay(day) {
|
|
27
|
+
return new Date(`${assertAnalyticsDay(day)}T00:00:00.000Z`);
|
|
28
|
+
}
|
|
29
|
+
export function compareAnalyticsDays(left, right) {
|
|
30
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
31
|
+
}
|
package/dist/dedupe.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export declare class AnalyticsDedupeCache {
|
|
9
|
+
#private;
|
|
10
|
+
private readonly maxEntries;
|
|
11
|
+
private readonly windowMs;
|
|
12
|
+
constructor(maxEntries: number, windowMs: number);
|
|
13
|
+
/** Returns true for a replay; otherwise records this event as the newest occurrence. */
|
|
14
|
+
check(key: string, nowMs: number): boolean;
|
|
15
|
+
/** Remove a provisional entry when persistence failed. */
|
|
16
|
+
forget(key: string): void;
|
|
17
|
+
private evict;
|
|
18
|
+
}
|
package/dist/dedupe.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export class AnalyticsDedupeCache {
|
|
9
|
+
maxEntries;
|
|
10
|
+
windowMs;
|
|
11
|
+
#entries = new Map();
|
|
12
|
+
constructor(maxEntries, windowMs) {
|
|
13
|
+
this.maxEntries = maxEntries;
|
|
14
|
+
this.windowMs = windowMs;
|
|
15
|
+
}
|
|
16
|
+
/** Returns true for a replay; otherwise records this event as the newest occurrence. */
|
|
17
|
+
check(key, nowMs) {
|
|
18
|
+
const previous = this.#entries.get(key);
|
|
19
|
+
if (previous != null && nowMs - previous < this.windowMs)
|
|
20
|
+
return true;
|
|
21
|
+
this.#entries.delete(key);
|
|
22
|
+
this.#entries.set(key, nowMs);
|
|
23
|
+
this.evict(nowMs);
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
/** Remove a provisional entry when persistence failed. */
|
|
27
|
+
forget(key) {
|
|
28
|
+
this.#entries.delete(key);
|
|
29
|
+
}
|
|
30
|
+
evict(nowMs) {
|
|
31
|
+
for (const [key, timestamp] of this.#entries) {
|
|
32
|
+
if (this.#entries.size <= this.maxEntries && nowMs - timestamp < this.windowMs)
|
|
33
|
+
break;
|
|
34
|
+
this.#entries.delete(key);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export { Analytics, type CreateAnalyticsOptions, createAnalytics, } from './analytics.js';
|
|
9
|
+
export { ANALYTICS_DEDUPE_WINDOW_MS, ANALYTICS_DEFAULT_CARDINALITY_CAP, ANALYTICS_EVENT_RETENTION_DAYS, ANALYTICS_FIXED_DASHBOARD_PERIODS, ANALYTICS_LONGEST_DASHBOARD_PERIOD_DAYS, ANALYTICS_MAX_BODY_BYTES, ANALYTICS_MAX_PATH_LENGTH, ANALYTICS_MAX_TOP_LIMIT, ANALYTICS_MIN_CARDINALITY_CAP, ANALYTICS_OVERFLOW_KEY, type AnalyticsConfig, type ResolvedAnalyticsConfig, resolveAnalyticsConfig, } from './config.js';
|
|
10
|
+
export { addAnalyticsDays, analyticsDay, assertAnalyticsDay, compareAnalyticsDays, startOfAnalyticsDay, } from './date.js';
|
|
11
|
+
export { type AnalyticsMaintenanceOptions, type AnalyticsMaintenanceResult, type DefineAnalyticsRollupTaskOptions, defineAnalyticsRollupTask, runAnalyticsMaintenance, } from './maintenance.js';
|
|
12
|
+
export { isIgnoredAnalyticsPath, normalizeAnalyticsPath, normalizeCountry, normalizeReferrerHost, parseBeaconPayload, requestHost, } from './normalize.js';
|
|
13
|
+
export { type AnalyticsPrivacyStatement, type AnalyticsPrivacyStatementOptions, createAnalyticsPrivacyStatement, } from './privacy-statement.js';
|
|
14
|
+
export { getAnalytics, isAnalyticsRegistered, registerAnalytics, tryGetAnalytics, } from './runtime.js';
|
|
15
|
+
export { canonicalVisitorIdentity, hashAnalyticsVisitor } from './visitor.js';
|
|
16
|
+
export type { AnalyticsCountryTotal, AnalyticsDateRange, AnalyticsDay, AnalyticsDeleteEventsOptions, AnalyticsDropReason, AnalyticsEvent, AnalyticsEventKind, AnalyticsEventSource, AnalyticsIngestRequest, AnalyticsIngestResult, AnalyticsLimitQuery, AnalyticsMetricsSnapshot, AnalyticsPathTotal, AnalyticsPruneOptions, AnalyticsPruneResult, AnalyticsRankedTotals, AnalyticsReferrerTotal, AnalyticsRejectReason, AnalyticsReportCoverage, AnalyticsRollupDayOptions, AnalyticsStore, AnalyticsSummary, AnalyticsSummaryDay, AnalyticsTopQuery, } from './types.js';
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
export { Analytics, createAnalytics, } from './analytics.js';
|
|
9
|
+
export { ANALYTICS_DEDUPE_WINDOW_MS, ANALYTICS_DEFAULT_CARDINALITY_CAP, ANALYTICS_EVENT_RETENTION_DAYS, ANALYTICS_FIXED_DASHBOARD_PERIODS, ANALYTICS_LONGEST_DASHBOARD_PERIOD_DAYS, ANALYTICS_MAX_BODY_BYTES, ANALYTICS_MAX_PATH_LENGTH, ANALYTICS_MAX_TOP_LIMIT, ANALYTICS_MIN_CARDINALITY_CAP, ANALYTICS_OVERFLOW_KEY, resolveAnalyticsConfig, } from './config.js';
|
|
10
|
+
export { addAnalyticsDays, analyticsDay, assertAnalyticsDay, compareAnalyticsDays, startOfAnalyticsDay, } from './date.js';
|
|
11
|
+
export { defineAnalyticsRollupTask, runAnalyticsMaintenance, } from './maintenance.js';
|
|
12
|
+
export { isIgnoredAnalyticsPath, normalizeAnalyticsPath, normalizeCountry, normalizeReferrerHost, parseBeaconPayload, requestHost, } from './normalize.js';
|
|
13
|
+
export { createAnalyticsPrivacyStatement, } from './privacy-statement.js';
|
|
14
|
+
export { getAnalytics, isAnalyticsRegistered, registerAnalytics, tryGetAnalytics, } from './runtime.js';
|
|
15
|
+
export { canonicalVisitorIdentity, hashAnalyticsVisitor } from './visitor.js';
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This Source Code is subject to the terms of the Mozilla Public
|
|
3
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
4
|
+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
5
|
+
*
|
|
6
|
+
* Copyright (c) Infonomic Company Limited
|
|
7
|
+
*/
|
|
8
|
+
import { type RecurringTaskDefinition } from '@byline/core';
|
|
9
|
+
import type { Analytics } from './analytics.js';
|
|
10
|
+
export interface AnalyticsMaintenanceOptions {
|
|
11
|
+
analytics: Analytics;
|
|
12
|
+
/** Defaults to seven complete days per scheduler invocation. */
|
|
13
|
+
maxDaysPerRun?: number;
|
|
14
|
+
/** Test seam. */
|
|
15
|
+
now?: () => Date;
|
|
16
|
+
}
|
|
17
|
+
export interface AnalyticsMaintenanceResult {
|
|
18
|
+
processedDays: string[];
|
|
19
|
+
workRemaining: boolean;
|
|
20
|
+
}
|
|
21
|
+
export declare function runAnalyticsMaintenance(options: AnalyticsMaintenanceOptions, context?: {
|
|
22
|
+
signal?: AbortSignal;
|
|
23
|
+
heartbeat?: () => Promise<void>;
|
|
24
|
+
}): Promise<AnalyticsMaintenanceResult>;
|
|
25
|
+
export interface DefineAnalyticsRollupTaskOptions extends AnalyticsMaintenanceOptions {
|
|
26
|
+
intervalMs?: number;
|
|
27
|
+
leaseMs?: number;
|
|
28
|
+
}
|
|
29
|
+
export declare function defineAnalyticsRollupTask(options: DefineAnalyticsRollupTaskOptions): RecurringTaskDefinition;
|