@wtfalch/reporting 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -0
- package/dist/analytics/collector.d.ts +40 -0
- package/dist/analytics/collector.js +146 -0
- package/dist/analytics/reader.d.ts +63 -0
- package/dist/analytics/reader.js +89 -0
- package/dist/analytics/schema.d.ts +59 -0
- package/dist/analytics/schema.js +126 -0
- package/dist/analytics/tasks.d.ts +32 -0
- package/dist/analytics/tasks.js +199 -0
- package/dist/analytics/write.d.ts +19 -0
- package/dist/analytics/write.js +20 -0
- package/dist/browser/index.d.ts +53 -0
- package/dist/browser/index.js +330 -0
- package/dist/errors/capture.d.ts +52 -0
- package/dist/errors/capture.js +187 -0
- package/dist/errors/fingerprint.d.ts +36 -0
- package/dist/errors/fingerprint.js +211 -0
- package/dist/errors/reader.d.ts +19 -0
- package/dist/errors/reader.js +66 -0
- package/dist/errors/redact.d.ts +46 -0
- package/dist/errors/redact.js +93 -0
- package/dist/errors/tasks.d.ts +16 -0
- package/dist/errors/tasks.js +50 -0
- package/dist/fake/index.d.ts +5 -1
- package/dist/fake/index.js +28 -0
- package/dist/housekeeping/index.d.ts +5 -0
- package/dist/housekeeping/index.js +18 -6
- package/dist/index.d.ts +5 -1
- package/dist/index.js +61 -5
- package/dist/ingest-limit.d.ts +22 -0
- package/dist/ingest-limit.js +39 -0
- package/dist/migrations/0002_analytics.sql +350 -0
- package/dist/migrations/0003_errors.sql +118 -0
- package/dist/next/client-errors.d.ts +46 -0
- package/dist/next/client-errors.js +165 -0
- package/dist/next/index.d.ts +55 -0
- package/dist/next/index.js +72 -0
- package/dist/react/charts.d.ts +24 -0
- package/dist/react/charts.js +45 -0
- package/dist/react/errors.d.ts +12 -0
- package/dist/react/errors.js +17 -0
- package/dist/react/index.d.ts +21 -0
- package/dist/react/index.js +52 -0
- package/dist/schema.d.ts +8 -0
- package/dist/schema.js +8 -0
- package/dist/settings.d.ts +1 -0
- package/dist/settings.js +15 -8
- package/dist/tables.d.ts +2287 -253
- package/dist/tables.js +111 -1
- package/dist/types.d.ts +70 -2
- package/package.json +19 -1
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
-- @wtfalch/reporting 0.2.0: the analytics record. Mirrors src/tables.ts.
|
|
2
|
+
--
|
|
3
|
+
-- Three tables (docs/plans/reporting.md D13). `reporting_analytics` holds
|
|
4
|
+
-- raw rows for a window; the two rollups hold counts and nothing that
|
|
5
|
+
-- identifies anybody, which is what lets them outlive the raw rows and be
|
|
6
|
+
-- shown to a customer organisation. Every statement is idempotent; the file
|
|
7
|
+
-- is applied by the host's own migrate script after `reporting-migrations`
|
|
8
|
+
-- copies it in as the next number.
|
|
9
|
+
|
|
10
|
+
-- -------------------------------------------------------------- raw rows
|
|
11
|
+
-- No ip, no user agent, no email, no name, no free text longer than a path.
|
|
12
|
+
-- `user_id` is set only server-side from the session, never from the beacon;
|
|
13
|
+
-- `visitor_id` is the first-party cookie's value when consent gave one.
|
|
14
|
+
-- `path` is the pathname only, normalised to a route pattern before it gets
|
|
15
|
+
-- here (addendum A7), so no identifier or token survives into storage.
|
|
16
|
+
create table if not exists reporting_analytics (
|
|
17
|
+
id bigint generated always as identity primary key,
|
|
18
|
+
occurred_at timestamptz not null,
|
|
19
|
+
received_at timestamptz not null default now(),
|
|
20
|
+
site text not null,
|
|
21
|
+
tenant_id uuid,
|
|
22
|
+
visitor_id text,
|
|
23
|
+
session_id text,
|
|
24
|
+
user_id text,
|
|
25
|
+
name text not null,
|
|
26
|
+
path text not null,
|
|
27
|
+
referrer_host text,
|
|
28
|
+
device text not null,
|
|
29
|
+
country text,
|
|
30
|
+
props jsonb not null default '{}'::jsonb,
|
|
31
|
+
schema_version smallint not null default 1,
|
|
32
|
+
constraint reporting_analytics_site_check check (site ~ '^[a-z][a-z0-9-]{0,63}$'),
|
|
33
|
+
constraint reporting_analytics_name_check check (name ~ '^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)?$'),
|
|
34
|
+
constraint reporting_analytics_path_check check (length(path) between 1 and 512 and position('?' in path) = 0),
|
|
35
|
+
constraint reporting_analytics_referrer_check check (referrer_host is null or length(referrer_host) <= 253),
|
|
36
|
+
constraint reporting_analytics_device_check check (device in ('desktop', 'mobile', 'tablet', 'bot', 'unknown')),
|
|
37
|
+
constraint reporting_analytics_country_check check (country is null or country ~ '^[A-Z]{2}$'),
|
|
38
|
+
constraint reporting_analytics_ids_check check (
|
|
39
|
+
(visitor_id is null or length(visitor_id) between 8 and 64)
|
|
40
|
+
and (session_id is null or length(session_id) between 8 and 64)
|
|
41
|
+
and (user_id is null or length(user_id) between 1 and 256)
|
|
42
|
+
),
|
|
43
|
+
-- The beacon's clock is a claim; ten minutes either side of arrival is
|
|
44
|
+
-- the most it may say.
|
|
45
|
+
constraint reporting_analytics_clock_check check (abs(extract(epoch from (occurred_at - received_at))) <= 600),
|
|
46
|
+
constraint reporting_analytics_props_object_check check (jsonb_typeof(props) = 'object'),
|
|
47
|
+
constraint reporting_analytics_props_size_check check (octet_length(props::text) <= 4096),
|
|
48
|
+
constraint reporting_analytics_props_flat_check check (not jsonb_path_exists(props, 'strict $.* ? (@.type() == "object" || @.type() == "array")', '{}', true)),
|
|
49
|
+
constraint reporting_analytics_props_keys_check check (
|
|
50
|
+
not (props ?| array['email', 'name', 'display_name', 'displayName', 'password', 'secret', 'token', 'authorization'])
|
|
51
|
+
)
|
|
52
|
+
);
|
|
53
|
+
create index if not exists reporting_analytics_site_time_idx
|
|
54
|
+
on reporting_analytics (site, occurred_at desc, id desc);
|
|
55
|
+
create index if not exists reporting_analytics_tenant_time_idx
|
|
56
|
+
on reporting_analytics (tenant_id, occurred_at desc, id desc) where tenant_id is not null;
|
|
57
|
+
create index if not exists reporting_analytics_user_idx
|
|
58
|
+
on reporting_analytics (user_id) where user_id is not null;
|
|
59
|
+
create index if not exists reporting_analytics_site_name_time_idx
|
|
60
|
+
on reporting_analytics (site, name, occurred_at desc);
|
|
61
|
+
-- The rollups read one UTC day at a time.
|
|
62
|
+
create index if not exists reporting_analytics_day_idx
|
|
63
|
+
on reporting_analytics (((occurred_at at time zone 'UTC')::date), site);
|
|
64
|
+
alter table reporting_analytics set (autovacuum_vacuum_scale_factor = 0.02);
|
|
65
|
+
|
|
66
|
+
-- ------------------------------------------------------------ daily rollup
|
|
67
|
+
-- One row per day, site, grain and dimension value. `grain` names which
|
|
68
|
+
-- dimension the row is grouped by (addendum A4: distinct counts are stored
|
|
69
|
+
-- at every level a reader promises, never summed from finer groups):
|
|
70
|
+
-- site the whole site (every dimension null)
|
|
71
|
+
-- tenant per organisation
|
|
72
|
+
-- name per event name
|
|
73
|
+
-- path page views per normalised route, top 200 a day plus '(other)'
|
|
74
|
+
-- device, country, referrer per dimension, referrers top 50 plus '(other)'
|
|
75
|
+
-- A day's whole set is replaced atomically by reporting_rollup_day, so a
|
|
76
|
+
-- bucket that leaves the top list does not linger. The generated key
|
|
77
|
+
-- columns make the nullable dimensions part of the primary key.
|
|
78
|
+
create table if not exists reporting_analytics_daily (
|
|
79
|
+
day date not null,
|
|
80
|
+
site text not null,
|
|
81
|
+
grain text not null,
|
|
82
|
+
tenant_id uuid,
|
|
83
|
+
name text,
|
|
84
|
+
path text,
|
|
85
|
+
device text,
|
|
86
|
+
country text,
|
|
87
|
+
referrer_host text,
|
|
88
|
+
views integer not null,
|
|
89
|
+
visitors integer not null,
|
|
90
|
+
people integer not null,
|
|
91
|
+
tenant_key text generated always as (coalesce(tenant_id::text, '')) stored,
|
|
92
|
+
name_key text generated always as (coalesce(name, '')) stored,
|
|
93
|
+
path_key text generated always as (coalesce(path, '')) stored,
|
|
94
|
+
device_key text generated always as (coalesce(device, '')) stored,
|
|
95
|
+
country_key text generated always as (coalesce(country, '')) stored,
|
|
96
|
+
referrer_key text generated always as (coalesce(referrer_host, '')) stored,
|
|
97
|
+
constraint reporting_analytics_daily_grain_check check (grain in ('site', 'tenant', 'name', 'path', 'device', 'country', 'referrer')),
|
|
98
|
+
constraint reporting_analytics_daily_pkey primary key (day, site, grain, tenant_key, name_key, path_key, device_key, country_key, referrer_key)
|
|
99
|
+
);
|
|
100
|
+
create index if not exists reporting_analytics_daily_site_grain_idx
|
|
101
|
+
on reporting_analytics_daily (site, grain, day desc);
|
|
102
|
+
|
|
103
|
+
-- ----------------------------------------------------------- weekly rollup
|
|
104
|
+
-- Visitors and people per ISO week (the Monday), and how many of them were
|
|
105
|
+
-- seen in one of the four weeks before, inside the raw window: the one
|
|
106
|
+
-- question the daily table cannot answer.
|
|
107
|
+
create table if not exists reporting_analytics_weekly (
|
|
108
|
+
week date not null,
|
|
109
|
+
site text not null,
|
|
110
|
+
tenant_id uuid,
|
|
111
|
+
visitors integer not null,
|
|
112
|
+
returning_visitors integer not null,
|
|
113
|
+
people integer not null,
|
|
114
|
+
returning_people integer not null,
|
|
115
|
+
tenant_key text generated always as (coalesce(tenant_id::text, '')) stored,
|
|
116
|
+
constraint reporting_analytics_weekly_pkey primary key (week, site, tenant_key)
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
-- The rollups' completed-history watermark (addendum A3), one text value
|
|
120
|
+
-- per task, moved in the same transaction as the aggregate set it vouches
|
|
121
|
+
-- for and fenced by the claim token.
|
|
122
|
+
alter table reporting_tasks add column if not exists watermark text;
|
|
123
|
+
|
|
124
|
+
-- Settings the next release reserved: the raw window, the consent mode and
|
|
125
|
+
-- the signed-in switch, per site through the key's suffix.
|
|
126
|
+
alter table reporting_settings drop constraint if exists reporting_settings_key_check;
|
|
127
|
+
alter table reporting_settings add constraint reporting_settings_key_check
|
|
128
|
+
check (key ~ '^(events|analytics)\.[a-z][a-z0-9_]*$');
|
|
129
|
+
|
|
130
|
+
-- --------------------------------------------------------- reporting_rollup_day
|
|
131
|
+
-- Replaces one UTC day's aggregate set. Definer, because the runtime role
|
|
132
|
+
-- may not write the rollups directly: their contents are derived, and only
|
|
133
|
+
-- this function derives them. Returns the number of rows written.
|
|
134
|
+
create or replace function reporting_rollup_day(d date)
|
|
135
|
+
returns integer
|
|
136
|
+
language plpgsql
|
|
137
|
+
security definer
|
|
138
|
+
set search_path = pg_catalog, public
|
|
139
|
+
as $$
|
|
140
|
+
declare
|
|
141
|
+
n integer;
|
|
142
|
+
lo timestamptz := (d::timestamp at time zone 'UTC');
|
|
143
|
+
hi timestamptz := ((d + 1)::timestamp at time zone 'UTC');
|
|
144
|
+
begin
|
|
145
|
+
if d is null then
|
|
146
|
+
raise exception 'reporting_rollup_day: a day is required';
|
|
147
|
+
end if;
|
|
148
|
+
delete from reporting_analytics_daily where day = d;
|
|
149
|
+
with raw as (
|
|
150
|
+
select site, tenant_id, visitor_id, user_id, name, path, device, country, referrer_host
|
|
151
|
+
from reporting_analytics
|
|
152
|
+
where occurred_at >= lo and occurred_at < hi
|
|
153
|
+
),
|
|
154
|
+
top_paths as (
|
|
155
|
+
select site, path from (
|
|
156
|
+
select site, path, count(*) as c,
|
|
157
|
+
row_number() over (partition by site order by count(*) desc, path) as rn
|
|
158
|
+
from raw where name = 'page.view' group by site, path
|
|
159
|
+
) t where rn <= 200
|
|
160
|
+
),
|
|
161
|
+
top_refs as (
|
|
162
|
+
select site, referrer_host from (
|
|
163
|
+
select site, referrer_host, count(*) as c,
|
|
164
|
+
row_number() over (partition by site order by count(*) desc, referrer_host) as rn
|
|
165
|
+
from raw where referrer_host is not null group by site, referrer_host
|
|
166
|
+
) t where rn <= 50
|
|
167
|
+
),
|
|
168
|
+
rows_out as (
|
|
169
|
+
-- site
|
|
170
|
+
select site, 'site' as grain, null::uuid as tenant_id, null::text as name, null::text as path,
|
|
171
|
+
null::text as device, null::text as country, null::text as referrer_host,
|
|
172
|
+
count(*) as views, count(distinct visitor_id) as visitors, count(distinct user_id) as people
|
|
173
|
+
from raw group by site
|
|
174
|
+
union all
|
|
175
|
+
-- tenant
|
|
176
|
+
select site, 'tenant', tenant_id, null, null, null, null, null,
|
|
177
|
+
count(*), count(distinct visitor_id), count(distinct user_id)
|
|
178
|
+
from raw where tenant_id is not null group by site, tenant_id
|
|
179
|
+
union all
|
|
180
|
+
-- name
|
|
181
|
+
select site, 'name', null, name, null, null, null, null,
|
|
182
|
+
count(*), count(distinct visitor_id), count(distinct user_id)
|
|
183
|
+
from raw group by site, name
|
|
184
|
+
union all
|
|
185
|
+
-- path: page views only, top 200 plus (other)
|
|
186
|
+
select r.site, 'path', null, null,
|
|
187
|
+
case when tp.path is null then '(other)' else r.path end,
|
|
188
|
+
null, null, null,
|
|
189
|
+
count(*), count(distinct r.visitor_id), count(distinct r.user_id)
|
|
190
|
+
from raw r left join top_paths tp on tp.site = r.site and tp.path = r.path
|
|
191
|
+
where r.name = 'page.view'
|
|
192
|
+
group by r.site, case when tp.path is null then '(other)' else r.path end
|
|
193
|
+
union all
|
|
194
|
+
-- device
|
|
195
|
+
select site, 'device', null, null, null, device, null, null,
|
|
196
|
+
count(*), count(distinct visitor_id), count(distinct user_id)
|
|
197
|
+
from raw group by site, device
|
|
198
|
+
union all
|
|
199
|
+
-- country
|
|
200
|
+
select site, 'country', null, null, null, null, coalesce(country, 'ZZ'), null,
|
|
201
|
+
count(*), count(distinct visitor_id), count(distinct user_id)
|
|
202
|
+
from raw group by site, coalesce(country, 'ZZ')
|
|
203
|
+
union all
|
|
204
|
+
-- referrer: top 50 plus (other); rows with no referrer are '(direct)'
|
|
205
|
+
select r.site, 'referrer', null, null, null, null, null,
|
|
206
|
+
case when r.referrer_host is null then '(direct)'
|
|
207
|
+
when tr.referrer_host is null then '(other)'
|
|
208
|
+
else r.referrer_host end,
|
|
209
|
+
count(*), count(distinct r.visitor_id), count(distinct r.user_id)
|
|
210
|
+
from raw r left join top_refs tr on tr.site = r.site and tr.referrer_host = r.referrer_host
|
|
211
|
+
group by r.site, case when r.referrer_host is null then '(direct)'
|
|
212
|
+
when tr.referrer_host is null then '(other)'
|
|
213
|
+
else r.referrer_host end
|
|
214
|
+
)
|
|
215
|
+
insert into reporting_analytics_daily
|
|
216
|
+
(day, site, grain, tenant_id, name, path, device, country, referrer_host, views, visitors, people)
|
|
217
|
+
select d, site, grain, tenant_id, name, path, device, country, referrer_host,
|
|
218
|
+
views::integer, visitors::integer, people::integer
|
|
219
|
+
from rows_out;
|
|
220
|
+
get diagnostics n = row_count;
|
|
221
|
+
return n;
|
|
222
|
+
end
|
|
223
|
+
$$;
|
|
224
|
+
revoke all on function reporting_rollup_day(date) from public;
|
|
225
|
+
|
|
226
|
+
-- -------------------------------------------------------- reporting_rollup_week
|
|
227
|
+
-- Replaces one ISO week's rows (w is the Monday). Returning means seen in
|
|
228
|
+
-- any of the four weeks before, which is the raw lookback this rollup
|
|
229
|
+
-- declares; reporting_prune_analytics is told about it.
|
|
230
|
+
create or replace function reporting_rollup_week(w date)
|
|
231
|
+
returns integer
|
|
232
|
+
language plpgsql
|
|
233
|
+
security definer
|
|
234
|
+
set search_path = pg_catalog, public
|
|
235
|
+
as $$
|
|
236
|
+
declare
|
|
237
|
+
n integer;
|
|
238
|
+
lo timestamptz := (w::timestamp at time zone 'UTC');
|
|
239
|
+
hi timestamptz := ((w + 7)::timestamp at time zone 'UTC');
|
|
240
|
+
back timestamptz := ((w - 28)::timestamp at time zone 'UTC');
|
|
241
|
+
begin
|
|
242
|
+
if w is null or extract(isodow from w) <> 1 then
|
|
243
|
+
raise exception 'reporting_rollup_week: a Monday is required';
|
|
244
|
+
end if;
|
|
245
|
+
delete from reporting_analytics_weekly where week = w;
|
|
246
|
+
with this_week as (
|
|
247
|
+
select site, tenant_id, visitor_id, user_id
|
|
248
|
+
from reporting_analytics where occurred_at >= lo and occurred_at < hi
|
|
249
|
+
),
|
|
250
|
+
earlier as (
|
|
251
|
+
select distinct site, visitor_id, user_id
|
|
252
|
+
from reporting_analytics where occurred_at >= back and occurred_at < lo
|
|
253
|
+
),
|
|
254
|
+
scopes as (
|
|
255
|
+
select site, null::uuid as tenant_id, visitor_id, user_id from this_week
|
|
256
|
+
union all
|
|
257
|
+
select site, tenant_id, visitor_id, user_id from this_week where tenant_id is not null
|
|
258
|
+
)
|
|
259
|
+
insert into reporting_analytics_weekly
|
|
260
|
+
(week, site, tenant_id, visitors, returning_visitors, people, returning_people)
|
|
261
|
+
select w, s.site, s.tenant_id,
|
|
262
|
+
count(distinct s.visitor_id),
|
|
263
|
+
count(distinct s.visitor_id) filter (where exists (
|
|
264
|
+
select 1 from earlier e where e.site = s.site and e.visitor_id = s.visitor_id)),
|
|
265
|
+
count(distinct s.user_id),
|
|
266
|
+
count(distinct s.user_id) filter (where exists (
|
|
267
|
+
select 1 from earlier e where e.site = s.site and e.user_id = s.user_id))
|
|
268
|
+
from scopes s
|
|
269
|
+
group by s.site, s.tenant_id;
|
|
270
|
+
get diagnostics n = row_count;
|
|
271
|
+
return n;
|
|
272
|
+
end
|
|
273
|
+
$$;
|
|
274
|
+
revoke all on function reporting_rollup_week(date) from public;
|
|
275
|
+
|
|
276
|
+
-- ------------------------------------------------------ reporting_prune_analytics
|
|
277
|
+
-- The one way raw rows leave by age. `safe_before` is the earliest day the
|
|
278
|
+
-- rollups still need (the daily watermark, and the weekly watermark less
|
|
279
|
+
-- its four-week lookback, whichever is earlier): a cutoff at or past it is
|
|
280
|
+
-- refused, so pruning never outruns aggregation (addendum A3).
|
|
281
|
+
create or replace function reporting_prune_analytics(retention interval, batch integer, safe_before date)
|
|
282
|
+
returns integer
|
|
283
|
+
language plpgsql
|
|
284
|
+
security definer
|
|
285
|
+
set search_path = pg_catalog, public
|
|
286
|
+
as $$
|
|
287
|
+
declare
|
|
288
|
+
w interval := greatest(interval '7 days', least(coalesce(retention, interval '90 days'), interval '400 days'));
|
|
289
|
+
b integer := greatest(1, least(coalesce(batch, 5000), 5000));
|
|
290
|
+
cutoff timestamptz := now() - w;
|
|
291
|
+
n integer;
|
|
292
|
+
begin
|
|
293
|
+
if safe_before is null then
|
|
294
|
+
raise exception 'reporting_prune_analytics: the rollups have not completed a day yet; nothing is safe to prune';
|
|
295
|
+
end if;
|
|
296
|
+
if cutoff >= (safe_before::timestamp at time zone 'UTC') then
|
|
297
|
+
raise exception 'reporting_prune_analytics: cutoff % is not before the rollups'' watermark %', cutoff, safe_before;
|
|
298
|
+
end if;
|
|
299
|
+
with victims as (
|
|
300
|
+
select id from reporting_analytics
|
|
301
|
+
where occurred_at < cutoff
|
|
302
|
+
order by occurred_at, id
|
|
303
|
+
limit b
|
|
304
|
+
)
|
|
305
|
+
delete from reporting_analytics a using victims v where a.id = v.id;
|
|
306
|
+
get diagnostics n = row_count;
|
|
307
|
+
return n;
|
|
308
|
+
end
|
|
309
|
+
$$;
|
|
310
|
+
revoke all on function reporting_prune_analytics(interval, integer, date) from public;
|
|
311
|
+
|
|
312
|
+
-- -------------------------------------------------------- reporting_erase_person
|
|
313
|
+
-- The subject's raw rows go; the rollups hold no identifier and stay. Called
|
|
314
|
+
-- by the host inside its erasure transaction. Returns the rows deleted.
|
|
315
|
+
create or replace function reporting_erase_person(subject text)
|
|
316
|
+
returns integer
|
|
317
|
+
language plpgsql
|
|
318
|
+
security definer
|
|
319
|
+
set search_path = pg_catalog, public
|
|
320
|
+
as $$
|
|
321
|
+
declare
|
|
322
|
+
n integer;
|
|
323
|
+
begin
|
|
324
|
+
if subject is null or length(subject) = 0 then
|
|
325
|
+
raise exception 'reporting_erase_person: a subject id is required';
|
|
326
|
+
end if;
|
|
327
|
+
delete from reporting_analytics where user_id = subject;
|
|
328
|
+
get diagnostics n = row_count;
|
|
329
|
+
return n;
|
|
330
|
+
end
|
|
331
|
+
$$;
|
|
332
|
+
revoke all on function reporting_erase_person(text) from public;
|
|
333
|
+
|
|
334
|
+
-- The runtime role inserts and reads raw rows, reads the rollups, and
|
|
335
|
+
-- changes either only through the functions above.
|
|
336
|
+
do $$
|
|
337
|
+
declare
|
|
338
|
+
rt text := current_database() || '_rt';
|
|
339
|
+
begin
|
|
340
|
+
if exists (select 1 from pg_roles where rolname = rt) then
|
|
341
|
+
execute format('revoke update, delete, truncate on reporting_analytics from %I', rt);
|
|
342
|
+
execute format('revoke insert, update, delete, truncate on reporting_analytics_daily from %I', rt);
|
|
343
|
+
execute format('revoke insert, update, delete, truncate on reporting_analytics_weekly from %I', rt);
|
|
344
|
+
execute format('grant execute on function reporting_rollup_day(date) to %I', rt);
|
|
345
|
+
execute format('grant execute on function reporting_rollup_week(date) to %I', rt);
|
|
346
|
+
execute format('grant execute on function reporting_prune_analytics(interval, integer, date) to %I', rt);
|
|
347
|
+
execute format('grant execute on function reporting_erase_person(text) to %I', rt);
|
|
348
|
+
end if;
|
|
349
|
+
end
|
|
350
|
+
$$;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
-- @wtfalch/reporting 0.3.0: the error group table. Mirrors src/tables.ts.
|
|
2
|
+
--
|
|
3
|
+
-- One row per fingerprint (docs/plans/errors.md): an uncaught exception's
|
|
4
|
+
-- stack groups repeats into a count and a last-seen time instead of a
|
|
5
|
+
-- firehose of identical reporting_events rows. Every statement is
|
|
6
|
+
-- idempotent. Applied by the host's own migrate script after the host
|
|
7
|
+
-- copies this file into its drizzle/ directory as the next number
|
|
8
|
+
-- (reporting-migrations); never edited there.
|
|
9
|
+
--
|
|
10
|
+
-- Estate-shaped: the DO block at the end assumes the runtime-role convention
|
|
11
|
+
-- (a role named <database>_rt that owns nothing and serves the app). A host
|
|
12
|
+
-- without that role gets the table and no revokes take effect, which the
|
|
13
|
+
-- package README says plainly.
|
|
14
|
+
|
|
15
|
+
create table if not exists reporting_errors (
|
|
16
|
+
fingerprint text primary key,
|
|
17
|
+
site text not null,
|
|
18
|
+
kind text not null,
|
|
19
|
+
message text not null,
|
|
20
|
+
stack text,
|
|
21
|
+
runtime text not null,
|
|
22
|
+
release text,
|
|
23
|
+
first_seen_at timestamptz not null,
|
|
24
|
+
last_seen_at timestamptz not null,
|
|
25
|
+
occurrences bigint not null default 1,
|
|
26
|
+
state text not null default 'open',
|
|
27
|
+
resolved_at timestamptz,
|
|
28
|
+
resolved_by text,
|
|
29
|
+
tenant_id uuid,
|
|
30
|
+
request_id text,
|
|
31
|
+
-- The digest that groups occurrences (docs/plans/errors.md: sha256 of the
|
|
32
|
+
-- kind and the top normalised frames, truncated to 32 hex). Lower case
|
|
33
|
+
-- only: two casings of the same digest must not become two groups.
|
|
34
|
+
constraint reporting_errors_fingerprint_check check (fingerprint ~ '^[0-9a-f]{32}$'),
|
|
35
|
+
constraint reporting_errors_site_check check (site ~ '^[a-z][a-z0-9-]{0,63}$'),
|
|
36
|
+
constraint reporting_errors_message_check check (length(message) between 1 and 512),
|
|
37
|
+
-- Redacted before it is queued (src/schema.ts); this bound is a second net
|
|
38
|
+
-- against a pathological stack, not the redaction itself.
|
|
39
|
+
constraint reporting_errors_stack_check check (stack is null or length(stack) <= 16384),
|
|
40
|
+
constraint reporting_errors_runtime_check check (runtime in ('server', 'edge', 'browser')),
|
|
41
|
+
constraint reporting_errors_state_check check (state in ('open', 'resolved', 'ignored')),
|
|
42
|
+
-- The row exists because one occurrence created it; it can never describe
|
|
43
|
+
-- zero.
|
|
44
|
+
constraint reporting_errors_occurrences_check check (occurrences > 0),
|
|
45
|
+
-- The upsert only ever moves this forward. If it moved backward, the count
|
|
46
|
+
-- and the window it claims to describe would disagree.
|
|
47
|
+
constraint reporting_errors_seen_check check (last_seen_at >= first_seen_at),
|
|
48
|
+
-- Same pairing as reporting_events_actor_pair_check: who resolved it is
|
|
49
|
+
-- meaningless without when, and vice versa.
|
|
50
|
+
constraint reporting_errors_resolved_pair_check check ((resolved_at is null) = (resolved_by is null))
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
comment on column reporting_errors.tenant_id is
|
|
54
|
+
'From the most recent sample. No foreign key and no cascade, same as reporting_events.actor_id: the group outlives the request, and erasure must not delete the count.';
|
|
55
|
+
comment on column reporting_errors.request_id is
|
|
56
|
+
'From the most recent sample; overwritten on every occurrence, not a history.';
|
|
57
|
+
|
|
58
|
+
-- The default operator view: this site, still open, newest first. A partial
|
|
59
|
+
-- index because most errors end up resolved and should not cost this one.
|
|
60
|
+
create index if not exists reporting_errors_site_open_idx
|
|
61
|
+
on reporting_errors (site, last_seen_at desc) where state = 'open';
|
|
62
|
+
-- The same page's state filter, switched to resolved or ignored.
|
|
63
|
+
create index if not exists reporting_errors_state_idx
|
|
64
|
+
on reporting_errors (state, last_seen_at desc);
|
|
65
|
+
|
|
66
|
+
-- The one way rows leave reporting_errors on a timer. Unlike
|
|
67
|
+
-- reporting_prune_events, eligibility is state AND age, not age alone: only
|
|
68
|
+
-- 'resolved' or 'ignored' rows past the window are candidates. A row still
|
|
69
|
+
-- 'open' is NEVER a candidate here, no matter how old — an unresolved error
|
|
70
|
+
-- vanishing on a timer is the failure mode that makes an error tracker
|
|
71
|
+
-- untrustworthy, so the only way one leaves this table is an operator
|
|
72
|
+
-- resolving or ignoring it first, and it ageing out after. Same clamp and
|
|
73
|
+
-- batching shape as reporting_prune_events: the window is clamped to between
|
|
74
|
+
-- seven and four hundred days inside the function, and the batch is capped
|
|
75
|
+
-- at 5,000 so no statement holds a lock for long.
|
|
76
|
+
create or replace function reporting_prune_errors(retention interval, batch integer)
|
|
77
|
+
returns integer
|
|
78
|
+
language plpgsql
|
|
79
|
+
security definer
|
|
80
|
+
set search_path = pg_catalog, public
|
|
81
|
+
as $$
|
|
82
|
+
declare
|
|
83
|
+
w interval := greatest(interval '7 days', least(coalesce(retention, interval '30 days'), interval '400 days'));
|
|
84
|
+
b integer := greatest(1, least(coalesce(batch, 5000), 5000));
|
|
85
|
+
n integer;
|
|
86
|
+
begin
|
|
87
|
+
with victims as (
|
|
88
|
+
select fingerprint from reporting_errors
|
|
89
|
+
where state in ('resolved', 'ignored')
|
|
90
|
+
and last_seen_at < now() - w
|
|
91
|
+
order by last_seen_at, fingerprint
|
|
92
|
+
limit b
|
|
93
|
+
)
|
|
94
|
+
delete from reporting_errors e using victims v where e.fingerprint = v.fingerprint;
|
|
95
|
+
get diagnostics n = row_count;
|
|
96
|
+
return n;
|
|
97
|
+
end
|
|
98
|
+
$$;
|
|
99
|
+
revoke all on function reporting_prune_errors(interval, integer) from public;
|
|
100
|
+
|
|
101
|
+
comment on function reporting_prune_errors(interval, integer) is
|
|
102
|
+
'Deletes only resolved or ignored rows past the retention window. An open row is never a candidate, regardless of age: an unresolved error disappearing on a timer is the one failure mode that makes an error tracker untrustworthy. It leaves this table only after an operator resolves or ignores it, and then ages out.';
|
|
103
|
+
|
|
104
|
+
-- The runtime role keeps SELECT, INSERT and UPDATE: the capture path upserts
|
|
105
|
+
-- on repeat occurrences and an operator moves state through this table.
|
|
106
|
+
-- Nothing that deletes, the same as reporting_tasks and reporting_settings;
|
|
107
|
+
-- reporting_prune_errors runs as the function owner, not the runtime role,
|
|
108
|
+
-- so the grant below is execute only, same shape as reporting_prune_events.
|
|
109
|
+
do $$
|
|
110
|
+
declare
|
|
111
|
+
rt text := current_database() || '_rt';
|
|
112
|
+
begin
|
|
113
|
+
if exists (select 1 from pg_roles where rolname = rt) then
|
|
114
|
+
execute format('revoke delete, truncate on reporting_errors from %I', rt);
|
|
115
|
+
execute format('grant execute on function reporting_prune_errors(interval, integer) to %I', rt);
|
|
116
|
+
end if;
|
|
117
|
+
end
|
|
118
|
+
$$;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { Reporting } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The public ingest route for browser-reported errors
|
|
4
|
+
* (docs/plans/errors.md, "Surface"): modelled exactly on the analytics
|
|
5
|
+
* collector (`../analytics/collector.js`) — always 204, the same origin
|
|
6
|
+
* allowance, the same body-size cap — because that ingest problem is
|
|
7
|
+
* already solved there and a second set of limits would be a second set to
|
|
8
|
+
* get wrong.
|
|
9
|
+
*
|
|
10
|
+
* Two things differ from the analytics collector, both because a
|
|
11
|
+
* `Reporting` instance is already bound to one site:
|
|
12
|
+
* 1. There is no `sites` map to route a foreign origin's claim to a
|
|
13
|
+
* different site's table — `origins` below is a flat allow-list, and a
|
|
14
|
+
* batch claiming any other site is dropped, same-origin or not.
|
|
15
|
+
* 2. A capture goes through `reporting.captureError`, not a direct insert,
|
|
16
|
+
* so there is no `now`/`db` to take here: capture.ts already has its own.
|
|
17
|
+
*/
|
|
18
|
+
export interface ClientErrorCollectorOptions {
|
|
19
|
+
readonly reporting: Reporting;
|
|
20
|
+
/** Foreign origins allowed to post errors for this site, beside same-origin. */
|
|
21
|
+
readonly origins?: readonly string[];
|
|
22
|
+
/**
|
|
23
|
+
* The signed-in person, from the host's session, for a same-origin
|
|
24
|
+
* request only — a foreign origin never reads it (addendum A5's rule,
|
|
25
|
+
* same as the analytics collector's `getUserId`).
|
|
26
|
+
*/
|
|
27
|
+
readonly getUserId?: (request: Request) => Promise<string | null>;
|
|
28
|
+
/**
|
|
29
|
+
* The host's tenant lookup for that person. Unlike the analytics
|
|
30
|
+
* collector's `tenantFor`, there is no pathname to hand over: an error
|
|
31
|
+
* batch names no single route the way a page view does.
|
|
32
|
+
*/
|
|
33
|
+
readonly tenantFor?: (userId: string) => Promise<string | null>;
|
|
34
|
+
/**
|
|
35
|
+
* Captures allowed per minute per client address. An unauthenticated route
|
|
36
|
+
* that writes rows needs a ceiling that does not depend on the caller being
|
|
37
|
+
* our own beacon: the browser's 5-distinct/20-total cap is one line of
|
|
38
|
+
* client code away from being ignored, and `reporting_errors` never prunes
|
|
39
|
+
* an open row. Lower than the analytics default because an error is rarer
|
|
40
|
+
* than a page view by design; a client hitting this is already broken.
|
|
41
|
+
*/
|
|
42
|
+
readonly ratePerMinute?: number;
|
|
43
|
+
}
|
|
44
|
+
export declare function createClientErrorCollector(options: ClientErrorCollectorOptions): {
|
|
45
|
+
handle(request: Request): Promise<Response>;
|
|
46
|
+
};
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { ANALYTICS_LIMITS } from '../analytics/schema.js';
|
|
4
|
+
import { Buckets } from '../ingest-limit.js';
|
|
5
|
+
import { SITE_PATTERN } from '../schema.js';
|
|
6
|
+
/** At most this many entries of a batch are ever captured; the rest are dropped silently. */
|
|
7
|
+
const BATCH_MAX = 10;
|
|
8
|
+
/**
|
|
9
|
+
* Not `z.strictObject`, unlike the analytics beacon schema: a client that
|
|
10
|
+
* adds `tenantId`, `userId` or `requestId` here must have that field
|
|
11
|
+
* silently stripped, not the whole batch refused. `handle()` below never
|
|
12
|
+
* reads such a field either way — identity and the request id come only
|
|
13
|
+
* from this request itself (`getUserId`/`tenantFor`, and `cf-ray`/a fresh
|
|
14
|
+
* uuid), never from the body. The browser is not a trusted caller.
|
|
15
|
+
*/
|
|
16
|
+
const entrySchema = z.object({
|
|
17
|
+
kind: z.string().min(1),
|
|
18
|
+
message: z.string().min(1),
|
|
19
|
+
stack: z.string().nullish(),
|
|
20
|
+
release: z.string().nullish(),
|
|
21
|
+
});
|
|
22
|
+
const batchSchema = z.object({
|
|
23
|
+
site: z.string().regex(SITE_PATTERN),
|
|
24
|
+
errors: z.array(entrySchema).min(1),
|
|
25
|
+
});
|
|
26
|
+
const NO_CONTENT = () => new Response(null, { status: 204 });
|
|
27
|
+
/**
|
|
28
|
+
* The client's address as the tunnel presents it. `cf-connecting-ip` first
|
|
29
|
+
* because every request to this estate arrives through Cloudflare, then the
|
|
30
|
+
* first hop of `x-forwarded-for`; a header a client sets itself is no worse
|
|
31
|
+
* here than no key at all, since the bucket only ever costs that same client.
|
|
32
|
+
*/
|
|
33
|
+
function clientAddress(request) {
|
|
34
|
+
const cf = request.headers.get('cf-connecting-ip');
|
|
35
|
+
if (cf)
|
|
36
|
+
return cf.trim();
|
|
37
|
+
const xff = request.headers.get('x-forwarded-for');
|
|
38
|
+
return xff ? (xff.split(',')[0] ?? '').trim() || null : null;
|
|
39
|
+
}
|
|
40
|
+
/** `<16 hex>-<colo>`, as Cloudflare emits it — the same pattern `./index.ts` uses; kept here too, rather than imported, to avoid a circular import between the two files. */
|
|
41
|
+
const RAY_PATTERN = /^[0-9a-f]{16}-[A-Z0-9]{3,6}$/;
|
|
42
|
+
function requestIdOf(request) {
|
|
43
|
+
const ray = request.headers.get('cf-ray');
|
|
44
|
+
return ray && RAY_PATTERN.test(ray) ? ray : randomUUID();
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Same shape as the collector's own `originFor` (`../analytics/collector.js`,
|
|
48
|
+
* private there): duplicated rather than exported, per this task's scope —
|
|
49
|
+
* it is ~10 lines and the two callers now diverge (no `sites` map here).
|
|
50
|
+
*/
|
|
51
|
+
function originOf(request) {
|
|
52
|
+
const origin = request.headers.get('origin');
|
|
53
|
+
const host = request.headers.get('x-forwarded-host') ?? request.headers.get('host');
|
|
54
|
+
if (!origin)
|
|
55
|
+
return host ? { kind: 'same', origin: `https://${host}` } : null;
|
|
56
|
+
let originHost;
|
|
57
|
+
try {
|
|
58
|
+
originHost = new URL(origin).host;
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
return { kind: originHost === host ? 'same' : 'foreign', origin };
|
|
64
|
+
}
|
|
65
|
+
function corsFor(response, origin) {
|
|
66
|
+
response.headers.set('access-control-allow-origin', origin);
|
|
67
|
+
response.headers.set('vary', 'origin');
|
|
68
|
+
return response;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* An `Error` carrying exactly the client's own message and stack, so
|
|
72
|
+
* `captureError`'s existing redaction and fingerprinting run unchanged on
|
|
73
|
+
* it (errors/capture.ts is not this task's to edit). The class name is
|
|
74
|
+
* never read from this: `context.kind` below overrides it with the
|
|
75
|
+
* client's own `kind`.
|
|
76
|
+
*/
|
|
77
|
+
function toCapturable(entry) {
|
|
78
|
+
const err = new Error(entry.message);
|
|
79
|
+
err.stack = entry.stack ?? undefined;
|
|
80
|
+
return err;
|
|
81
|
+
}
|
|
82
|
+
export function createClientErrorCollector(options) {
|
|
83
|
+
const { reporting } = options;
|
|
84
|
+
const buckets = new Buckets(options.ratePerMinute ?? 60, () => new Date());
|
|
85
|
+
async function handle(request) {
|
|
86
|
+
if (request.method !== 'POST')
|
|
87
|
+
return NO_CONTENT();
|
|
88
|
+
const where = originOf(request);
|
|
89
|
+
if (!where)
|
|
90
|
+
return NO_CONTENT();
|
|
91
|
+
if (where.kind === 'foreign' && !(options.origins ?? []).includes(where.origin)) {
|
|
92
|
+
return NO_CONTENT();
|
|
93
|
+
}
|
|
94
|
+
const length = Number(request.headers.get('content-length') ?? '0');
|
|
95
|
+
if (length > ANALYTICS_LIMITS.bodyBytes)
|
|
96
|
+
return NO_CONTENT();
|
|
97
|
+
let text;
|
|
98
|
+
try {
|
|
99
|
+
text = await request.text();
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return NO_CONTENT();
|
|
103
|
+
}
|
|
104
|
+
if (text.length > ANALYTICS_LIMITS.bodyBytes)
|
|
105
|
+
return NO_CONTENT();
|
|
106
|
+
let json;
|
|
107
|
+
try {
|
|
108
|
+
json = JSON.parse(text);
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return NO_CONTENT();
|
|
112
|
+
}
|
|
113
|
+
const parsed = batchSchema.safeParse(json);
|
|
114
|
+
if (!parsed.success) {
|
|
115
|
+
reporting.log.info({ issues: parsed.error.issues.length }, 'reporting: client error batch refused');
|
|
116
|
+
return NO_CONTENT();
|
|
117
|
+
}
|
|
118
|
+
if (parsed.data.site !== reporting.site)
|
|
119
|
+
return NO_CONTENT();
|
|
120
|
+
// After parsing, so the cost of a refused body is not charged to the
|
|
121
|
+
// address, and before any capture, so the bucket bounds what is written
|
|
122
|
+
// rather than what was asked. Keyed on the address alone: a visitor id
|
|
123
|
+
// would be the client's own claim, and this is the route that exists
|
|
124
|
+
// because the client is not trusted.
|
|
125
|
+
const entries = parsed.data.errors.slice(0, BATCH_MAX);
|
|
126
|
+
if (!buckets.take(`ip:${clientAddress(request) ?? 'none'}`, entries.length)) {
|
|
127
|
+
return NO_CONTENT();
|
|
128
|
+
}
|
|
129
|
+
// Identity is the session's, same-origin only, and never the body's
|
|
130
|
+
// (the entry schema above has no tenantId/userId field to begin with).
|
|
131
|
+
let userId = null;
|
|
132
|
+
if (where.kind === 'same' && options.getUserId) {
|
|
133
|
+
try {
|
|
134
|
+
userId = await options.getUserId(request);
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
reporting.log.warn({ err: error instanceof Error ? error.message : String(error) }, 'reporting: client error getUserId failed; capturing without a tenant');
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
let tenantId = null;
|
|
141
|
+
if (userId && options.tenantFor) {
|
|
142
|
+
try {
|
|
143
|
+
tenantId = await options.tenantFor(userId);
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
tenantId = null;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
// This request's own id, cf-ray or a fresh uuid — never the body's;
|
|
150
|
+
// same rule `errorHandler` (./index.ts) follows for a server capture.
|
|
151
|
+
const requestId = requestIdOf(request);
|
|
152
|
+
for (const entry of entries) {
|
|
153
|
+
reporting.captureError(toCapturable(entry), {
|
|
154
|
+
kind: entry.kind,
|
|
155
|
+
runtime: 'browser',
|
|
156
|
+
release: entry.release ?? null,
|
|
157
|
+
tenantId,
|
|
158
|
+
requestId,
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
const response = NO_CONTENT();
|
|
162
|
+
return where.kind === 'foreign' ? corsFor(response, where.origin) : response;
|
|
163
|
+
}
|
|
164
|
+
return { handle };
|
|
165
|
+
}
|