@wtfalch/reporting 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -0
- package/dist/alerts.d.ts +17 -0
- package/dist/alerts.js +53 -0
- package/dist/bin/copy.d.ts +25 -0
- package/dist/bin/copy.js +57 -0
- package/dist/bin/migrations.d.ts +2 -0
- package/dist/bin/migrations.js +22 -0
- package/dist/fake/index.d.ts +16 -0
- package/dist/fake/index.js +105 -0
- package/dist/housekeeping/index.d.ts +63 -0
- package/dist/housekeeping/index.js +198 -0
- package/dist/housekeeping/tasks.d.ts +20 -0
- package/dist/housekeeping/tasks.js +82 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +82 -0
- package/dist/migrations/0001_reporting.sql +154 -0
- package/dist/next/index.d.ts +40 -0
- package/dist/next/index.js +67 -0
- package/dist/reader.d.ts +8 -0
- package/dist/reader.js +40 -0
- package/dist/schema.d.ts +78 -0
- package/dist/schema.js +80 -0
- package/dist/settings.d.ts +16 -0
- package/dist/settings.js +71 -0
- package/dist/tables.d.ts +1014 -0
- package/dist/tables.js +55 -0
- package/dist/types.d.ts +106 -0
- package/dist/types.js +1 -0
- package/dist/writer.d.ts +39 -0
- package/dist/writer.js +166 -0
- package/package.json +71 -0
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { eq, sql } from 'drizzle-orm';
|
|
2
|
+
import { KIND_PATTERN } from '../schema.js';
|
|
3
|
+
import { reportingTasks } from '../tables.js';
|
|
4
|
+
import { describe } from '../writer.js';
|
|
5
|
+
export function createHousekeeping(options) {
|
|
6
|
+
const { reporting } = options;
|
|
7
|
+
const db = options.db ?? reporting.db;
|
|
8
|
+
const now = options.now ?? (() => new Date());
|
|
9
|
+
const debounceMs = options.debounceMs ?? 60_000;
|
|
10
|
+
const concurrency = Math.max(1, options.concurrency ?? 2);
|
|
11
|
+
const registry = new Map();
|
|
12
|
+
let lastTick = 0;
|
|
13
|
+
let ticking = null;
|
|
14
|
+
let rowsEnsured = false;
|
|
15
|
+
function register(task) {
|
|
16
|
+
if (!KIND_PATTERN.test(task.name)) {
|
|
17
|
+
throw new Error(`housekeeping: task name "${task.name}" is not namespace.name`);
|
|
18
|
+
}
|
|
19
|
+
registry.set(task.name, task);
|
|
20
|
+
rowsEnsured = false;
|
|
21
|
+
}
|
|
22
|
+
async function ensureRows() {
|
|
23
|
+
if (rowsEnsured || registry.size === 0)
|
|
24
|
+
return;
|
|
25
|
+
await db
|
|
26
|
+
.insert(reportingTasks)
|
|
27
|
+
.values([...registry.keys()].map((task) => ({ task })))
|
|
28
|
+
.onConflictDoNothing({ target: reportingTasks.task });
|
|
29
|
+
rowsEnsured = true;
|
|
30
|
+
}
|
|
31
|
+
/** The one transaction that decides. Returns the token when this process now holds the task. */
|
|
32
|
+
async function claim(task) {
|
|
33
|
+
return db.transaction(async (tx) => {
|
|
34
|
+
const held = await tx
|
|
35
|
+
.select({ task: reportingTasks.task })
|
|
36
|
+
.from(reportingTasks)
|
|
37
|
+
.where(eq(reportingTasks.task, task.name))
|
|
38
|
+
.for('update', { skipLocked: true });
|
|
39
|
+
if (held.length === 0)
|
|
40
|
+
return null;
|
|
41
|
+
const claimed = await tx
|
|
42
|
+
.update(reportingTasks)
|
|
43
|
+
.set({
|
|
44
|
+
claimToken: sql `gen_random_uuid()`,
|
|
45
|
+
lastStartedAt: sql `now()`,
|
|
46
|
+
leaseExpiresAt: sql `now() + make_interval(secs => ${task.lease / 1000}::double precision)`,
|
|
47
|
+
lastOutcome: null,
|
|
48
|
+
lastError: null,
|
|
49
|
+
})
|
|
50
|
+
.where(sql `${reportingTasks.task} = ${task.name}
|
|
51
|
+
and ${reportingTasks.nextDueAt} <= now()
|
|
52
|
+
and (${reportingTasks.leaseExpiresAt} is null or ${reportingTasks.leaseExpiresAt} < now())`)
|
|
53
|
+
.returning({ token: reportingTasks.claimToken });
|
|
54
|
+
return claimed[0]?.token ?? null;
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
async function renew(task, token) {
|
|
58
|
+
const rows = await db
|
|
59
|
+
.update(reportingTasks)
|
|
60
|
+
.set({
|
|
61
|
+
leaseExpiresAt: sql `now() + make_interval(secs => ${task.lease / 1000}::double precision)`,
|
|
62
|
+
})
|
|
63
|
+
.where(sql `${reportingTasks.task} = ${task.name} and ${reportingTasks.claimToken} = ${token}`)
|
|
64
|
+
.returning({ task: reportingTasks.task });
|
|
65
|
+
return rows.length === 1;
|
|
66
|
+
}
|
|
67
|
+
async function complete(task, token, outcome, error) {
|
|
68
|
+
const delay = (outcome === 'error' ? task.retry : task.every) / 1000;
|
|
69
|
+
await db
|
|
70
|
+
.update(reportingTasks)
|
|
71
|
+
.set({
|
|
72
|
+
lastFinishedAt: sql `now()`,
|
|
73
|
+
leaseExpiresAt: null,
|
|
74
|
+
nextDueAt: sql `now() + make_interval(secs => ${delay}::double precision)`,
|
|
75
|
+
lastOutcome: outcome,
|
|
76
|
+
lastError: error ? error.slice(0, 512) : null,
|
|
77
|
+
})
|
|
78
|
+
.where(sql `${reportingTasks.task} = ${task.name} and ${reportingTasks.claimToken} = ${token}`);
|
|
79
|
+
}
|
|
80
|
+
async function runClaimed(task, token, settings) {
|
|
81
|
+
const started = now();
|
|
82
|
+
const deadline = new Date(started.getTime() + task.lease);
|
|
83
|
+
let timer;
|
|
84
|
+
const timeout = new Promise((_, reject) => {
|
|
85
|
+
timer = setTimeout(() => reject(new Error(`timed out after ${task.lease} ms`)), task.lease);
|
|
86
|
+
timer.unref?.();
|
|
87
|
+
});
|
|
88
|
+
try {
|
|
89
|
+
await Promise.race([
|
|
90
|
+
task.run({
|
|
91
|
+
reporting,
|
|
92
|
+
db,
|
|
93
|
+
settings,
|
|
94
|
+
now,
|
|
95
|
+
renew: () => renew(task, token),
|
|
96
|
+
deadline,
|
|
97
|
+
}),
|
|
98
|
+
timeout,
|
|
99
|
+
]);
|
|
100
|
+
await complete(task, token, 'ok', null);
|
|
101
|
+
reporting.event({
|
|
102
|
+
kind: 'housekeeping.task_ran',
|
|
103
|
+
message: `${task.name} ran in ${now().getTime() - started.getTime()} ms`,
|
|
104
|
+
data: { task: task.name, ms: now().getTime() - started.getTime() },
|
|
105
|
+
});
|
|
106
|
+
return 'ok';
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
const text = describe(error);
|
|
110
|
+
await complete(task, token, 'error', text).catch(() => undefined);
|
|
111
|
+
reporting.event({
|
|
112
|
+
kind: 'housekeeping.task_failed',
|
|
113
|
+
level: 'error',
|
|
114
|
+
message: `${task.name} failed: ${text}`.slice(0, 512),
|
|
115
|
+
data: { task: task.name, error: text.slice(0, 200) },
|
|
116
|
+
});
|
|
117
|
+
return 'error';
|
|
118
|
+
}
|
|
119
|
+
finally {
|
|
120
|
+
if (timer)
|
|
121
|
+
clearTimeout(timer);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
async function runOne(task, settings) {
|
|
125
|
+
const token = await claim(task);
|
|
126
|
+
if (!token)
|
|
127
|
+
return 'skipped';
|
|
128
|
+
return runClaimed(task, token, settings);
|
|
129
|
+
}
|
|
130
|
+
async function tickNow() {
|
|
131
|
+
await ensureRows();
|
|
132
|
+
const settings = await reporting.settings.get();
|
|
133
|
+
const queue = [...registry.values()];
|
|
134
|
+
const counts = { claimed: 0, skipped: 0, ran: 0, failed: 0 };
|
|
135
|
+
const workers = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
|
|
136
|
+
for (let task = queue.shift(); task; task = queue.shift()) {
|
|
137
|
+
try {
|
|
138
|
+
const outcome = await runOne(task, settings);
|
|
139
|
+
if (outcome === 'skipped')
|
|
140
|
+
counts.skipped += 1;
|
|
141
|
+
else {
|
|
142
|
+
counts.claimed += 1;
|
|
143
|
+
if (outcome === 'ok')
|
|
144
|
+
counts.ran += 1;
|
|
145
|
+
else
|
|
146
|
+
counts.failed += 1;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
// The claim itself failed (the database was away). Nothing to
|
|
151
|
+
// complete; the row is untouched and the next tick tries again.
|
|
152
|
+
counts.skipped += 1;
|
|
153
|
+
reporting.log.error({ err: describe(error), task: task.name }, 'housekeeping: claim failed');
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
});
|
|
157
|
+
await Promise.all(workers);
|
|
158
|
+
if (counts.claimed > 0) {
|
|
159
|
+
reporting.event({
|
|
160
|
+
kind: 'housekeeping.tick',
|
|
161
|
+
message: `tick: ${counts.ran} ran, ${counts.failed} failed, ${counts.skipped} skipped`,
|
|
162
|
+
data: counts,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
await reporting.flush({ deadlineMs: 5000 });
|
|
166
|
+
}
|
|
167
|
+
return {
|
|
168
|
+
register,
|
|
169
|
+
tasks: () => [...registry.values()],
|
|
170
|
+
async tick() {
|
|
171
|
+
const t = Date.now();
|
|
172
|
+
if (ticking)
|
|
173
|
+
return ticking;
|
|
174
|
+
if (t - lastTick < debounceMs)
|
|
175
|
+
return;
|
|
176
|
+
lastTick = t;
|
|
177
|
+
ticking = tickNow()
|
|
178
|
+
.catch((error) => {
|
|
179
|
+
reporting.log.error({ err: describe(error) }, 'housekeeping: tick failed');
|
|
180
|
+
})
|
|
181
|
+
.finally(() => {
|
|
182
|
+
ticking = null;
|
|
183
|
+
});
|
|
184
|
+
return ticking;
|
|
185
|
+
},
|
|
186
|
+
async runNow(name) {
|
|
187
|
+
const task = registry.get(name);
|
|
188
|
+
if (!task)
|
|
189
|
+
throw new Error(`housekeeping: no task "${name}"`);
|
|
190
|
+
await ensureRows();
|
|
191
|
+
const settings = await reporting.settings.get();
|
|
192
|
+
const outcome = await runOne(task, settings);
|
|
193
|
+
await reporting.flush({ deadlineMs: 5000 });
|
|
194
|
+
return outcome === 'ok' ? 'ran' : outcome === 'skipped' ? 'skipped' : 'failed';
|
|
195
|
+
},
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
export { pruneEvents, retentionLag } from './tasks.js';
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Db } from '../types.js';
|
|
2
|
+
import type { Task } from './index.js';
|
|
3
|
+
/** `db.execute` returns an array on postgres-js and `{ rows }` on PGlite; this reads either. */
|
|
4
|
+
export declare function rowsOf(result: unknown): Record<string, unknown>[];
|
|
5
|
+
/**
|
|
6
|
+
* Deletes rows older than the operator's window, up to twenty batches of
|
|
7
|
+
* 5,000 per run, each in its own statement with a bounded timeout, renewing
|
|
8
|
+
* the lease between batches. The function clamps the window itself, so a
|
|
9
|
+
* setting of one day still keeps a week.
|
|
10
|
+
*/
|
|
11
|
+
export declare const pruneEvents: Task;
|
|
12
|
+
export declare function pruneBatch(db: Db, days: number, batch: number): Promise<number>;
|
|
13
|
+
/**
|
|
14
|
+
* The watchdog on the promise prune makes: if the oldest row is older than
|
|
15
|
+
* one and a half windows, prune has not been running and somebody should
|
|
16
|
+
* know. It shares the tick's failure domain and cannot report a tick that
|
|
17
|
+
* is entirely down; it catches the case where the tick runs and prune
|
|
18
|
+
* keeps failing or never gets claimed.
|
|
19
|
+
*/
|
|
20
|
+
export declare const retentionLag: Task;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { sql } from 'drizzle-orm';
|
|
2
|
+
import { reportingEvents } from '../tables.js';
|
|
3
|
+
/** `db.execute` returns an array on postgres-js and `{ rows }` on PGlite; this reads either. */
|
|
4
|
+
export function rowsOf(result) {
|
|
5
|
+
if (Array.isArray(result))
|
|
6
|
+
return result;
|
|
7
|
+
const r = result;
|
|
8
|
+
return Array.isArray(r?.rows) ? r.rows : [];
|
|
9
|
+
}
|
|
10
|
+
const HOUR = 60 * 60 * 1000;
|
|
11
|
+
const DAY = 24 * HOUR;
|
|
12
|
+
/**
|
|
13
|
+
* Deletes rows older than the operator's window, up to twenty batches of
|
|
14
|
+
* 5,000 per run, each in its own statement with a bounded timeout, renewing
|
|
15
|
+
* the lease between batches. The function clamps the window itself, so a
|
|
16
|
+
* setting of one day still keeps a week.
|
|
17
|
+
*/
|
|
18
|
+
export const pruneEvents = {
|
|
19
|
+
name: 'reporting.prune_events',
|
|
20
|
+
every: HOUR,
|
|
21
|
+
lease: 10 * 60 * 1000,
|
|
22
|
+
retry: 15 * 60 * 1000,
|
|
23
|
+
async run(ctx) {
|
|
24
|
+
const days = ctx.settings['events.retention_days'];
|
|
25
|
+
let total = 0;
|
|
26
|
+
for (let i = 0; i < 20; i += 1) {
|
|
27
|
+
if (ctx.now().getTime() > ctx.deadline.getTime() - 30_000)
|
|
28
|
+
break;
|
|
29
|
+
const n = await pruneBatch(ctx.db, days, 5000);
|
|
30
|
+
total += n;
|
|
31
|
+
if (n < 5000)
|
|
32
|
+
break;
|
|
33
|
+
if (!(await ctx.renew()))
|
|
34
|
+
break;
|
|
35
|
+
}
|
|
36
|
+
if (total > 0) {
|
|
37
|
+
ctx.reporting.event({
|
|
38
|
+
kind: 'reporting.pruned',
|
|
39
|
+
message: `pruned ${total} event(s) older than ${days} days`,
|
|
40
|
+
data: { rows: total, days },
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
export async function pruneBatch(db, days, batch) {
|
|
46
|
+
return db.transaction(async (tx) => {
|
|
47
|
+
await tx.execute(sql `set local statement_timeout = '30s'`);
|
|
48
|
+
const result = await tx.execute(sql `select reporting_prune_events(make_interval(days => ${days}::int), ${batch}::int) as n`);
|
|
49
|
+
const [row] = rowsOf(result);
|
|
50
|
+
return Number(row?.n ?? 0);
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* The watchdog on the promise prune makes: if the oldest row is older than
|
|
55
|
+
* one and a half windows, prune has not been running and somebody should
|
|
56
|
+
* know. It shares the tick's failure domain and cannot report a tick that
|
|
57
|
+
* is entirely down; it catches the case where the tick runs and prune
|
|
58
|
+
* keeps failing or never gets claimed.
|
|
59
|
+
*/
|
|
60
|
+
export const retentionLag = {
|
|
61
|
+
name: 'reporting.retention_lag',
|
|
62
|
+
every: DAY,
|
|
63
|
+
lease: 60 * 1000,
|
|
64
|
+
retry: HOUR,
|
|
65
|
+
async run(ctx) {
|
|
66
|
+
const days = ctx.settings['events.retention_days'];
|
|
67
|
+
const [row] = await ctx.db
|
|
68
|
+
.select({ oldest: sql `min(${reportingEvents.occurredAt})` })
|
|
69
|
+
.from(reportingEvents);
|
|
70
|
+
const oldest = row?.oldest ? new Date(row.oldest) : null;
|
|
71
|
+
if (!oldest)
|
|
72
|
+
return;
|
|
73
|
+
const lagDays = (ctx.now().getTime() - oldest.getTime()) / DAY;
|
|
74
|
+
if (lagDays > days * 1.5) {
|
|
75
|
+
ctx.reporting.alert({
|
|
76
|
+
check: 'retention_lag',
|
|
77
|
+
message: `the oldest event is ${Math.floor(lagDays)} days old against a ${days}-day window; prune is not keeping up`,
|
|
78
|
+
detail: { oldestDays: Math.floor(lagDays), windowDays: days },
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
};
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Reporting, ReportingOptions } from './types.js';
|
|
2
|
+
export * from './schema.js';
|
|
3
|
+
export * from './tables.js';
|
|
4
|
+
export * from './types.js';
|
|
5
|
+
export { alertKind, projectDetail } from './alerts.js';
|
|
6
|
+
export { DEFAULT_SETTINGS, RETENTION_BOUNDS, SETTING_KEYS } from './settings.js';
|
|
7
|
+
/**
|
|
8
|
+
* The host's one instance, built once over its drizzle handle and its
|
|
9
|
+
* logger. Framework-neutral: `defer` is how the host says "after this unit of
|
|
10
|
+
* work" (the `./next` entry passes `after()`), and nothing here reads a
|
|
11
|
+
* request, a session or an environment variable other than NODE_ENV.
|
|
12
|
+
*/
|
|
13
|
+
export declare function createReporting(options: ReportingOptions): Reporting;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { alertRow } from './alerts.js';
|
|
2
|
+
import { eventsPage } from './reader.js';
|
|
3
|
+
import { siteSchema } from './schema.js';
|
|
4
|
+
import { getSettings, setSettings } from './settings.js';
|
|
5
|
+
import { tables } from './tables.js';
|
|
6
|
+
import { Writer, describe } from './writer.js';
|
|
7
|
+
export * from './schema.js';
|
|
8
|
+
export * from './tables.js';
|
|
9
|
+
export * from './types.js';
|
|
10
|
+
export { alertKind, projectDetail } from './alerts.js';
|
|
11
|
+
export { DEFAULT_SETTINGS, RETENTION_BOUNDS, SETTING_KEYS } from './settings.js';
|
|
12
|
+
function modeFromEnv() {
|
|
13
|
+
const env = process.env.NODE_ENV;
|
|
14
|
+
return env === 'development' || env === 'test' ? env : 'production';
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The host's one instance, built once over its drizzle handle and its
|
|
18
|
+
* logger. Framework-neutral: `defer` is how the host says "after this unit of
|
|
19
|
+
* work" (the `./next` entry passes `after()`), and nothing here reads a
|
|
20
|
+
* request, a session or an environment variable other than NODE_ENV.
|
|
21
|
+
*/
|
|
22
|
+
export function createReporting(options) {
|
|
23
|
+
const site = siteSchema.parse(options.site);
|
|
24
|
+
const now = options.now ?? (() => new Date());
|
|
25
|
+
const mode = options.mode ?? modeFromEnv();
|
|
26
|
+
const writer = new Writer({
|
|
27
|
+
db: options.db,
|
|
28
|
+
log: options.log,
|
|
29
|
+
site,
|
|
30
|
+
mode,
|
|
31
|
+
now,
|
|
32
|
+
defer: options.defer ??
|
|
33
|
+
((fn) => {
|
|
34
|
+
setTimeout(() => void fn(), 0);
|
|
35
|
+
}),
|
|
36
|
+
queueLimit: options.queueLimit ?? 1000,
|
|
37
|
+
batchSize: options.batchSize ?? 500,
|
|
38
|
+
flushEveryMs: options.flushEveryMs ?? 1000,
|
|
39
|
+
});
|
|
40
|
+
const reporting = {
|
|
41
|
+
site,
|
|
42
|
+
db: options.db,
|
|
43
|
+
log: options.log,
|
|
44
|
+
tables,
|
|
45
|
+
event: (input) => writer.event(input),
|
|
46
|
+
alert(finding) {
|
|
47
|
+
writer.event(alertRow(finding));
|
|
48
|
+
if (!options.onAlert)
|
|
49
|
+
return;
|
|
50
|
+
try {
|
|
51
|
+
options.onAlert(finding);
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
options.log.error({ err: describe(error) }, 'reporting: onAlert threw');
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
flush: (opts) => writer.flush(opts),
|
|
58
|
+
events: { page: (opts) => eventsPage(options.db, opts) },
|
|
59
|
+
settings: {
|
|
60
|
+
get: () => getSettings(options.db),
|
|
61
|
+
async set(patch, by) {
|
|
62
|
+
const change = await setSettings(options.db, patch, by);
|
|
63
|
+
if (change.changed.length > 0) {
|
|
64
|
+
writer.event({
|
|
65
|
+
kind: 'reporting.settings_changed',
|
|
66
|
+
message: `settings changed: ${change.changed.join(', ')}`,
|
|
67
|
+
actor: by,
|
|
68
|
+
data: Object.fromEntries(change.changed.flatMap((k) => [
|
|
69
|
+
[`${k}.before`, change.before[k]],
|
|
70
|
+
[`${k}.after`, change.after[k]],
|
|
71
|
+
])),
|
|
72
|
+
});
|
|
73
|
+
if (options.audit)
|
|
74
|
+
await options.audit(change);
|
|
75
|
+
}
|
|
76
|
+
return change;
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
stats: () => writer.stats(),
|
|
80
|
+
};
|
|
81
|
+
return reporting;
|
|
82
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
-- @wtfalch/reporting: the operational event log, the housekeeping task rows
|
|
2
|
+
-- and the settings the housekeeping reads. Mirrors src/tables.ts.
|
|
3
|
+
--
|
|
4
|
+
-- Every statement is idempotent. Applied by the host's own migrate script
|
|
5
|
+
-- after the host copies this file into its drizzle/ directory as the next
|
|
6
|
+
-- number (reporting-migrations); never edited there.
|
|
7
|
+
--
|
|
8
|
+
-- Estate-shaped: the DO block at the end assumes the runtime-role convention
|
|
9
|
+
-- (a role named <database>_rt that owns nothing and serves the app). A host
|
|
10
|
+
-- without that role gets the tables and the function and no revokes take
|
|
11
|
+
-- effect, which the package README says plainly.
|
|
12
|
+
|
|
13
|
+
create table if not exists reporting_events (
|
|
14
|
+
id bigint generated always as identity primary key,
|
|
15
|
+
occurred_at timestamptz not null default now(),
|
|
16
|
+
level text not null,
|
|
17
|
+
kind text not null,
|
|
18
|
+
kind_ns text generated always as (split_part(kind, '.', 1)) stored,
|
|
19
|
+
site text not null,
|
|
20
|
+
tenant_id uuid,
|
|
21
|
+
actor_class text,
|
|
22
|
+
-- An issuer or credential id. No foreign key and no cascade on purpose:
|
|
23
|
+
-- rows leave this table by age through reporting_prune_events, never by
|
|
24
|
+
-- erasure. That is a deliberate retention exception for a linkable
|
|
25
|
+
-- identifier, written down in the host's tenancy notes, not a claim that
|
|
26
|
+
-- the id is anonymous.
|
|
27
|
+
actor_id text,
|
|
28
|
+
request_id text,
|
|
29
|
+
target_type text,
|
|
30
|
+
target_id text,
|
|
31
|
+
message text not null,
|
|
32
|
+
data jsonb not null default '{}'::jsonb,
|
|
33
|
+
schema_version smallint not null default 1,
|
|
34
|
+
constraint reporting_events_level_check check (level in ('info', 'warn', 'error', 'alert')),
|
|
35
|
+
constraint reporting_events_kind_check check (kind ~ '^[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*$'),
|
|
36
|
+
constraint reporting_events_site_check check (site ~ '^[a-z][a-z0-9-]{0,63}$'),
|
|
37
|
+
constraint reporting_events_actor_class_check check (actor_class is null or actor_class in ('human', 'api_key', 'agent', 'service')),
|
|
38
|
+
constraint reporting_events_actor_pair_check check ((actor_id is null) = (actor_class is null)),
|
|
39
|
+
constraint reporting_events_actor_id_check check (actor_id is null or length(actor_id) between 1 and 256),
|
|
40
|
+
constraint reporting_events_request_id_check check (request_id is null or length(request_id) between 1 and 128),
|
|
41
|
+
constraint reporting_events_target_type_check check (target_type is null or length(target_type) between 1 and 64),
|
|
42
|
+
constraint reporting_events_target_id_check check (target_id is null or length(target_id) between 1 and 256),
|
|
43
|
+
constraint reporting_events_target_pair_check check ((target_id is null) = (target_type is null)),
|
|
44
|
+
constraint reporting_events_message_check check (length(message) between 1 and 512),
|
|
45
|
+
constraint reporting_events_data_object_check check (jsonb_typeof(data) = 'object'),
|
|
46
|
+
constraint reporting_events_data_size_check check (octet_length(data::text) <= 16384),
|
|
47
|
+
-- Flat: no value is an object or an array. A nested payload is where a
|
|
48
|
+
-- provider response or a request body sneaks in whole. `strict`, because
|
|
49
|
+
-- lax mode unwraps an array before the filter sees it and would never
|
|
50
|
+
-- call one an array.
|
|
51
|
+
constraint reporting_events_data_flat_check check (not jsonb_path_exists(data, 'strict $.* ? (@.type() == "object" || @.type() == "array")')),
|
|
52
|
+
-- The keys that carry a person or a secret by name. A guardrail, not a
|
|
53
|
+
-- proof: src/schema.ts refuses the same list before a row is queued, and
|
|
54
|
+
-- schema.test.ts pins the two lists equal.
|
|
55
|
+
constraint reporting_events_data_keys_check check (not (data ?| array['email', 'name', 'display_name', 'displayName', 'password', 'secret', 'token', 'authorization']))
|
|
56
|
+
);
|
|
57
|
+
|
|
58
|
+
comment on column reporting_events.actor_id is
|
|
59
|
+
'An issuer or credential id. No foreign key and no cascade on purpose: rows leave by age, not by erasure.';
|
|
60
|
+
|
|
61
|
+
-- Newest first is every reader's order; the partial indexes serve the
|
|
62
|
+
-- filters the operator page offers without indexing what nobody asks for.
|
|
63
|
+
create index if not exists reporting_events_time_idx
|
|
64
|
+
on reporting_events (occurred_at desc, id desc);
|
|
65
|
+
create index if not exists reporting_events_tenant_time_idx
|
|
66
|
+
on reporting_events (tenant_id, occurred_at desc, id desc) where tenant_id is not null;
|
|
67
|
+
create index if not exists reporting_events_ns_time_idx
|
|
68
|
+
on reporting_events (kind_ns, occurred_at desc, id desc);
|
|
69
|
+
create index if not exists reporting_events_problem_time_idx
|
|
70
|
+
on reporting_events (occurred_at desc, id desc) where level in ('error', 'alert');
|
|
71
|
+
create index if not exists reporting_events_request_idx
|
|
72
|
+
on reporting_events (request_id, occurred_at desc) where request_id is not null;
|
|
73
|
+
-- Hourly bulk deletes are the pattern the default scale factor handles badly.
|
|
74
|
+
alter table reporting_events set (autovacuum_vacuum_scale_factor = 0.02);
|
|
75
|
+
|
|
76
|
+
-- One row per registered housekeeping task: when it is next due, who holds
|
|
77
|
+
-- it now (a claim token and a lease), and how its last run ended. Claimed
|
|
78
|
+
-- with FOR UPDATE SKIP LOCKED in a short transaction; never with an advisory
|
|
79
|
+
-- lock, which is session state on whichever pooled connection served it.
|
|
80
|
+
create table if not exists reporting_tasks (
|
|
81
|
+
task text primary key,
|
|
82
|
+
next_due_at timestamptz not null default now(),
|
|
83
|
+
last_started_at timestamptz,
|
|
84
|
+
last_finished_at timestamptz,
|
|
85
|
+
lease_expires_at timestamptz,
|
|
86
|
+
claim_token uuid,
|
|
87
|
+
last_outcome text,
|
|
88
|
+
last_error text,
|
|
89
|
+
-- Free for a task's own bookkeeping: the analytics rollup keeps its
|
|
90
|
+
-- completed-day watermark here.
|
|
91
|
+
watermark text,
|
|
92
|
+
constraint reporting_tasks_task_check check (task ~ '^[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*$'),
|
|
93
|
+
constraint reporting_tasks_outcome_check check (last_outcome is null or last_outcome in ('ok', 'error', 'skipped')),
|
|
94
|
+
constraint reporting_tasks_error_check check (last_error is null or length(last_error) <= 512)
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
-- Operator-edited settings, one row per key. `events.*` is this release's;
|
|
98
|
+
-- `analytics.*` is reserved for the next.
|
|
99
|
+
create table if not exists reporting_settings (
|
|
100
|
+
key text primary key,
|
|
101
|
+
value jsonb not null,
|
|
102
|
+
updated_at timestamptz not null default now(),
|
|
103
|
+
updated_by text,
|
|
104
|
+
constraint reporting_settings_key_check check (key ~ '^(events|analytics)\.[a-z][a-z0-9_]*$')
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
-- The one way rows leave reporting_events. The window is clamped to between
|
|
108
|
+
-- seven and four hundred days inside the function, so a caller (the app,
|
|
109
|
+
-- through a setting an operator edits) can shorten history to a week and no
|
|
110
|
+
-- further, and can never empty the table. Eligibility is by occurred_at,
|
|
111
|
+
-- never by an id boundary: ids do not order occurrence when two containers
|
|
112
|
+
-- buffer and flush independently, and a delayed insert must not make newer
|
|
113
|
+
-- rows eligible. Batched so no statement holds a lock for long.
|
|
114
|
+
create or replace function reporting_prune_events(retention interval, batch integer)
|
|
115
|
+
returns integer
|
|
116
|
+
language plpgsql
|
|
117
|
+
security definer
|
|
118
|
+
set search_path = pg_catalog, public
|
|
119
|
+
as $$
|
|
120
|
+
declare
|
|
121
|
+
w interval := greatest(interval '7 days', least(coalesce(retention, interval '30 days'), interval '400 days'));
|
|
122
|
+
b integer := greatest(1, least(coalesce(batch, 5000), 5000));
|
|
123
|
+
n integer;
|
|
124
|
+
begin
|
|
125
|
+
with victims as (
|
|
126
|
+
select id from reporting_events
|
|
127
|
+
where occurred_at < now() - w
|
|
128
|
+
order by occurred_at, id
|
|
129
|
+
limit b
|
|
130
|
+
)
|
|
131
|
+
delete from reporting_events e using victims v where e.id = v.id;
|
|
132
|
+
get diagnostics n = row_count;
|
|
133
|
+
return n;
|
|
134
|
+
end
|
|
135
|
+
$$;
|
|
136
|
+
revoke all on function reporting_prune_events(interval, integer) from public;
|
|
137
|
+
|
|
138
|
+
-- The runtime role keeps SELECT and INSERT on all three tables, UPDATE on
|
|
139
|
+
-- tasks and settings because the claim protocol and the settings page write
|
|
140
|
+
-- them, and nothing that deletes. The host's default privileges granted the
|
|
141
|
+
-- rest at create time; the revokes here win because they run later in the
|
|
142
|
+
-- same file.
|
|
143
|
+
do $$
|
|
144
|
+
declare
|
|
145
|
+
rt text := current_database() || '_rt';
|
|
146
|
+
begin
|
|
147
|
+
if exists (select 1 from pg_roles where rolname = rt) then
|
|
148
|
+
execute format('revoke update, delete, truncate on reporting_events from %I', rt);
|
|
149
|
+
execute format('revoke delete, truncate on reporting_tasks from %I', rt);
|
|
150
|
+
execute format('revoke delete, truncate on reporting_settings from %I', rt);
|
|
151
|
+
execute format('grant execute on function reporting_prune_events(interval, integer) to %I', rt);
|
|
152
|
+
end if;
|
|
153
|
+
end
|
|
154
|
+
$$;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { Reporting } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The Next bindings: everything in the package that knows what a request is.
|
|
4
|
+
* Peer on `next`; a host without Next never imports this entry.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* The health route, and the heartbeat behind it. `connection()` first,
|
|
8
|
+
* because under `cacheComponents` a route with no dynamic read is
|
|
9
|
+
* prerendered, and the tick would otherwise run at build time with no
|
|
10
|
+
* database. The tick runs after the response and never throws.
|
|
11
|
+
*/
|
|
12
|
+
export declare function healthHandler(housekeeping: {
|
|
13
|
+
tick(): Promise<void>;
|
|
14
|
+
}): () => Promise<Response>;
|
|
15
|
+
export interface RequestContext {
|
|
16
|
+
/** Cloudflare's `cf-ray` when the request came through the tunnel, else a uuid. */
|
|
17
|
+
readonly id: string;
|
|
18
|
+
readonly ip: string | null;
|
|
19
|
+
readonly userAgent: string | null;
|
|
20
|
+
/** Two letters from `cf-ipcountry`, or null. */
|
|
21
|
+
readonly country: string | null;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Read once where a request enters (a Server Action, a route handler, a
|
|
25
|
+
* page's data component) and handed down. Outside a request scope it returns
|
|
26
|
+
* a fresh id and nulls, but only after `unstable_rethrow`: under
|
|
27
|
+
* `cacheComponents`, `headers()` in a prerender throws the signal Next uses
|
|
28
|
+
* to bail out, and swallowing that would poison the render.
|
|
29
|
+
*/
|
|
30
|
+
export declare function requestContext(): Promise<RequestContext>;
|
|
31
|
+
/** `after()` inside a request; a timer outside one (boot, a script), so a row queued there still flushes. */
|
|
32
|
+
export declare const deferWithAfter: (fn: () => Promise<void>) => void;
|
|
33
|
+
/**
|
|
34
|
+
* One best-effort drain on SIGTERM, bounded so a rolling deploy's drain
|
|
35
|
+
* window is respected. Not a durable queue: a forced kill or a slow database
|
|
36
|
+
* loses what is queued, and the logger already has every row.
|
|
37
|
+
*/
|
|
38
|
+
export declare function registerShutdownFlush(reporting: Reporting, opts?: {
|
|
39
|
+
deadlineMs?: number;
|
|
40
|
+
}): void;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { headers } from 'next/headers';
|
|
3
|
+
import { unstable_rethrow } from 'next/navigation';
|
|
4
|
+
import { after, connection } from 'next/server';
|
|
5
|
+
/**
|
|
6
|
+
* The Next bindings: everything in the package that knows what a request is.
|
|
7
|
+
* Peer on `next`; a host without Next never imports this entry.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* The health route, and the heartbeat behind it. `connection()` first,
|
|
11
|
+
* because under `cacheComponents` a route with no dynamic read is
|
|
12
|
+
* prerendered, and the tick would otherwise run at build time with no
|
|
13
|
+
* database. The tick runs after the response and never throws.
|
|
14
|
+
*/
|
|
15
|
+
export function healthHandler(housekeeping) {
|
|
16
|
+
return async () => {
|
|
17
|
+
await connection();
|
|
18
|
+
after(() => housekeeping.tick());
|
|
19
|
+
return Response.json({ ok: true, ts: new Date().toISOString() });
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Read once where a request enters (a Server Action, a route handler, a
|
|
24
|
+
* page's data component) and handed down. Outside a request scope it returns
|
|
25
|
+
* a fresh id and nulls, but only after `unstable_rethrow`: under
|
|
26
|
+
* `cacheComponents`, `headers()` in a prerender throws the signal Next uses
|
|
27
|
+
* to bail out, and swallowing that would poison the render.
|
|
28
|
+
*/
|
|
29
|
+
export async function requestContext() {
|
|
30
|
+
try {
|
|
31
|
+
const h = await headers();
|
|
32
|
+
const country = h.get('cf-ipcountry');
|
|
33
|
+
return {
|
|
34
|
+
id: (h.get('cf-ray') ?? randomUUID()).slice(0, 128),
|
|
35
|
+
ip: h.get('cf-connecting-ip'),
|
|
36
|
+
userAgent: h.get('user-agent'),
|
|
37
|
+
country: country && /^[A-Z]{2}$/.test(country) ? country : null,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
unstable_rethrow(error);
|
|
42
|
+
return { id: randomUUID(), ip: null, userAgent: null, country: null };
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** `after()` inside a request; a timer outside one (boot, a script), so a row queued there still flushes. */
|
|
46
|
+
export const deferWithAfter = (fn) => {
|
|
47
|
+
try {
|
|
48
|
+
after(fn);
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
setTimeout(() => void fn(), 0);
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
let shutdownRegistered = false;
|
|
55
|
+
/**
|
|
56
|
+
* One best-effort drain on SIGTERM, bounded so a rolling deploy's drain
|
|
57
|
+
* window is respected. Not a durable queue: a forced kill or a slow database
|
|
58
|
+
* loses what is queued, and the logger already has every row.
|
|
59
|
+
*/
|
|
60
|
+
export function registerShutdownFlush(reporting, opts = {}) {
|
|
61
|
+
if (shutdownRegistered)
|
|
62
|
+
return;
|
|
63
|
+
shutdownRegistered = true;
|
|
64
|
+
process.once('SIGTERM', () => {
|
|
65
|
+
void reporting.flush({ deadlineMs: opts.deadlineMs ?? 2000 });
|
|
66
|
+
});
|
|
67
|
+
}
|
package/dist/reader.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Db, EventsPage, EventsPageOptions } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Newest first, keyset-paged on `(occurred_at, id)`, which is the table's
|
|
4
|
+
* first index. Every filter is an equality on an indexed column. No
|
|
5
|
+
* permission is applied here: the host gates, and this returns rows as
|
|
6
|
+
* stored.
|
|
7
|
+
*/
|
|
8
|
+
export declare function eventsPage(db: Db, opts?: EventsPageOptions): Promise<EventsPage>;
|