@wtfalch/reporting 0.1.2 → 0.3.0

package/dist/errors/tasks.d.ts

@@ -0,0 +1,16 @@
+import type { Task } from '../housekeeping/index.js';
+import type { Db } from '../types.js';
+/**
+ * Deletes resolved/ignored error rows past the operator's window, up to
+ * twenty batches of 5,000 per run, each in its own statement with a bounded
+ * timeout, renewing the lease between batches. `reporting_prune_errors`
+ * (0003_errors.sql) does the clamping and, critically, never touches a row
+ * still in state 'open' regardless of age.
+ *
+ * Reuses `events.retention_days` rather than adding an `errors.*` setting: a
+ * new setting means a migration to reporting_settings and an operator
+ * control nobody asked for. One retention window for the whole operational
+ * log (events and the errors grouped from them) is the simpler default.
+ */
+export declare const pruneErrors: Task;
+export declare function pruneErrorsBatch(db: Db, days: number, batch: number): Promise<number>;

package/dist/errors/tasks.js

@@ -0,0 +1,50 @@
+import { sql } from 'drizzle-orm';
+import { rowsOf } from '../housekeeping/tasks.js';
+const HOUR = 60 * 60 * 1000;
+/**
+ * Deletes resolved/ignored error rows past the operator's window, up to
+ * twenty batches of 5,000 per run, each in its own statement with a bounded
+ * timeout, renewing the lease between batches. `reporting_prune_errors`
+ * (0003_errors.sql) does the clamping and, critically, never touches a row
+ * still in state 'open' regardless of age.
+ *
+ * Reuses `events.retention_days` rather than adding an `errors.*` setting: a
+ * new setting means a migration to reporting_settings and an operator
+ * control nobody asked for. One retention window for the whole operational
+ * log (events and the errors grouped from them) is the simpler default.
+ */
+export const pruneErrors = {
+    name: 'reporting.prune_errors',
+    every: HOUR,
+    lease: 10 * 60 * 1000,
+    retry: 15 * 60 * 1000,
+    async run(ctx) {
+        const days = ctx.settings['events.retention_days'];
+        let total = 0;
+        for (let i = 0; i < 20; i += 1) {
+            if (ctx.now().getTime() > ctx.deadline.getTime() - 30_000)
+                break;
+            const n = await pruneErrorsBatch(ctx.db, days, 5000);
+            total += n;
+            if (n < 5000)
+                break;
+            if (!(await ctx.renew()))
+                break;
+        }
+        if (total > 0) {
+            ctx.reporting.event({
+                kind: 'reporting.pruned_errors',
+                message: `pruned ${total} error(s) older than ${days} days`,
+                data: { rows: total, days },
+            });
+        }
+    },
+};
+export async function pruneErrorsBatch(db, days, batch) {
+    return db.transaction(async (tx) => {
+        await tx.execute(sql `set local statement_timeout = '30s'`);
+        const result = await tx.execute(sql `select reporting_prune_errors(make_interval(days => ${days}::int), ${batch}::int) as n`);
+        const [row] = rowsOf(result);
+        return Number(row?.n ?? 0);
+    });
+}

package/dist/fake/index.d.ts

@@ -1,5 +1,5 @@
 import type { ReportingEventRow } from '../tables.js';
-import type { AlertFinding, Logger, Reporting } from '../types.js';
+import type { AlertFinding, Logger, Reporting, TrackInput } from '../types.js';
 /**
  * An in-memory `Reporting` for a host's unit tests: same validation, same
  * shape, no database. `rows` is the table; `alerts` is what `onAlert` would
@@ -8,6 +8,10 @@ import type { AlertFinding, Logger, Reporting } from '../types.js';
 export interface FakeReporting extends Reporting {
     readonly rows: ReportingEventRow[];
     readonly alerts: AlertFinding[];
+    /** What `analytics.track` received. */
+    readonly tracked: (TrackInput & {
+        site: string;
+    })[];
     reset(): void;
 }
 export declare function createFakeReporting(opts?: {

package/dist/fake/index.js

@@ -1,4 +1,5 @@
 import { alertRow } from '../alerts.js';
+import { createCapture } from '../errors/capture.js';
 import { eventInputSchema } from '../schema.js';
 import { DEFAULT_SETTINGS, RETENTION_BOUNDS } from '../settings.js';
 import { tables } from '../tables.js';
@@ -8,14 +9,26 @@ export function createFakeReporting(opts = {}) {
     const log = opts.log ?? silent;
     const rows = [];
     const alerts = [];
+    const tracked = [];
     let settings = { ...DEFAULT_SETTINGS };
     let id = 0;
+    // No group table to fake, so the upsert never runs (`defer` is a no-op);
+    // the timeline row through `fake.event` is what a host's test asserts on.
+    const capture = createCapture({
+        db: undefined,
+        log,
+        site,
+        now: () => new Date(),
+        defer: () => { },
+        event: (input) => fake.event(input),
+    });
     const fake = {
         site,
         log,
         tables,
         rows,
         alerts,
+        tracked,
         event(input) {
             const v = eventInputSchema.parse(input);
             id += 1;
@@ -41,6 +54,7 @@ export function createFakeReporting(opts = {}) {
             fake.event(alertRow(finding));
             alerts.push(finding);
         },
+        captureError: (error, context) => capture(error, context),
         async flush() { },
         events: {
             async page(o = {}) {
@@ -88,6 +102,20 @@ export function createFakeReporting(opts = {}) {
                 return { before, after: settings, by, changed };
             },
         },
+        analytics: {
+            async track(input) {
+                tracked.push({ ...input, site });
+            },
+            async series() {
+                return [];
+            },
+            async weekly() {
+                return [];
+            },
+            async recent() {
+                return [];
+            },
+        },
         stats: () => ({ queued: 0, dropped: 0, invalid: 0, flushed: rows.length }),
         reset() {
             rows.length = 0;

package/dist/housekeeping/index.d.ts

@@ -40,6 +40,10 @@ export interface TaskContext {
     renew(): Promise<boolean>;
     /** The run must finish, or stop scheduling work, by this instant. */
     readonly deadline: Date;
+    /** The task's completed-history watermark as claimed, or null. */
+    readonly watermark: string | null;
+    /** Moves the watermark, fenced by the claim token; false when the lease was lost. Pass a transaction to commit it with the work it vouches for. */
+    commitWatermark(value: string, tx?: Db): Promise<boolean>;
 }
 export interface HousekeepingOptions {
     reporting: Reporting;
@@ -59,3 +63,4 @@ export interface Housekeeping {
 }
 export declare function createHousekeeping(options: HousekeepingOptions): Housekeeping;
 export { pruneEvents, retentionLag } from './tasks.js';
+export { pruneAnalytics, rollupAnalytics, rollupAnalyticsWeekly } from '../analytics/tasks.js';

package/dist/housekeeping/index.js

@@ -51,10 +51,19 @@ export function createHousekeeping(options) {
                 .where(sql `${reportingTasks.task} = ${task.name}
             and ${reportingTasks.nextDueAt} <= now()
             and (${reportingTasks.leaseExpiresAt} is null or ${reportingTasks.leaseExpiresAt} < now())`)
-                .returning({ token: reportingTasks.claimToken });
-            return claimed[0]?.token ?? null;
+                .returning({ token: reportingTasks.claimToken, watermark: reportingTasks.watermark });
+            const row = claimed[0];
+            return row?.token ? { token: row.token, watermark: row.watermark ?? null } : null;
         });
     }
+    async function commitWatermark(task, token, value, handle) {
+        const rows = await handle
+            .update(reportingTasks)
+            .set({ watermark: value })
+            .where(sql `${reportingTasks.task} = ${task.name} and ${reportingTasks.claimToken} = ${token}`)
+            .returning({ task: reportingTasks.task });
+        return rows.length === 1;
+    }
     async function renew(task, token) {
         const rows = await db
             .update(reportingTasks)
@@ -78,7 +87,7 @@ export function createHousekeeping(options) {
         })
             .where(sql `${reportingTasks.task} = ${task.name} and ${reportingTasks.claimToken} = ${token}`);
     }
-    async function runClaimed(task, token, settings) {
+    async function runClaimed(task, token, watermark, settings) {
         const started = now();
         const deadline = new Date(started.getTime() + task.lease);
         let timer;
@@ -95,6 +104,8 @@ export function createHousekeeping(options) {
                     now,
                     renew: () => renew(task, token),
                     deadline,
+                    watermark,
+                    commitWatermark: (value, tx) => commitWatermark(task, token, value, tx ?? db),
                 }),
                 timeout,
             ]);
@@ -123,10 +134,10 @@ export function createHousekeeping(options) {
         }
     }
     async function runOne(task, settings) {
-        const token = await claim(task);
-        if (!token)
+        const claimed = await claim(task);
+        if (!claimed)
             return 'skipped';
-        return runClaimed(task, token, settings);
+        return runClaimed(task, claimed.token, claimed.watermark, settings);
     }
     async function tickNow() {
         await ensureRows();
@@ -197,3 +208,4 @@ export function createHousekeeping(options) {
     };
 }
 export { pruneEvents, retentionLag } from './tasks.js';
+export { pruneAnalytics, rollupAnalytics, rollupAnalyticsWeekly } from '../analytics/tasks.js';

package/dist/index.d.ts

@@ -3,7 +3,11 @@ export * from './schema.js';
 export * from './tables.js';
 export * from './types.js';
 export { alertKind, projectDetail } from './alerts.js';
-export { DEFAULT_SETTINGS, RETENTION_BOUNDS } from './settings.js';
+export { CONSENT_MODES, DEFAULT_SETTINGS, RETENTION_BOUNDS } from './settings.js';
+export * from './analytics/schema.js';
+export { createCollector, type CollectorOptions, type Sites } from './analytics/collector.js';
+export { analyticsRecent, analyticsSeries, analyticsWeekly, hideSmallGroups, type Grain, type RecentEvent, type RecentOptions, type SeriesOptions, type SeriesPoint, type WeeklyPoint, } from './analytics/reader.js';
+export type { AnalyticsRow } from './analytics/write.js';
 /**
  * The host's one instance, built once over its drizzle handle and its
  * logger. Framework-neutral: `defer` is how the host says "after this unit of

package/dist/index.js

@@ -1,4 +1,8 @@
 import { alertRow } from './alerts.js';
+import { analyticsRecent, analyticsSeries, analyticsWeekly } from './analytics/reader.js';
+import { ANALYTICS_NAME_PATTERN, normalisePath, propsSchema } from './analytics/schema.js';
+import { insertAnalytics } from './analytics/write.js';
+import { createCapture } from './errors/capture.js';
 import { eventsPage } from './reader.js';
 import { siteSchema } from './schema.js';
 import { getSettings, setSettings } from './settings.js';
@@ -8,7 +12,10 @@ export * from './schema.js';
 export * from './tables.js';
 export * from './types.js';
 export { alertKind, projectDetail } from './alerts.js';
-export { DEFAULT_SETTINGS, RETENTION_BOUNDS } from './settings.js';
+export { CONSENT_MODES, DEFAULT_SETTINGS, RETENTION_BOUNDS } from './settings.js';
+export * from './analytics/schema.js';
+export { createCollector } from './analytics/collector.js';
+export { analyticsRecent, analyticsSeries, analyticsWeekly, hideSmallGroups, } from './analytics/reader.js';
 function modeFromEnv() {
     const env = process.env.NODE_ENV;
     return env === 'development' || env === 'test' ? env : 'production';
@@ -23,25 +30,38 @@ export function createReporting(options) {
     const site = siteSchema.parse(options.site);
     const now = options.now ?? (() => new Date());
     const mode = options.mode ?? modeFromEnv();
+    const defer = options.defer ??
+        ((fn) => {
+            setTimeout(() => void fn(), 0);
+        });
     const writer = new Writer({
         db: options.db,
         log: options.log,
         site,
         mode,
         now,
-        defer: options.defer ??
-            ((fn) => {
-                setTimeout(() => void fn(), 0);
-            }),
+        defer,
         queueLimit: options.queueLimit ?? 1000,
         batchSize: options.batchSize ?? 500,
         flushEveryMs: options.flushEveryMs ?? 1000,
     });
+    // Same db, log, defer, now and site as the writer: one instance, one
+    // upsert path, no second connection or timer to keep in sync.
+    const capture = createCapture({
+        db: options.db,
+        log: options.log,
+        site,
+        now,
+        defer,
+        event: (input) => writer.event(input),
+        release: process.env.REPORTING_RELEASE ?? null,
+    });
     const reporting = {
         site,
         log: options.log,
         tables,
         event: (input) => writer.event(input),
+        captureError: (error, context) => capture(error, context),
         alert(finding) {
             writer.event(alertRow(finding));
             if (!options.onAlert)
@@ -75,6 +95,42 @@ export function createReporting(options) {
                 return change;
             },
         },
+        analytics: {
+            async track(input) {
+                try {
+                    if (!ANALYTICS_NAME_PATTERN.test(input.name)) {
+                        throw new Error(`analytics.track: "${input.name}" is not a name`);
+                    }
+                    const props = propsSchema.parse(input.props ?? {});
+                    const at = input.occurredAt ?? now();
+                    await insertAnalytics(options.db, [
+                        {
+                            occurredAt: at,
+                            receivedAt: now(),
+                            site,
+                            tenantId: input.tenantId ?? null,
+                            visitorId: input.visitorId ?? null,
+                            sessionId: input.sessionId ?? null,
+                            userId: input.userId ?? null,
+                            name: input.name,
+                            path: (options.normalisePath ?? normalisePath)(input.path ?? '/'),
+                            referrerHost: null,
+                            device: input.device ?? 'unknown',
+                            country: input.country ?? null,
+                            props,
+                        },
+                    ]);
+                }
+                catch (error) {
+                    if (mode !== 'production')
+                        throw error;
+                    options.log.error({ err: describe(error), name: input.name }, 'reporting: track failed');
+                }
+            },
+            series: (opts) => analyticsSeries(options.db, opts),
+            weekly: (opts) => analyticsWeekly(options.db, opts),
+            recent: (opts) => analyticsRecent(options.db, opts),
+        },
         stats: () => writer.stats(),
     };
     return reporting;

package/dist/ingest-limit.d.ts

@@ -0,0 +1,22 @@
+/**
+ * The token bucket both public ingest routes share.
+ *
+ * `./next`'s analytics collector and its client-error collector are the only
+ * two routes in this package a host mounts unauthenticated, and both write to
+ * the database on a stranger's say-so. They had one limiter between them: the
+ * analytics one. The error route went without until it was noticed that
+ * `reporting_errors` never prunes an open row (migrations/0003_errors.sql),
+ * so an unbounded public write lands in a table nothing ages out — the two
+ * decisions are safe apart and not together.
+ *
+ * Per process, not per deploy: several containers each keep their own, so the
+ * real ceiling is this times the container count. Enough to bound abuse,
+ * never to meter.
+ */
+export declare class Buckets {
+    private readonly perMinute;
+    private readonly now;
+    private readonly buckets;
+    constructor(perMinute: number, now: () => Date);
+    take(key: string, n: number): boolean;
+}

package/dist/ingest-limit.js

@@ -0,0 +1,39 @@
+/**
+ * The token bucket both public ingest routes share.
+ *
+ * `./next`'s analytics collector and its client-error collector are the only
+ * two routes in this package a host mounts unauthenticated, and both write to
+ * the database on a stranger's say-so. They had one limiter between them: the
+ * analytics one. The error route went without until it was noticed that
+ * `reporting_errors` never prunes an open row (migrations/0003_errors.sql),
+ * so an unbounded public write lands in a table nothing ages out — the two
+ * decisions are safe apart and not together.
+ *
+ * Per process, not per deploy: several containers each keep their own, so the
+ * real ceiling is this times the container count. Enough to bound abuse,
+ * never to meter.
+ */
+export class Buckets {
+    perMinute;
+    now;
+    buckets = new Map();
+    constructor(perMinute, now) {
+        this.perMinute = perMinute;
+        this.now = now;
+    }
+    take(key, n) {
+        const t = this.now().getTime();
+        const b = this.buckets.get(key) ?? { tokens: this.perMinute, at: t };
+        b.tokens = Math.min(this.perMinute, b.tokens + ((t - b.at) / 60_000) * this.perMinute);
+        b.at = t;
+        if (b.tokens < n) {
+            this.buckets.set(key, b);
+            return false;
+        }
+        b.tokens -= n;
+        this.buckets.set(key, b);
+        if (this.buckets.size > 10_000)
+            this.buckets.clear();
+        return true;
+    }
+}