cursedbelt-server 4.18.0 → 4.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/activity/index.d.ts +2 -1
- package/dist/server/activity/index.js +2 -1
- package/dist/server/auth/passwordCost.d.ts +21 -0
- package/dist/server/auth/passwordCost.js +80 -0
- package/dist/server/bench/index.d.ts +1 -0
- package/dist/server/bench/index.js +1 -0
- package/dist/server/bench/tail.d.ts +110 -0
- package/dist/server/bench/tail.js +182 -0
- package/dist/server/d1/index.d.ts +1 -2
- package/dist/server/d1/index.js +9 -9
- package/dist/server/d1/pullD1.js +16 -3
- package/dist/server/engagement/api.d.ts +71 -0
- package/dist/server/engagement/api.js +84 -0
- package/dist/server/engagement/env.d.ts +18 -0
- package/dist/server/engagement/env.js +52 -0
- package/dist/server/engagement/index.d.ts +55 -0
- package/dist/server/engagement/index.js +55 -0
- package/dist/server/engagement/places.d.ts +22 -0
- package/dist/server/engagement/places.js +63 -0
- package/dist/server/engagement/policy.d.ts +168 -0
- package/dist/server/engagement/policy.js +202 -0
- package/dist/server/engagement/store.d.ts +92 -0
- package/dist/server/engagement/store.js +223 -0
- package/dist/server/engagement/summary.d.ts +102 -0
- package/dist/server/engagement/summary.js +127 -0
- package/dist/server/engagement/types.d.ts +42 -0
- package/dist/server/engagement/types.js +12 -0
- package/dist/server/maps-budget/mapsBudget.d.ts +194 -0
- package/dist/server/maps-budget/mapsBudget.js +193 -0
- package/dist/server/satellite/config.d.ts +173 -0
- package/dist/server/satellite/config.js +259 -0
- package/dist/server/satellite/door.d.ts +112 -0
- package/dist/server/satellite/door.js +149 -0
- package/dist/server/storage/binaryStore.d.ts +18 -0
- package/dist/server/storage/binaryStore.js +32 -1
- package/dist/server/storage/derivatives.d.ts +253 -0
- package/dist/server/storage/derivatives.js +266 -0
- package/dist/server/storage/uploadSession.d.ts +75 -0
- package/dist/server/storage/uploadSession.js +74 -0
- package/docs/THE-DEV-DEPENDENCY-CYCLE.md +55 -0
- package/docs/activity.md +43 -0
- package/docs/engagement.md +47 -0
- package/docs/notifications.md +43 -0
- package/docs/retention.md +81 -0
- package/docs/skipped-tests.md +19 -0
- package/package.json +46 -9
- package/src/barrelsReachNoOptionalPeer.spec.ts +5 -3
- package/src/leafSubpathsImportNothing.spec.ts +49 -0
- package/src/server/activity/index.ts +2 -1
- package/src/server/auth/passwordCost.spec.ts +42 -0
- package/src/server/auth/passwordCost.ts +87 -0
- package/src/server/bench/index.ts +13 -0
- package/src/server/bench/tail.spec.ts +126 -0
- package/src/server/bench/tail.ts +237 -0
- package/src/server/d1/index.ts +9 -9
- package/src/server/d1/pullD1.spec.ts +20 -0
- package/src/server/d1/pullD1.ts +18 -2
- package/src/server/engagement/api.ts +119 -0
- package/src/server/engagement/engagement.spec.ts +462 -0
- package/src/server/engagement/env.ts +73 -0
- package/src/server/engagement/index.ts +92 -0
- package/src/server/engagement/places.ts +76 -0
- package/src/server/engagement/policy.ts +250 -0
- package/src/server/engagement/store.ts +272 -0
- package/src/server/engagement/summary.ts +216 -0
- package/src/server/engagement/types.ts +61 -0
- package/src/server/maps-budget/mapsBudget.spec.ts +366 -0
- package/src/server/maps-budget/mapsBudget.ts +304 -0
- package/src/server/satellite/config.ts +389 -0
- package/src/server/satellite/door.ts +169 -0
- package/src/server/satellite/satellite.spec.ts +161 -0
- package/src/server/storage/binaryStore.ts +31 -1
- package/src/server/storage/derivatives.spec.ts +125 -0
- package/src/server/storage/derivatives.ts +329 -0
- package/src/server/storage/uploadSession.spec.ts +132 -0
- package/src/server/storage/uploadSession.ts +114 -0
- package/dist/server/d1/kysely.d.ts +0 -56
- package/dist/server/d1/kysely.js +0 -138
- package/src/server/d1/kysely.spec.ts +0 -145
- package/src/server/d1/kysely.ts +0 -169
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The rules that decide what a place-view is allowed to say — pure, so every
|
|
3
|
+
* one of them is a test rather than a comment.
|
|
4
|
+
*
|
|
5
|
+
* ── Why product analytics is a SEPARATE thing from `metrics/` ───────────────
|
|
6
|
+
* `metrics/` answers "is the fleet healthy": throughput, latency, status codes,
|
|
7
|
+
* 5xx, memory. Its store header makes a promise in capital letters — no user
|
|
8
|
+
* id, no query string, no path — and that promise is what makes it safe to run
|
|
9
|
+
* the same middleware inside the zero-knowledge vault and the owner's notes.
|
|
10
|
+
*
|
|
11
|
+
* This module answers a different question the owner actually asked: "who uses
|
|
12
|
+
* what apps, popularity stats, which users like what features/apps, and what is
|
|
13
|
+
* holding and not holding interest". That question is USER-KEYED by definition,
|
|
14
|
+
* so it cannot live behind that promise — and weakening the promise to make
|
|
15
|
+
* room for it would quietly re-point twelve apps' request log at a user id.
|
|
16
|
+
* Two stores, two guarantees, one of which stays absolute.
|
|
17
|
+
*
|
|
18
|
+
* ── The privacy design, in three rules ──────────────────────────────────────
|
|
19
|
+
* 1. **A place is an ALLOWLISTED id, never a path.** The allowlist is the app's
|
|
20
|
+
* own `routes.manifest.json` — the list it already maintains for the route
|
|
21
|
+
* health check. Anything not on it records as `OTHER_PLACE`. So a note id, a
|
|
22
|
+
* ROM title, a vault entry name or a search string cannot become a place
|
|
23
|
+
* even if a caller tries: there is no code path from request text to a
|
|
24
|
+
* stored string. This is structural, not a sanitizer that can be outgrown.
|
|
25
|
+
* 2. **Some apps are excluded by NAME, in the kit.** `apps/vault` is
|
|
26
|
+
* zero-knowledge by construction and `fractals`/`patterns` are public with
|
|
27
|
+
* no accounts at all. Making that a per-app config would mean the vault's
|
|
28
|
+
* protection was "somebody remembered"; making it a refusal here means an
|
|
29
|
+
* agent who wires the beacon into the vault gets a no-op and a red gate.
|
|
30
|
+
* 3. **Counts and clocks only.** A row is (user, place, day) → views, active
|
|
31
|
+
* ms, sessions, first/last. There is no event log, so there is nothing to
|
|
32
|
+
* correlate against and nothing to leak beyond "this account opened this
|
|
33
|
+
* page N times that day".
|
|
34
|
+
*
|
|
35
|
+
* ── Honest at n = 3 ─────────────────────────────────────────────────────────
|
|
36
|
+
* This fleet has one real user plus family. Every number here is one a human
|
|
37
|
+
* can check by hand: whole counts, whole days, an explicit "never opened", and
|
|
38
|
+
* a drop-off signal stated in days rather than as a rate. Nothing is a ratio
|
|
39
|
+
* over a denominator of two.
|
|
40
|
+
*/
|
|
41
|
+
/** The bucket a view lands in when its place is not on the app's allowlist. */
|
|
42
|
+
export const OTHER_PLACE = "other";
|
|
43
|
+
/**
|
|
44
|
+
* Apps that must never record engagement, whatever they ask for.
|
|
45
|
+
*
|
|
46
|
+
* 🔴 Do not turn this into a config option. See rule 2 in the file header: the
|
|
47
|
+
* point is that the vault's exclusion cannot be forgotten, mis-set, or lost in
|
|
48
|
+
* a merge. `scripts/engagementRoster.test.ts` fails if one of these ever mounts
|
|
49
|
+
* the recorder.
|
|
50
|
+
*/
|
|
51
|
+
export const ENGAGEMENT_EXCLUDED_APPS = [
|
|
52
|
+
// Zero-knowledge by construction. The server stores opaque ciphertext and
|
|
53
|
+
// must not learn even which pages the owner spends time on — a per-place
|
|
54
|
+
// dwell profile over a password manager is a map of which credentials
|
|
55
|
+
// matter, which is exactly the thing the design refuses to hold.
|
|
56
|
+
"vault",
|
|
57
|
+
// Public, no accounts, no session — there is no user to key a row on, and
|
|
58
|
+
// tsk_01KZN61WJ8X15SKGPMFABBJ889 exists because an analytics beacon reached
|
|
59
|
+
// somewhere it had no business being. A public app gets request metrics.
|
|
60
|
+
"fractals",
|
|
61
|
+
"patterns",
|
|
62
|
+
// The dev-only host-header proxy. Never deployed, serves no pages of its own.
|
|
63
|
+
"gateway",
|
|
64
|
+
];
|
|
65
|
+
/** Is this app allowed to record engagement at all? */
|
|
66
|
+
export function engagementAllowedForApp(app) {
|
|
67
|
+
return !ENGAGEMENT_EXCLUDED_APPS.includes(app.trim().toLowerCase());
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Resolve what a browser reported to one of the app's declared places.
|
|
71
|
+
*
|
|
72
|
+
* A caller may send either the place ID (an app that knows its own route
|
|
73
|
+
* names) or the LOCATION it is at (`#/calendar`, `/#/t/scifi`) — and the
|
|
74
|
+
* location form is the one that matters, because it is what lets twelve apps
|
|
75
|
+
* adopt the beacon without each writing a `route.view → place id` mapping that
|
|
76
|
+
* can drift. Everything unrecognized collapses to {@link OTHER_PLACE}.
|
|
77
|
+
*
|
|
78
|
+
* 🔴 A `:param` segment matches ANY single segment and contributes NOTHING to
|
|
79
|
+
* what is stored: `/#/t/my-divorce-reading-list` resolves to the id `tracker`,
|
|
80
|
+
* and the shelf name is discarded here, before any database sees it. That is
|
|
81
|
+
* the mechanism behind rule 1 in the file header — the parameterized routes are
|
|
82
|
+
* exactly the ones whose URLs carry user content, so they are exactly the ones
|
|
83
|
+
* that must reduce to a name.
|
|
84
|
+
*/
|
|
85
|
+
export function resolvePlace(claimed, allowed) {
|
|
86
|
+
// NOT short-circuited on empty: the home page normalizes to `""` from both
|
|
87
|
+
// sides (the manifest writes it `/`, a browser reports `/` or `#/`), so the
|
|
88
|
+
// root is a legitimate match rather than a missing value. An app whose
|
|
89
|
+
// manifest declares no root still gets `other` from the loop below.
|
|
90
|
+
const want = normalizePlaceId(claimed);
|
|
91
|
+
const wantSegments = want.split("/");
|
|
92
|
+
for (const entry of allowed) {
|
|
93
|
+
const ref = typeof entry === "string" ? { id: entry, path: "" } : entry;
|
|
94
|
+
if (normalizePlaceId(ref.id) === want)
|
|
95
|
+
return ref.id;
|
|
96
|
+
if (ref.path && pathMatches(ref.path, wantSegments))
|
|
97
|
+
return ref.id;
|
|
98
|
+
}
|
|
99
|
+
return OTHER_PLACE;
|
|
100
|
+
}
|
|
101
|
+
/** Does `segments` match this manifest path, treating `:param` as a wildcard? */
|
|
102
|
+
function pathMatches(manifestPath, segments) {
|
|
103
|
+
const pattern = normalizePlaceId(manifestPath);
|
|
104
|
+
// The manifest writes the home page as `/`, which normalizes to the empty
|
|
105
|
+
// string — and so does a browser sitting at `/` or `#/`. Both are one empty
|
|
106
|
+
// segment, so the generic comparison below already agrees; this is only here
|
|
107
|
+
// to say that the empty case is intended rather than an accident.
|
|
108
|
+
const parts = pattern.split("/");
|
|
109
|
+
if (parts.length !== segments.length)
|
|
110
|
+
return false;
|
|
111
|
+
return parts.every((part, i) => part.startsWith(":") || part === segments[i]);
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* `#/deep/Link` → `deep/link`. Exported for the tests that pin rule 1.
|
|
115
|
+
*
|
|
116
|
+
* The leading `/#` a manifest path carries and the bare `#` a browser reports
|
|
117
|
+
* are both stripped, because `/#/calendar`, `#/calendar` and `/calendar` are
|
|
118
|
+
* one page and three spellings — and three spellings would read as three
|
|
119
|
+
* unpopular features rather than one popular one.
|
|
120
|
+
*/
|
|
121
|
+
export function normalizePlaceId(raw) {
|
|
122
|
+
return (raw
|
|
123
|
+
.trim()
|
|
124
|
+
.toLowerCase()
|
|
125
|
+
// The query string goes FIRST and unconditionally — `#/search?q=<what
|
|
126
|
+
// they typed>` is the single most likely way user text would arrive here.
|
|
127
|
+
.replace(/\?.*$/, "")
|
|
128
|
+
.replace(/^\/?#/, "")
|
|
129
|
+
.replace(/^\/+|\/+$/g, ""));
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* How long a gap between two views before the second one starts a new session.
|
|
133
|
+
*
|
|
134
|
+
* 30 minutes is the web-analytics convention and it is the right one here for a
|
|
135
|
+
* reason specific to this fleet: the owner leaves tabs open for days. Without a
|
|
136
|
+
* gap rule, "session length" would be measured in hours of an idle laptop and
|
|
137
|
+
* every engagement number would be fiction.
|
|
138
|
+
*/
|
|
139
|
+
export const SESSION_GAP_MS = 30 * 60 * 1000;
|
|
140
|
+
/**
|
|
141
|
+
* The largest slice of time a single view may contribute to `activeMs`.
|
|
142
|
+
*
|
|
143
|
+
* Dwell is measured BETWEEN beacons, so the last view before someone walks away
|
|
144
|
+
* has no successor to bound it — and the next beacon, whenever it comes, would
|
|
145
|
+
* otherwise donate the whole absence to the previous page. Capping each slice
|
|
146
|
+
* means an abandoned tab contributes one cap's worth and stops, which
|
|
147
|
+
* under-reports a long genuine read and cannot over-report an empty room. That
|
|
148
|
+
* is the correct direction for a signal whose job is to say what holds
|
|
149
|
+
* interest: over-reporting attention nobody paid is the failure that matters.
|
|
150
|
+
*/
|
|
151
|
+
export const MAX_DWELL_SLICE_MS = 5 * 60 * 1000;
|
|
152
|
+
/**
|
|
153
|
+
* Fold one view into a user's history.
|
|
154
|
+
*
|
|
155
|
+
* `lastTs` is the previous view's timestamp for this USER (across places, not
|
|
156
|
+
* per place) — dwell is time spent on the page you were already on, so the
|
|
157
|
+
* credit goes backwards. A null `lastTs` is a first-ever view: a new session
|
|
158
|
+
* that credits nothing, because there is no earlier page to have been reading.
|
|
159
|
+
*
|
|
160
|
+
* Clock skew is handled by clamping rather than by trusting: a `ts` at or
|
|
161
|
+
* before `lastTs` yields zero dwell. A device with a wrong clock should make a
|
|
162
|
+
* number boring, never negative.
|
|
163
|
+
*/
|
|
164
|
+
export function foldView(lastTs, ts) {
|
|
165
|
+
if (lastTs === null)
|
|
166
|
+
return { newSession: true, activeMs: 0 };
|
|
167
|
+
const gap = ts - lastTs;
|
|
168
|
+
if (gap <= 0)
|
|
169
|
+
return { newSession: false, activeMs: 0 };
|
|
170
|
+
if (gap >= SESSION_GAP_MS)
|
|
171
|
+
return { newSession: true, activeMs: 0 };
|
|
172
|
+
return { newSession: false, activeMs: Math.min(gap, MAX_DWELL_SLICE_MS) };
|
|
173
|
+
}
|
|
174
|
+
/** `2026-08-14` for a timestamp, in UTC. The day bucket every row is keyed on. */
|
|
175
|
+
export function dayKey(ts) {
|
|
176
|
+
return new Date(ts).toISOString().slice(0, 10);
|
|
177
|
+
}
|
|
178
|
+
/** How many whole days ago `ts` was, relative to `now`. */
|
|
179
|
+
export function daysSince(ts, now) {
|
|
180
|
+
return Math.max(0, Math.floor((now - ts) / 86_400_000));
|
|
181
|
+
}
|
|
182
|
+
export const INTEREST_ACTIVE_DAYS = 7;
|
|
183
|
+
export const INTEREST_FADING_DAYS = 30;
|
|
184
|
+
/**
|
|
185
|
+
* Classify one (user, app) pair.
|
|
186
|
+
*
|
|
187
|
+
* `sessions` distinguishes the two ways of not coming back, and they mean
|
|
188
|
+
* different things: someone who used an app for a month and stopped
|
|
189
|
+
* (`dropped`) is a feature that stopped delivering, where someone who opened it
|
|
190
|
+
* once and never returned (`bounced`) is a first impression that failed. One
|
|
191
|
+
* bucket for both would hide whichever is rarer.
|
|
192
|
+
*/
|
|
193
|
+
export function classifyInterest(input, now) {
|
|
194
|
+
if (input.lastSeenTs === null)
|
|
195
|
+
return "never";
|
|
196
|
+
const age = daysSince(input.lastSeenTs, now);
|
|
197
|
+
if (age < INTEREST_ACTIVE_DAYS)
|
|
198
|
+
return "active";
|
|
199
|
+
if (age < INTEREST_FADING_DAYS)
|
|
200
|
+
return "fading";
|
|
201
|
+
return input.sessions <= 1 ? "bounced" : "dropped";
|
|
202
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The per-app ENGAGEMENT database — who opens which places, how often, and when
|
|
3
|
+
* they stopped.
|
|
4
|
+
*
|
|
5
|
+
* ── Its own file, beside `metrics.sqlite`, on purpose ───────────────────────
|
|
6
|
+
* `metrics/store.ts` opens with a promise: no user id, no query string, no
|
|
7
|
+
* path, in twelve apps including the zero-knowledge vault. That promise is the
|
|
8
|
+
* reason its middleware is safe to run everywhere, and a user-keyed table added
|
|
9
|
+
* to that file would retire it silently. So this is `engagement.sqlite`: a
|
|
10
|
+
* different file, a different guarantee, and an app that has never mounted the
|
|
11
|
+
* recorder simply has no such file at all.
|
|
12
|
+
*
|
|
13
|
+
* ── AGGREGATED, never an event log ──────────────────────────────────────────
|
|
14
|
+
* One row per (user, place, UTC day). The alternative — a row per view — buys
|
|
15
|
+
* nothing at this fleet's size and costs the one thing worth protecting: an
|
|
16
|
+
* event log is a timeline, and a timeline of a household's private apps is a
|
|
17
|
+
* far more sensitive object than a count. Sequence is exactly the information
|
|
18
|
+
* we do not want to be holding, so the schema cannot express it.
|
|
19
|
+
*
|
|
20
|
+
* The one place ordering DOES matter is sessionization, and it is resolved at
|
|
21
|
+
* WRITE time against `engagement_users.last_ts` — a single stored clock per
|
|
22
|
+
* user, which survives a restart and reveals nothing on its own. See
|
|
23
|
+
* `policy.ts:foldView`.
|
|
24
|
+
*
|
|
25
|
+
* ── Bounded like the rest of the telemetry ──────────────────────────────────
|
|
26
|
+
* Daily rows are trimmed by AGE, because the question this table answers ("is
|
|
27
|
+
* anyone still using it?") is a question about recent history, and a return
|
|
28
|
+
* curve longer than a quarter is not something a three-person fleet can read.
|
|
29
|
+
* The per-user roll-up is NOT trimmed by age: it is one row per (user, place)
|
|
30
|
+
* pair carrying first/last seen, and it is precisely the row that must survive
|
|
31
|
+
* for "dropped off" to be answerable. Dropping it after 90 days would make
|
|
32
|
+
* every lapsed user look like a new one.
|
|
33
|
+
*/
|
|
34
|
+
import { Database } from "bun:sqlite";
|
|
35
|
+
import type { EngagementDay, EngagementUser, EngagementUserPlace } from "./types.js";
|
|
36
|
+
export type { EngagementDay, EngagementUser, EngagementUserPlace } from "./types.js";
|
|
37
|
+
/** How much history the daily grain keeps. */
|
|
38
|
+
export declare const ENGAGEMENT_DAY_RETENTION_DAYS = 120;
|
|
39
|
+
export interface EngagementStoreOptions {
|
|
40
|
+
dbPath: string;
|
|
41
|
+
retentionDays?: number;
|
|
42
|
+
}
|
|
43
|
+
export declare class EngagementStore {
|
|
44
|
+
readonly handle: Database;
|
|
45
|
+
readonly retentionDays: number;
|
|
46
|
+
private writes;
|
|
47
|
+
constructor(options: EngagementStoreOptions);
|
|
48
|
+
private migrate;
|
|
49
|
+
/**
|
|
50
|
+
* Record one resolved place-view.
|
|
51
|
+
*
|
|
52
|
+
* `place` must ALREADY have been through `resolvePlace` — this method does not
|
|
53
|
+
* validate it, because the allowlist lives with the app that owns the route
|
|
54
|
+
* manifest and a second half-check here would be the one somebody trusts. The
|
|
55
|
+
* HTTP door (`api.ts`) is where resolution happens, and `engagement.test.ts`
|
|
56
|
+
* pins that it does.
|
|
57
|
+
*
|
|
58
|
+
* Returns the fold that was applied, so a caller (or a test) can assert on
|
|
59
|
+
* the sessionization rather than re-deriving it.
|
|
60
|
+
*/
|
|
61
|
+
recordView(input: {
|
|
62
|
+
userKey: string;
|
|
63
|
+
place: string;
|
|
64
|
+
ts: number;
|
|
65
|
+
}): {
|
|
66
|
+
newSession: boolean;
|
|
67
|
+
activeMs: number;
|
|
68
|
+
};
|
|
69
|
+
/** Every account's roll-up for this app, most recently seen first. */
|
|
70
|
+
users(): EngagementUser[];
|
|
71
|
+
/** Every (user, place) roll-up — the per-FEATURE grain, whole-fleet readable. */
|
|
72
|
+
places(): EngagementUserPlace[];
|
|
73
|
+
/** Daily buckets inside a window — the sessions-over-time and return curve. */
|
|
74
|
+
days(sinceTs: number): EngagementDay[];
|
|
75
|
+
/**
|
|
76
|
+
* Drop daily rows older than the retention window.
|
|
77
|
+
*
|
|
78
|
+
* 🔴 Only `engagement_days`. The two roll-ups are deliberately untouched —
|
|
79
|
+
* see the file header. An agent "completing" this by trimming them too would
|
|
80
|
+
* delete the only record that somebody used to be here, which is the entire
|
|
81
|
+
* drop-off signal.
|
|
82
|
+
*/
|
|
83
|
+
prune(at?: number): number;
|
|
84
|
+
/** Row counts + age bounds, for the retention card. */
|
|
85
|
+
stats(): {
|
|
86
|
+
days: number;
|
|
87
|
+
places: number;
|
|
88
|
+
users: number;
|
|
89
|
+
oldestDay: string | null;
|
|
90
|
+
};
|
|
91
|
+
close(): void;
|
|
92
|
+
}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The per-app ENGAGEMENT database — who opens which places, how often, and when
|
|
3
|
+
* they stopped.
|
|
4
|
+
*
|
|
5
|
+
* ── Its own file, beside `metrics.sqlite`, on purpose ───────────────────────
|
|
6
|
+
* `metrics/store.ts` opens with a promise: no user id, no query string, no
|
|
7
|
+
* path, in twelve apps including the zero-knowledge vault. That promise is the
|
|
8
|
+
* reason its middleware is safe to run everywhere, and a user-keyed table added
|
|
9
|
+
* to that file would retire it silently. So this is `engagement.sqlite`: a
|
|
10
|
+
* different file, a different guarantee, and an app that has never mounted the
|
|
11
|
+
* recorder simply has no such file at all.
|
|
12
|
+
*
|
|
13
|
+
* ── AGGREGATED, never an event log ──────────────────────────────────────────
|
|
14
|
+
* One row per (user, place, UTC day). The alternative — a row per view — buys
|
|
15
|
+
* nothing at this fleet's size and costs the one thing worth protecting: an
|
|
16
|
+
* event log is a timeline, and a timeline of a household's private apps is a
|
|
17
|
+
* far more sensitive object than a count. Sequence is exactly the information
|
|
18
|
+
* we do not want to be holding, so the schema cannot express it.
|
|
19
|
+
*
|
|
20
|
+
* The one place ordering DOES matter is sessionization, and it is resolved at
|
|
21
|
+
* WRITE time against `engagement_users.last_ts` — a single stored clock per
|
|
22
|
+
* user, which survives a restart and reveals nothing on its own. See
|
|
23
|
+
* `policy.ts:foldView`.
|
|
24
|
+
*
|
|
25
|
+
* ── Bounded like the rest of the telemetry ──────────────────────────────────
|
|
26
|
+
* Daily rows are trimmed by AGE, because the question this table answers ("is
|
|
27
|
+
* anyone still using it?") is a question about recent history, and a return
|
|
28
|
+
* curve longer than a quarter is not something a three-person fleet can read.
|
|
29
|
+
* The per-user roll-up is NOT trimmed by age: it is one row per (user, place)
|
|
30
|
+
* pair carrying first/last seen, and it is precisely the row that must survive
|
|
31
|
+
* for "dropped off" to be answerable. Dropping it after 90 days would make
|
|
32
|
+
* every lapsed user look like a new one.
|
|
33
|
+
*/
|
|
34
|
+
import { Database } from "bun:sqlite";
|
|
35
|
+
import { mkdirSync } from "node:fs";
|
|
36
|
+
import { dirname, resolve } from "node:path";
|
|
37
|
+
import { dayKey, foldView } from "./policy.js";
|
|
38
|
+
/**
|
|
39
|
+
* `bun:sqlite`'s `create: true` creates the FILE, never the directory above it, so a first boot
|
|
40
|
+
* into an empty data dir would throw here. Swallowed on purpose: the open that follows names the
|
|
41
|
+
* real problem, and telemetry must never be what takes an app's boot down.
|
|
42
|
+
*/
|
|
43
|
+
function ensureDbDirectory(dbPath) {
|
|
44
|
+
if (!dbPath || dbPath === ":memory:" || dbPath.startsWith("file::memory:"))
|
|
45
|
+
return;
|
|
46
|
+
try {
|
|
47
|
+
mkdirSync(dirname(resolve(dbPath)), { recursive: true });
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
// see above
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** How much history the daily grain keeps. */
|
|
54
|
+
export const ENGAGEMENT_DAY_RETENTION_DAYS = 120;
|
|
55
|
+
export class EngagementStore {
|
|
56
|
+
handle;
|
|
57
|
+
retentionDays;
|
|
58
|
+
writes = 0;
|
|
59
|
+
constructor(options) {
|
|
60
|
+
this.retentionDays = options.retentionDays ?? ENGAGEMENT_DAY_RETENTION_DAYS;
|
|
61
|
+
// `create: true` creates the file, never the directory above it — see
|
|
62
|
+
// `ensureDbDirectory`, which is the whole of notes' "NOT YET DIAGNOSED" waiver.
|
|
63
|
+
ensureDbDirectory(options.dbPath);
|
|
64
|
+
this.handle = new Database(options.dbPath, { create: true });
|
|
65
|
+
// busy_timeout FIRST — journal_mode takes a lock. scripts/sqliteBusyTimeout.test.ts
|
|
66
|
+
this.handle.run("PRAGMA busy_timeout = 5000");
|
|
67
|
+
this.handle.run("PRAGMA journal_mode = WAL");
|
|
68
|
+
// A beacon must never block a page on an fsync, and losing the last few
|
|
69
|
+
// views to a hard kill costs nothing a human would notice.
|
|
70
|
+
this.handle.run("PRAGMA synchronous = NORMAL");
|
|
71
|
+
this.migrate();
|
|
72
|
+
}
|
|
73
|
+
migrate() {
|
|
74
|
+
this.handle.run(`CREATE TABLE IF NOT EXISTS engagement_days (
|
|
75
|
+
user_key TEXT NOT NULL,
|
|
76
|
+
place TEXT NOT NULL,
|
|
77
|
+
day TEXT NOT NULL,
|
|
78
|
+
views INTEGER NOT NULL DEFAULT 0,
|
|
79
|
+
active_ms INTEGER NOT NULL DEFAULT 0,
|
|
80
|
+
sessions INTEGER NOT NULL DEFAULT 0,
|
|
81
|
+
first_ts INTEGER NOT NULL,
|
|
82
|
+
last_ts INTEGER NOT NULL,
|
|
83
|
+
PRIMARY KEY (user_key, place, day)
|
|
84
|
+
)`);
|
|
85
|
+
this.handle.run("CREATE INDEX IF NOT EXISTS idx_engagement_days_day ON engagement_days (day)");
|
|
86
|
+
// The (user, place) lifetime row. Separate from the daily grain because it
|
|
87
|
+
// must OUTLIVE it — see the header: this is the row that knows somebody
|
|
88
|
+
// stopped, and age-trimming it would turn every lapsed account into a new
|
|
89
|
+
// one on day 121.
|
|
90
|
+
this.handle.run(`CREATE TABLE IF NOT EXISTS engagement_places (
|
|
91
|
+
user_key TEXT NOT NULL,
|
|
92
|
+
place TEXT NOT NULL,
|
|
93
|
+
views INTEGER NOT NULL DEFAULT 0,
|
|
94
|
+
sessions INTEGER NOT NULL DEFAULT 0,
|
|
95
|
+
active_ms INTEGER NOT NULL DEFAULT 0,
|
|
96
|
+
first_ts INTEGER NOT NULL,
|
|
97
|
+
last_ts INTEGER NOT NULL,
|
|
98
|
+
PRIMARY KEY (user_key, place)
|
|
99
|
+
)`);
|
|
100
|
+
this.handle.run("CREATE INDEX IF NOT EXISTS idx_engagement_places_last ON engagement_places (last_ts)");
|
|
101
|
+
// One row per ACCOUNT. `last_ts` here is the sessionization clock — the only
|
|
102
|
+
// piece of ordering the schema keeps, and it keeps exactly one timestamp
|
|
103
|
+
// rather than a sequence (header, "AGGREGATED, never an event log").
|
|
104
|
+
this.handle.run(`CREATE TABLE IF NOT EXISTS engagement_users (
|
|
105
|
+
user_key TEXT PRIMARY KEY,
|
|
106
|
+
views INTEGER NOT NULL DEFAULT 0,
|
|
107
|
+
sessions INTEGER NOT NULL DEFAULT 0,
|
|
108
|
+
active_ms INTEGER NOT NULL DEFAULT 0,
|
|
109
|
+
first_ts INTEGER NOT NULL,
|
|
110
|
+
last_ts INTEGER NOT NULL,
|
|
111
|
+
last_place TEXT NOT NULL DEFAULT ''
|
|
112
|
+
)`);
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Record one resolved place-view.
|
|
116
|
+
*
|
|
117
|
+
* `place` must ALREADY have been through `resolvePlace` — this method does not
|
|
118
|
+
* validate it, because the allowlist lives with the app that owns the route
|
|
119
|
+
* manifest and a second half-check here would be the one somebody trusts. The
|
|
120
|
+
* HTTP door (`api.ts`) is where resolution happens, and `engagement.test.ts`
|
|
121
|
+
* pins that it does.
|
|
122
|
+
*
|
|
123
|
+
* Returns the fold that was applied, so a caller (or a test) can assert on
|
|
124
|
+
* the sessionization rather than re-deriving it.
|
|
125
|
+
*/
|
|
126
|
+
recordView(input) {
|
|
127
|
+
const { userKey, place, ts } = input;
|
|
128
|
+
const prior = this.handle
|
|
129
|
+
.query("SELECT last_ts, last_place FROM engagement_users WHERE user_key = ?")
|
|
130
|
+
.get(userKey);
|
|
131
|
+
const fold = foldView(prior ? prior.last_ts : null, ts);
|
|
132
|
+
// Dwell is credited BACKWARDS, to the page they were already on — which is
|
|
133
|
+
// why `last_place` is stored at all. Crediting it to the page just opened
|
|
134
|
+
// would mean the most-abandoned page in the app scored highest.
|
|
135
|
+
if (fold.activeMs > 0 && prior?.last_place) {
|
|
136
|
+
const creditDay = dayKey(prior.last_ts);
|
|
137
|
+
this.handle.run("UPDATE engagement_days SET active_ms = active_ms + ? WHERE user_key = ? AND place = ? AND day = ?", [fold.activeMs, userKey, prior.last_place, creditDay]);
|
|
138
|
+
this.handle.run("UPDATE engagement_places SET active_ms = active_ms + ? WHERE user_key = ? AND place = ?", [fold.activeMs, userKey, prior.last_place]);
|
|
139
|
+
}
|
|
140
|
+
const sessionDelta = fold.newSession ? 1 : 0;
|
|
141
|
+
this.handle.run(`INSERT INTO engagement_days (user_key, place, day, views, active_ms, sessions, first_ts, last_ts)
|
|
142
|
+
VALUES (?, ?, ?, 1, 0, ?, ?, ?)
|
|
143
|
+
ON CONFLICT(user_key, place, day) DO UPDATE SET
|
|
144
|
+
views = views + 1,
|
|
145
|
+
sessions = sessions + excluded.sessions,
|
|
146
|
+
last_ts = excluded.last_ts`, [userKey, place, dayKey(ts), sessionDelta, ts, ts]);
|
|
147
|
+
this.handle.run(`INSERT INTO engagement_places (user_key, place, views, sessions, active_ms, first_ts, last_ts)
|
|
148
|
+
VALUES (?, ?, 1, ?, 0, ?, ?)
|
|
149
|
+
ON CONFLICT(user_key, place) DO UPDATE SET
|
|
150
|
+
views = views + 1,
|
|
151
|
+
sessions = sessions + excluded.sessions,
|
|
152
|
+
last_ts = excluded.last_ts`, [userKey, place, sessionDelta, ts, ts]);
|
|
153
|
+
this.handle.run(`INSERT INTO engagement_users (user_key, views, sessions, active_ms, first_ts, last_ts, last_place)
|
|
154
|
+
VALUES (?, 1, ?, 0, ?, ?, ?)
|
|
155
|
+
ON CONFLICT(user_key) DO UPDATE SET
|
|
156
|
+
views = views + 1,
|
|
157
|
+
sessions = sessions + excluded.sessions,
|
|
158
|
+
active_ms = active_ms + ?,
|
|
159
|
+
last_ts = excluded.last_ts,
|
|
160
|
+
last_place = excluded.last_place`, [userKey, sessionDelta, ts, ts, place, fold.activeMs]);
|
|
161
|
+
// Amortised, exactly as the metrics store trims: a DELETE per beacon would
|
|
162
|
+
// cost more than the beacon, and never trimming lets a long-lived daemon
|
|
163
|
+
// keep every day it has ever served.
|
|
164
|
+
if (++this.writes % 256 === 0)
|
|
165
|
+
this.prune(ts);
|
|
166
|
+
return fold;
|
|
167
|
+
}
|
|
168
|
+
/** Every account's roll-up for this app, most recently seen first. */
|
|
169
|
+
users() {
|
|
170
|
+
return this.handle
|
|
171
|
+
.query(`SELECT u.user_key AS userKey, u.views AS views, u.sessions AS sessions,
|
|
172
|
+
u.active_ms AS activeMs, u.first_ts AS firstTs, u.last_ts AS lastTs,
|
|
173
|
+
(SELECT COUNT(*) FROM engagement_places p WHERE p.user_key = u.user_key) AS places
|
|
174
|
+
FROM engagement_users u
|
|
175
|
+
ORDER BY u.last_ts DESC`)
|
|
176
|
+
.all();
|
|
177
|
+
}
|
|
178
|
+
/** Every (user, place) roll-up — the per-FEATURE grain, whole-fleet readable. */
|
|
179
|
+
places() {
|
|
180
|
+
return this.handle
|
|
181
|
+
.query(`SELECT user_key AS userKey, place, views, sessions, active_ms AS activeMs,
|
|
182
|
+
first_ts AS firstTs, last_ts AS lastTs
|
|
183
|
+
FROM engagement_places
|
|
184
|
+
ORDER BY views DESC`)
|
|
185
|
+
.all();
|
|
186
|
+
}
|
|
187
|
+
/** Daily buckets inside a window — the sessions-over-time and return curve. */
|
|
188
|
+
days(sinceTs) {
|
|
189
|
+
return this.handle
|
|
190
|
+
.query(`SELECT user_key AS userKey, place, day, views, active_ms AS activeMs, sessions
|
|
191
|
+
FROM engagement_days WHERE day >= ?
|
|
192
|
+
ORDER BY day ASC`)
|
|
193
|
+
.all(dayKey(sinceTs));
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Drop daily rows older than the retention window.
|
|
197
|
+
*
|
|
198
|
+
* 🔴 Only `engagement_days`. The two roll-ups are deliberately untouched —
|
|
199
|
+
* see the file header. An agent "completing" this by trimming them too would
|
|
200
|
+
* delete the only record that somebody used to be here, which is the entire
|
|
201
|
+
* drop-off signal.
|
|
202
|
+
*/
|
|
203
|
+
prune(at = Date.now()) {
|
|
204
|
+
const cutoff = dayKey(at - this.retentionDays * 86_400_000);
|
|
205
|
+
return this.handle.run("DELETE FROM engagement_days WHERE day < ?", [cutoff]).changes;
|
|
206
|
+
}
|
|
207
|
+
/** Row counts + age bounds, for the retention card. */
|
|
208
|
+
stats() {
|
|
209
|
+
const n = (sql) => (this.handle.query(sql).get()?.n ?? 0);
|
|
210
|
+
const oldest = this.handle
|
|
211
|
+
.query("SELECT MIN(day) AS d FROM engagement_days")
|
|
212
|
+
.get();
|
|
213
|
+
return {
|
|
214
|
+
days: n("SELECT COUNT(*) AS n FROM engagement_days"),
|
|
215
|
+
places: n("SELECT COUNT(*) AS n FROM engagement_places"),
|
|
216
|
+
users: n("SELECT COUNT(*) AS n FROM engagement_users"),
|
|
217
|
+
oldestDay: oldest?.d ?? null,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
close() {
|
|
221
|
+
this.handle.close();
|
|
222
|
+
}
|
|
223
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One app's engagement, folded into the shape the console draws.
|
|
3
|
+
*
|
|
4
|
+
* ── Why the APP does the folding ────────────────────────────────────────────
|
|
5
|
+
* The same argument `metrics/inventory.ts` makes: the console runs on this Mac
|
|
6
|
+
* and most of the fleet runs on a checkout-less box reachable only over HTTPS,
|
|
7
|
+
* so the app that owns the data is the only thing that can read it. Shipping
|
|
8
|
+
* raw daily rows would also mean shipping the per-day timeline of a household's
|
|
9
|
+
* private apps across the wire on every poll, when the console draws totals.
|
|
10
|
+
*
|
|
11
|
+
* ── Everything here is pure over rows ───────────────────────────────────────
|
|
12
|
+
* `summarize` takes the three roll-ups and a clock, so `engagement.test.ts`
|
|
13
|
+
* drives it with hand-written rows and checks arithmetic a human can verify —
|
|
14
|
+
* which is the whole design constraint at n = 3. No I/O, no `Date.now()`.
|
|
15
|
+
*/
|
|
16
|
+
import { type EngagementDay, type EngagementUser, type EngagementUserPlace, type InterestVerdict } from "./types.js";
|
|
17
|
+
/** One account's relationship with one app. */
|
|
18
|
+
export interface UserEngagement {
|
|
19
|
+
userKey: string;
|
|
20
|
+
views: number;
|
|
21
|
+
sessions: number;
|
|
22
|
+
activeMs: number;
|
|
23
|
+
firstTs: number;
|
|
24
|
+
lastTs: number;
|
|
25
|
+
/** Distinct places this account has opened here. */
|
|
26
|
+
places: number;
|
|
27
|
+
interest: InterestVerdict;
|
|
28
|
+
/** Whole days since this account was last seen in this app. */
|
|
29
|
+
daysSinceSeen: number;
|
|
30
|
+
}
|
|
31
|
+
/** One PLACE inside an app, across every account — the per-feature grain. */
|
|
32
|
+
export interface PlaceEngagement {
|
|
33
|
+
place: string;
|
|
34
|
+
/** Distinct accounts that have ever opened it. */
|
|
35
|
+
users: number;
|
|
36
|
+
views: number;
|
|
37
|
+
sessions: number;
|
|
38
|
+
activeMs: number;
|
|
39
|
+
lastTs: number;
|
|
40
|
+
/** Mean ms of attention per view. The "does it hold interest" number. */
|
|
41
|
+
msPerView: number;
|
|
42
|
+
}
|
|
43
|
+
/** One day of the app's whole activity. */
|
|
44
|
+
export interface DailyEngagement {
|
|
45
|
+
day: string;
|
|
46
|
+
views: number;
|
|
47
|
+
sessions: number;
|
|
48
|
+
/** Distinct accounts active that day. */
|
|
49
|
+
users: number;
|
|
50
|
+
}
|
|
51
|
+
export interface AppEngagement {
|
|
52
|
+
app: string;
|
|
53
|
+
generatedAt: number;
|
|
54
|
+
users: UserEngagement[];
|
|
55
|
+
places: PlaceEngagement[];
|
|
56
|
+
/** The (user, place) cross grain — what lets the console answer "which
|
|
57
|
+
* features does THIS person like", rather than only the two margins. */
|
|
58
|
+
userPlaces: Array<{
|
|
59
|
+
userKey: string;
|
|
60
|
+
place: string;
|
|
61
|
+
views: number;
|
|
62
|
+
activeMs: number;
|
|
63
|
+
lastTs: number;
|
|
64
|
+
}>;
|
|
65
|
+
daily: DailyEngagement[];
|
|
66
|
+
totals: {
|
|
67
|
+
users: number;
|
|
68
|
+
views: number;
|
|
69
|
+
sessions: number;
|
|
70
|
+
activeMs: number;
|
|
71
|
+
/** Accounts with more than one session — they came back. */
|
|
72
|
+
returning: number;
|
|
73
|
+
/** Accounts with exactly one session, ever. */
|
|
74
|
+
oneTime: number;
|
|
75
|
+
/** Accounts seen in the last week. */
|
|
76
|
+
activeUsers: number;
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
export interface SummarizeInput {
|
|
80
|
+
app: string;
|
|
81
|
+
now: number;
|
|
82
|
+
users: EngagementUser[];
|
|
83
|
+
places: EngagementUserPlace[];
|
|
84
|
+
days: EngagementDay[];
|
|
85
|
+
}
|
|
86
|
+
export declare function summarize(input: SummarizeInput): AppEngagement;
|
|
87
|
+
/**
|
|
88
|
+
* The return curve: of the accounts first seen on a given day, how many came
|
|
89
|
+
* back at least once afterwards.
|
|
90
|
+
*
|
|
91
|
+
* Stated as WHOLE ACCOUNTS on a named cohort day rather than as a percentage,
|
|
92
|
+
* because a percentage of one person is a number that lies confidently. The
|
|
93
|
+
* console renders "2 of 3 came back" for the same reason.
|
|
94
|
+
*/
|
|
95
|
+
export declare function returnCurve(users: Array<{
|
|
96
|
+
firstTs: number;
|
|
97
|
+
sessions: number;
|
|
98
|
+
}>, now: number, windowDays?: number): Array<{
|
|
99
|
+
day: string;
|
|
100
|
+
joined: number;
|
|
101
|
+
returned: number;
|
|
102
|
+
}>;
|