@stacksjs/ts-analytics 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. package/CHANGELOG.md +68 -0
  2. package/LICENSE.md +21 -0
  3. package/README.md +361 -0
  4. package/bin/cli.ts +169 -0
  5. package/dist/Analytics.d.ts +558 -0
  6. package/dist/api.d.ts +109 -0
  7. package/dist/batching.d.ts +93 -0
  8. package/dist/chunk-2mx7fq49.js +4 -0
  9. package/dist/chunk-3z29508k.js +204 -0
  10. package/dist/chunk-deephkz6.js +56 -0
  11. package/dist/chunk-j261vgyp.js +305 -0
  12. package/dist/chunk-xga17tz7.js +207 -0
  13. package/dist/config.d.ts +132 -0
  14. package/dist/dashboard/components/index.d.ts +80 -0
  15. package/dist/dashboard/composables/useAnalytics.d.ts +99 -0
  16. package/dist/dashboard/index.d.ts +110 -0
  17. package/dist/dashboard/types/index.d.ts +144 -0
  18. package/dist/dashboard/utils/index.d.ts +81 -0
  19. package/dist/dynamodb.d.ts +86 -0
  20. package/dist/funnels.d.ts +104 -0
  21. package/dist/geolocation.d.ts +96 -0
  22. package/dist/index.d.ts +371 -0
  23. package/dist/index.js +10211 -0
  24. package/dist/infrastructure/cdk.d.ts +60 -0
  25. package/dist/infrastructure/cloudformation.d.ts +44 -0
  26. package/dist/infrastructure/index.d.ts +45 -0
  27. package/dist/infrastructure/setup.d.ts +133 -0
  28. package/dist/integrations/cloudflare.d.ts +76 -0
  29. package/dist/integrations/hono.d.ts +60 -0
  30. package/dist/integrations/index.d.ts +24 -0
  31. package/dist/integrations/nuxt.d.ts +7 -0
  32. package/dist/integrations/nuxt.js +42 -0
  33. package/dist/integrations/runtime/use-ts-analytics.d.ts +18 -0
  34. package/dist/integrations/runtime/use-ts-analytics.js +17 -0
  35. package/dist/integrations/stx.d.ts +88 -0
  36. package/dist/integrations/stx.js +18 -0
  37. package/dist/lib/crypto-random.d.ts +4 -0
  38. package/dist/lib/salt.d.ts +16 -0
  39. package/dist/local.d.ts +56 -0
  40. package/dist/model-connector.d.ts +145 -0
  41. package/dist/models/AggregatedStats.d.ts +9 -0
  42. package/dist/models/CampaignStats.d.ts +9 -0
  43. package/dist/models/Conversion.d.ts +11 -0
  44. package/dist/models/CustomEvent.d.ts +11 -0
  45. package/dist/models/DeviceStats.d.ts +9 -0
  46. package/dist/models/EventStats.d.ts +9 -0
  47. package/dist/models/GeoStats.d.ts +9 -0
  48. package/dist/models/Goal.d.ts +9 -0
  49. package/dist/models/GoalStats.d.ts +9 -0
  50. package/dist/models/PageStats.d.ts +11 -0
  51. package/dist/models/PageView.d.ts +13 -0
  52. package/dist/models/RealtimeStats.d.ts +9 -0
  53. package/dist/models/ReferrerStats.d.ts +9 -0
  54. package/dist/models/Session.d.ts +11 -0
  55. package/dist/models/Site.d.ts +11 -0
  56. package/dist/models/index.d.ts +28 -0
  57. package/dist/models/types.d.ts +60 -0
  58. package/dist/sqs-buffering.d.ts +243 -0
  59. package/dist/stacks-integration.d.ts +159 -0
  60. package/dist/tracking-script.d.ts +71 -0
  61. package/dist/tracking.d.ts +17 -0
  62. package/dist/tracking.js +26 -0
  63. package/dist/types.d.ts +595 -0
  64. package/dist/utils/geolocation.d.ts +111 -0
  65. package/dist/utils/user-agent.d.ts +17 -0
  66. package/dist/version.d.ts +7 -0
  67. package/package.json +119 -0
  68. package/src/Analytics.ts +3349 -0
  69. package/src/api.ts +1286 -0
  70. package/src/assets/crosswind.css +2220 -0
  71. package/src/batching.ts +452 -0
  72. package/src/components/dashboard/index.ts +18 -0
  73. package/src/config.ts +456 -0
  74. package/src/dashboard/Dashboard.stx +1517 -0
  75. package/src/dashboard/components/AlertCard.stx +177 -0
  76. package/src/dashboard/components/AnalyticsDashboard.stx +354 -0
  77. package/src/dashboard/components/AnimatedNumber.stx +86 -0
  78. package/src/dashboard/components/BarChart.stx +220 -0
  79. package/src/dashboard/components/BrowserBreakdown.stx +98 -0
  80. package/src/dashboard/components/BrowsersTable.stx +125 -0
  81. package/src/dashboard/components/CampaignBreakdown.stx +163 -0
  82. package/src/dashboard/components/CampaignTable.stx +238 -0
  83. package/src/dashboard/components/CountryList.stx +101 -0
  84. package/src/dashboard/components/DataTable.stx +226 -0
  85. package/src/dashboard/components/DateRangePicker.stx +77 -0
  86. package/src/dashboard/components/DeviceBreakdown.stx +94 -0
  87. package/src/dashboard/components/DevicesTable.stx +163 -0
  88. package/src/dashboard/components/DonutChart.stories.ts +55 -0
  89. package/src/dashboard/components/DonutChart.stx +157 -0
  90. package/src/dashboard/components/EmptyState.stx +90 -0
  91. package/src/dashboard/components/EngagementMetrics.stx +183 -0
  92. package/src/dashboard/components/EventsSection.stx +192 -0
  93. package/src/dashboard/components/FilterBar.stx +104 -0
  94. package/src/dashboard/components/FullAnalyticsDashboard.stx +455 -0
  95. package/src/dashboard/components/FunnelChart.stx +142 -0
  96. package/src/dashboard/components/GeoTable.stx +337 -0
  97. package/src/dashboard/components/GoalsPanel.stx +109 -0
  98. package/src/dashboard/components/Header.stx +39 -0
  99. package/src/dashboard/components/HeatmapChart.stx +140 -0
  100. package/src/dashboard/components/LiveActivityFeed.stx +172 -0
  101. package/src/dashboard/components/MetricComparison.stx +106 -0
  102. package/src/dashboard/components/MiniStats.stx +96 -0
  103. package/src/dashboard/components/OSBreakdown.stx +124 -0
  104. package/src/dashboard/components/PageDetailCard.stx +102 -0
  105. package/src/dashboard/components/PagesTable.stx +127 -0
  106. package/src/dashboard/components/ProgressRing.stx +89 -0
  107. package/src/dashboard/components/RealtimeCounter.stories.ts +45 -0
  108. package/src/dashboard/components/RealtimeCounter.stx +46 -0
  109. package/src/dashboard/components/ReferrersTable.stx +180 -0
  110. package/src/dashboard/components/SparklineChart.stx +160 -0
  111. package/src/dashboard/components/StatCard.stories.ts +58 -0
  112. package/src/dashboard/components/StatCard.stx +90 -0
  113. package/src/dashboard/components/SummaryStats.stx +81 -0
  114. package/src/dashboard/components/TabNav.stx +66 -0
  115. package/src/dashboard/components/ThemeSwitcher.stx +124 -0
  116. package/src/dashboard/components/TimeSeriesChart.stx +106 -0
  117. package/src/dashboard/components/TopList.stories.ts +58 -0
  118. package/src/dashboard/components/TopList.stx +74 -0
  119. package/src/dashboard/components/TrendIndicator.stx +84 -0
  120. package/src/dashboard/components/heatmap/ClickHeatmap.stx +264 -0
  121. package/src/dashboard/components/heatmap/ElementClickList.stx +125 -0
  122. package/src/dashboard/components/heatmap/HeatmapControls.stx +125 -0
  123. package/src/dashboard/components/heatmap/HeatmapLegend.stx +69 -0
  124. package/src/dashboard/components/heatmap/PageHeatmap.stx +264 -0
  125. package/src/dashboard/components/heatmap/ScrollHeatmap.stx +127 -0
  126. package/src/dashboard/components/heatmap/index.ts +12 -0
  127. package/src/dashboard/components/index.ts +86 -0
  128. package/src/dashboard/composables/useAnalytics.ts +465 -0
  129. package/src/dashboard/demo/DemoApp.stx +370 -0
  130. package/src/dashboard/demo/index.ts +8 -0
  131. package/src/dashboard/demo/mockData.ts +234 -0
  132. package/src/dashboard/index.ts +117 -0
  133. package/src/dashboard/stx-shim.d.ts +4 -0
  134. package/src/dashboard/types/index.ts +203 -0
  135. package/src/dashboard/utils/index.ts +426 -0
  136. package/src/dynamodb.ts +344 -0
  137. package/src/funnels.ts +534 -0
  138. package/src/geolocation.ts +515 -0
  139. package/src/handlers/alerts.ts +328 -0
  140. package/src/handlers/annotations.ts +128 -0
  141. package/src/handlers/api-keys.ts +240 -0
  142. package/src/handlers/auth.ts +1020 -0
  143. package/src/handlers/authz.ts +137 -0
  144. package/src/handlers/collect.ts +724 -0
  145. package/src/handlers/data.ts +625 -0
  146. package/src/handlers/experiments.ts +138 -0
  147. package/src/handlers/funnels.ts +216 -0
  148. package/src/handlers/goals.ts +218 -0
  149. package/src/handlers/heatmaps.ts +272 -0
  150. package/src/handlers/index.ts +56 -0
  151. package/src/handlers/lib/read-cache.ts +63 -0
  152. package/src/handlers/misc.ts +486 -0
  153. package/src/handlers/oauth.ts +233 -0
  154. package/src/handlers/performance.ts +388 -0
  155. package/src/handlers/sessions.ts +413 -0
  156. package/src/handlers/sharing.ts +198 -0
  157. package/src/handlers/stats.ts +1368 -0
  158. package/src/handlers/team.ts +161 -0
  159. package/src/handlers/uptime.ts +283 -0
  160. package/src/handlers/views.ts +390 -0
  161. package/src/handlers/webhooks.ts +226 -0
  162. package/src/heatmap/index.ts +32 -0
  163. package/src/heatmap/tracking-script.ts +452 -0
  164. package/src/heatmap/types.ts +79 -0
  165. package/src/index.ts +387 -0
  166. package/src/infrastructure/cdk.ts +496 -0
  167. package/src/infrastructure/cloudformation.ts +595 -0
  168. package/src/infrastructure/index.ts +48 -0
  169. package/src/infrastructure/setup.ts +611 -0
  170. package/src/integrations/cloudflare.ts +732 -0
  171. package/src/integrations/hono.ts +589 -0
  172. package/src/integrations/index.ts +27 -0
  173. package/src/integrations/nuxt.ts +78 -0
  174. package/src/integrations/runtime/use-ts-analytics.ts +32 -0
  175. package/src/integrations/stx.ts +138 -0
  176. package/src/jobs/index.ts +127 -0
  177. package/src/lib/crypto-random.ts +41 -0
  178. package/src/lib/ddb-errors.ts +20 -0
  179. package/src/lib/dynamodb.ts +216 -0
  180. package/src/lib/email.ts +38 -0
  181. package/src/lib/ga-import.ts +471 -0
  182. package/src/lib/ga4-api.ts +244 -0
  183. package/src/lib/goals.ts +205 -0
  184. package/src/lib/index.ts +6 -0
  185. package/src/lib/ingest-counters.ts +123 -0
  186. package/src/lib/log.ts +36 -0
  187. package/src/lib/membership.ts +98 -0
  188. package/src/lib/plans.ts +90 -0
  189. package/src/lib/quota.ts +46 -0
  190. package/src/lib/rate-limit.ts +27 -0
  191. package/src/lib/rollups.ts +472 -0
  192. package/src/lib/salt.ts +97 -0
  193. package/src/lib/scheduler.ts +97 -0
  194. package/src/lib/significance.ts +66 -0
  195. package/src/lib/site-retention.ts +56 -0
  196. package/src/local.ts +360 -0
  197. package/src/model-connector.ts +616 -0
  198. package/src/models/AggregatedStats.ts +162 -0
  199. package/src/models/CampaignStats.ts +141 -0
  200. package/src/models/Conversion.ts +135 -0
  201. package/src/models/CustomEvent.ts +123 -0
  202. package/src/models/DeviceStats.ts +105 -0
  203. package/src/models/EventStats.ts +108 -0
  204. package/src/models/GeoStats.ts +112 -0
  205. package/src/models/Goal.ts +105 -0
  206. package/src/models/GoalStats.ts +106 -0
  207. package/src/models/PageStats.ts +152 -0
  208. package/src/models/PageView.ts +277 -0
  209. package/src/models/RealtimeStats.ts +96 -0
  210. package/src/models/ReferrerStats.ts +115 -0
  211. package/src/models/Session.ts +235 -0
  212. package/src/models/Site.ts +115 -0
  213. package/src/models/index.ts +50 -0
  214. package/src/models/orm/index.ts +2043 -0
  215. package/src/models/types.ts +71 -0
  216. package/src/router.ts +521 -0
  217. package/src/sqs-buffering.ts +806 -0
  218. package/src/stacks-integration.ts +527 -0
  219. package/src/tracking-script.ts +944 -0
  220. package/src/tracking.ts +27 -0
  221. package/src/types/analytics.ts +219 -0
  222. package/src/types/api.ts +75 -0
  223. package/src/types/bun-router.d.ts +15 -0
  224. package/src/types/dashboard.ts +62 -0
  225. package/src/types/index.ts +66 -0
  226. package/src/types/stx.d.ts +27 -0
  227. package/src/types/window.d.ts +101 -0
  228. package/src/types.ts +911 -0
  229. package/src/utils/cache.ts +148 -0
  230. package/src/utils/date.ts +118 -0
  231. package/src/utils/filters.ts +71 -0
  232. package/src/utils/geolocation.ts +323 -0
  233. package/src/utils/index.ts +9 -0
  234. package/src/utils/response.ts +180 -0
  235. package/src/utils/timezone-country.ts +242 -0
  236. package/src/utils/user-agent.ts +110 -0
  237. package/src/version.ts +7 -0
@@ -0,0 +1,472 @@
1
+ /**
2
+ * Daily pre-aggregation rollups (#94).
3
+ *
4
+ * Dashboards used to re-scan every raw PAGEVIEW#/SESSION# item on each query.
5
+ * A scheduled job now writes one ROLLUP#DAY#{YYYY-MM-DD} item per site per
6
+ * complete day (views, unique visitors, sessions, bounces, duration, events),
7
+ * and the heavy read paths (/stats, /timeseries) consume those — touching raw
8
+ * events only for days without a rollup (today's live window, or gaps).
9
+ *
10
+ * Semantics note: a rollup stores that day's unique visitors; summing across
11
+ * days counts a returning visitor once per day (Fathom-style "daily uniques"),
12
+ * whereas the raw path deduplicates across the whole range. Counts converge
13
+ * for single-day ranges and diverge slightly for multi-day ones.
14
+ *
15
+ * A day with no traffic still gets a zero rollup so it reads as covered —
16
+ * otherwise empty days would be re-scanned forever.
17
+ */
18
+ import { queryAllItems, querySessionItemsInRange, dynamodb, TABLE_NAME, marshall, unmarshall } from './dynamodb'
19
+
20
+ export interface DayRollup {
21
+ day: string // YYYY-MM-DD
22
+ views: number
23
+ visitors: number
24
+ sessions: number
25
+ bounces: number
26
+ totalDuration: number
27
+ events: number
28
+ }
29
+
30
+ /** All YYYY-MM-DD keys touched by [start, end], inclusive. Pure — unit tested. */
31
+ export function dayKeysBetween(start: Date, end: Date): string[] {
32
+ const days: string[] = []
33
+ const cur = new Date(Date.UTC(start.getUTCFullYear(), start.getUTCMonth(), start.getUTCDate()))
34
+ const last = end.toISOString().slice(0, 10)
35
+ while (true) {
36
+ const key = cur.toISOString().slice(0, 10)
37
+ days.push(key)
38
+ if (key >= last)
39
+ break
40
+ cur.setUTCDate(cur.getUTCDate() + 1)
41
+ }
42
+ return days
43
+ }
44
+
45
+ /** Whether a day is complete (strictly before today, UTC). Pure — unit tested. */
46
+ export function isCompleteDay(day: string, now: Date = new Date()): boolean {
47
+ return day < now.toISOString().slice(0, 10)
48
+ }
49
+
50
+ /**
51
+ * A day "settles" ROLLUP_SETTLE_DAYS after it completes. Only settled days may
52
+ * be SERVED from a rollup — the most-recent complete day(s) stay on the live raw
53
+ * path, so late-arriving events for an already-rolled day (unload beacons near
54
+ * the midnight boundary, retries, clock skew) are counted immediately instead of
55
+ * being silently dropped (#162). The rollup job recomputes the trailing window,
56
+ * so longer-tail late events are folded in before a day becomes settled. Pure.
57
+ */
58
+ export const ROLLUP_SETTLE_DAYS = 1
59
+ export function isSettledDay(day: string, now: Date = new Date()): boolean {
60
+ const cutoff = new Date(now.getTime() - ROLLUP_SETTLE_DAYS * 24 * 60 * 60 * 1000)
61
+ return day < cutoff.toISOString().slice(0, 10)
62
+ }
63
+
64
+ /**
65
+ * Days whose full 24h window lies inside [start, end] — only these may be
66
+ * served from a full-day rollup; partial first/last days must stay on the raw
67
+ * path or counts would include hours outside the range. Pure — unit tested.
68
+ */
69
+ export function fullyCoveredDays(start: Date, end: Date): string[] {
70
+ const startIso = start.toISOString()
71
+ const endIso = end.toISOString()
72
+ return dayKeysBetween(start, end).filter((day) => {
73
+ return startIso <= `${day}T00:00:00.000Z` && endIso >= `${day}T23:59:59.999Z`
74
+ })
75
+ }
76
+
77
+ /** Compute a single day's rollup from raw pageviews + sessions. */
78
+ export async function computeDayRollup(siteId: string, day: string): Promise<DayRollup> {
79
+ // Pageviews are timestamp-keyed → one bounded range query (paginated).
80
+ const visitors = new Set<string>()
81
+ let views = 0
82
+ let lastKey: Record<string, any> | undefined
83
+ do {
84
+ const page = await dynamodb.query({
85
+ TableName: TABLE_NAME,
86
+ KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
87
+ ExpressionAttributeValues: {
88
+ ':pk': { S: `SITE#${siteId}` },
89
+ ':start': { S: `PAGEVIEW#${day}T00:00:00.000Z` },
90
+ ':end': { S: `PAGEVIEW#${day}T23:59:59.999Z` },
91
+ },
92
+ ExclusiveStartKey: lastKey,
93
+ }) as { Items?: any[], LastEvaluatedKey?: Record<string, any> }
94
+ for (const raw of (page.Items || [])) {
95
+ const pv = unmarshall(raw)
96
+ views++
97
+ if (pv.visitorId)
98
+ visitors.add(pv.visitorId)
99
+ }
100
+ lastKey = page.LastEvaluatedKey
101
+ } while (lastKey)
102
+
103
+ let sessions = 0
104
+ let bounces = 0
105
+ let totalDuration = 0
106
+ let events = 0
107
+ // Day-bounded via the time-keyed GSI entry (#171): the rollup job used to
108
+ // scan EVERY session per site per recomputed day.
109
+ const dayStart = new Date(`${day}T00:00:00.000Z`)
110
+ const dayEnd = new Date(`${day}T23:59:59.999Z`)
111
+ {
112
+ const page = await querySessionItemsInRange(siteId, dayStart, dayEnd)
113
+ for (const raw of (page.Items || [])) {
114
+ const s = unmarshall(raw)
115
+ const startedDay = typeof s.startedAt === 'string' ? s.startedAt.slice(0, 10) : ''
116
+ if (startedDay !== day)
117
+ continue
118
+ sessions++
119
+ if (s.isBounce)
120
+ bounces++
121
+ // Real engaged time when departure pings exist (#167), else wall-clock.
122
+ totalDuration += (s.activeTime > 0 ? s.activeTime : (s.duration || 0))
123
+ events += s.eventCount || 0
124
+ }
125
+ }
126
+
127
+ return { day, views, visitors: visitors.size, sessions, bounces, totalDuration, events }
128
+ }
129
+
130
+ /** Write a day's rollup item. */
131
+ export async function writeDayRollup(siteId: string, rollup: DayRollup): Promise<void> {
132
+ await dynamodb.putItem({
133
+ TableName: TABLE_NAME,
134
+ Item: marshall({
135
+ pk: `SITE#${siteId}`,
136
+ sk: `ROLLUP#DAY#${rollup.day}`,
137
+ ...rollup,
138
+ computedAt: new Date().toISOString(),
139
+ }),
140
+ })
141
+ }
142
+
143
+ /** Read all day rollups for a site within [startDay, endDay] (YYYY-MM-DD). */
144
+ export async function readDayRollups(siteId: string, startDay: string, endDay: string): Promise<Map<string, DayRollup>> {
145
+ const result = await dynamodb.query({
146
+ TableName: TABLE_NAME,
147
+ KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
148
+ ExpressionAttributeValues: {
149
+ ':pk': { S: `SITE#${siteId}` },
150
+ ':start': { S: `ROLLUP#DAY#${startDay}` },
151
+ ':end': { S: `ROLLUP#DAY#${endDay}` },
152
+ },
153
+ }) as { Items?: any[] }
154
+
155
+ const map = new Map<string, DayRollup>()
156
+ for (const raw of (result.Items || [])) {
157
+ const r = unmarshall(raw)
158
+ if (r.day)
159
+ map.set(r.day, r as DayRollup)
160
+ }
161
+ return map
162
+ }
163
+
164
+ /**
165
+ * Recompute rollups for the last `daysBack` complete days of a site. Recomputing
166
+ * the whole trailing window (rather than skipping existing rollups) folds in
167
+ * events that arrived late for an already-rolled day, so they aren't silently
168
+ * dropped (#162). Rollups are deterministic from raw events, so re-runs are
169
+ * safe. Returns how many were written.
170
+ */
171
+ export async function ensureDayRollups(siteId: string, daysBack = 3): Promise<number> {
172
+ const now = new Date()
173
+ const end = new Date(now.getTime() - 24 * 60 * 60 * 1000) // yesterday
174
+ const start = new Date(now.getTime() - daysBack * 24 * 60 * 60 * 1000)
175
+ // Trailing settle window is always recomputed (late events, #162).
176
+ const recompute = new Set(dayKeysBetween(start, end).filter(d => isCompleteDay(d, now)))
177
+
178
+ // Fill-once: any older settled day within raw retention that's missing its
179
+ // dimensional rollup gets computed before the raw rows TTL out (#172).
180
+ const retentionStart = new Date(now.getTime() - 35 * 24 * 60 * 60 * 1000)
181
+ const fillable = dayKeysBetween(retentionStart, end).filter(d => isSettledDay(d, now))
182
+ if (fillable.length > 0) {
183
+ const existingDims = await readDayDimRollups(siteId, fillable[0], fillable[fillable.length - 1])
184
+ for (const day of fillable) {
185
+ if (!existingDims.has(day))
186
+ recompute.add(day)
187
+ }
188
+ }
189
+ if (recompute.size === 0)
190
+ return 0
191
+
192
+ // Imported history (#155) must survive the trailing-window recompute: GA
193
+ // days predate live tracking, so recomputing them from (empty) raw rows
194
+ // would replace real history with zeros.
195
+ const guardDays = [...recompute].sort()
196
+ const existing = await readDayRollups(siteId, guardDays[0], guardDays[guardDays.length - 1])
197
+ for (const [day, rollup] of existing) {
198
+ if ((rollup as { source?: string }).source === 'ga-import')
199
+ recompute.delete(day)
200
+ }
201
+ if (recompute.size === 0)
202
+ return 0
203
+
204
+ let written = 0
205
+ for (const day of [...recompute].sort()) {
206
+ const rollup = await computeDayRollup(siteId, day)
207
+ await writeDayRollup(siteId, rollup)
208
+ const dims = await computeDayDimRollup(siteId, day)
209
+ await writeDayDimRollup(siteId, dims)
210
+ written++
211
+ }
212
+ return written
213
+ }
214
+
215
+ /** Roll up every site's recent complete days (driven by the scheduled job). */
216
+ export async function runDailyRollups(daysBack = 3): Promise<number> {
217
+ const sitesResult = await dynamodb.query({
218
+ TableName: TABLE_NAME,
219
+ KeyConditionExpression: 'pk = :pk',
220
+ ExpressionAttributeValues: { ':pk': { S: 'SITES' } },
221
+ }) as { Items?: any[] }
222
+
223
+ let written = 0
224
+ for (const raw of (sitesResult.Items || [])) {
225
+ const site = unmarshall(raw)
226
+ const siteId = site.id || site.siteId
227
+ if (!siteId)
228
+ continue
229
+ try {
230
+ written += await ensureDayRollups(siteId, daysBack)
231
+ }
232
+ catch (e) {
233
+ console.error(`Rollup failed for site ${siteId}:`, (e as Error).message)
234
+ }
235
+ }
236
+ return written
237
+ }
238
+
239
+ // ============================================================================
240
+ // Dimensional day rollups (#172)
241
+ //
242
+ // Raw rows TTL out (30d), and every breakdown panel read them exclusively —
243
+ // so Pages/Referrers/Devices/… silently shrank to a 30-day window while the
244
+ // totals above them showed the full range, and "All Time" was structurally
245
+ // impossible. One ROLLUP#DIMS#{day} item per site per settled day holds
246
+ // per-dimension aggregates forever; read paths serve a rolled prefix and
247
+ // aggregate raw only for the live tail (same pattern as handleGetStats).
248
+ // Filtered queries can't be answered from rollups and stay raw-only.
249
+ // ============================================================================
250
+
251
+ /** Per-key metric cells (short field names keep the item small). */
252
+ export interface DimCellPage { w: number, v: number, e: number } // views, visitors, entries
253
+ export interface DimCellSource { v: number, w: number } // visitors, views
254
+ export interface DimCellVisitors { v: number }
255
+ export interface DimCellCampaign { v: number, s: number, src: string, med: string }
256
+ export interface DimCellEvent { c: number, v: number, val: number }
257
+ export interface DimCellEntry { s: number, b: number }
258
+ export interface DimCellExit { s: number }
259
+
260
+ export interface DayDimRollup {
261
+ day: string
262
+ pages: Record<string, DimCellPage>
263
+ sources: Record<string, DimCellSource>
264
+ devices: Record<string, DimCellVisitors>
265
+ browsers: Record<string, DimCellVisitors>
266
+ os: Record<string, DimCellVisitors>
267
+ countries: Record<string, DimCellVisitors>
268
+ /** Keyed `${country}:${region}` — mirrors handleGetRegions. */
269
+ regions: Record<string, DimCellVisitors>
270
+ /** Keyed `${country}:${region}:${city}` — mirrors handleGetCities. */
271
+ cities: Record<string, DimCellVisitors>
272
+ campaigns: Record<string, DimCellCampaign>
273
+ events: Record<string, DimCellEvent>
274
+ entries: Record<string, DimCellEntry>
275
+ exits: Record<string, DimCellExit>
276
+ }
277
+
278
+ /** Cap each dimension's stored keys so the item stays far below 400KB. */
279
+ const DIM_CAP = 300
280
+
281
+ function capDim<T>(map: Record<string, T>, primary: (cell: T) => number, dim: string, siteId: string, day: string): Record<string, T> {
282
+ const keys = Object.keys(map)
283
+ if (keys.length <= DIM_CAP)
284
+ return map
285
+ console.warn(`[rollups] ${siteId} ${day} ${dim}: ${keys.length} keys, keeping top ${DIM_CAP}`)
286
+ const kept = keys.sort((a, b) => primary(map[b]) - primary(map[a])).slice(0, DIM_CAP)
287
+ const out: Record<string, T> = {}
288
+ for (const k of kept) out[k] = map[k]
289
+ return out
290
+ }
291
+
292
+ /** Compute one day's per-dimension aggregates (three bounded passes). */
293
+ export async function computeDayDimRollup(siteId: string, day: string): Promise<DayDimRollup> {
294
+ const dayStart = new Date(`${day}T00:00:00.000Z`)
295
+ const dayEnd = new Date(`${day}T23:59:59.999Z`)
296
+
297
+ const pages: Record<string, { w: number, v: Set<string>, e: number }> = {}
298
+ {
299
+ const res = await queryAllItems({
300
+ TableName: TABLE_NAME,
301
+ KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
302
+ ExpressionAttributeValues: {
303
+ ':pk': { S: `SITE#${siteId}` },
304
+ ':start': { S: `PAGEVIEW#${day}T00:00:00.000Z` },
305
+ ':end': { S: `PAGEVIEW#${day}T23:59:59.999Z` },
306
+ },
307
+ })
308
+ for (const raw of (res.Items || [])) {
309
+ const pv = unmarshall(raw)
310
+ const path = pv.path || '/'
311
+ const cell = (pages[path] ||= { w: 0, v: new Set(), e: 0 })
312
+ cell.w++
313
+ if (pv.visitorId) cell.v.add(pv.visitorId)
314
+ if (pv.isUnique) cell.e++
315
+ }
316
+ }
317
+
318
+ const sources: Record<string, { v: Set<string>, w: number }> = {}
319
+ const simple: Record<'devices' | 'browsers' | 'os' | 'countries', Record<string, Set<string>>>
320
+ = { devices: {}, browsers: {}, os: {}, countries: {} }
321
+ const regions: Record<string, Set<string>> = {}
322
+ const cities: Record<string, Set<string>> = {}
323
+ const campaigns: Record<string, { v: Set<string>, s: number, src: string, med: string }> = {}
324
+ const entries: Record<string, DimCellEntry> = {}
325
+ const exits: Record<string, DimCellExit> = {}
326
+ {
327
+ const res = await querySessionItemsInRange(siteId, dayStart, dayEnd)
328
+ for (const raw of (res.Items || [])) {
329
+ const sess = unmarshall(raw)
330
+ if (typeof sess.startedAt === 'string' && sess.startedAt.slice(0, 10) !== day) continue
331
+ const vid = sess.visitorId
332
+ const source = sess.referrerSource || 'Direct'
333
+ ;(sources[source] ||= { v: new Set(), w: 0 })
334
+ if (vid) sources[source].v.add(vid)
335
+ sources[source].w += sess.pageViewCount || 1
336
+ const dims: Array<['devices' | 'browsers' | 'os' | 'countries', string | undefined]> = [
337
+ ['devices', sess.deviceType], ['browsers', sess.browser], ['os', sess.os],
338
+ ['countries', sess.country],
339
+ ]
340
+ for (const [dim, value] of dims) {
341
+ // Countries omit empty values entirely (the report drops the Unknown
342
+ // bucket); other dims keep the handlers' 'Unknown'/'unknown' fallback.
343
+ if (dim === 'countries' && !value) continue
344
+ const key = value || (dim === 'devices' ? 'unknown' : 'Unknown')
345
+ ;(simple[dim][key] ||= new Set())
346
+ if (vid) simple[dim][key].add(vid)
347
+ }
348
+ {
349
+ // Composite keys mirror the handlers (Unknown buckets included there).
350
+ const rKey = `${sess.country || 'Unknown'}:${sess.region || 'Unknown'}`
351
+ ;(regions[rKey] ||= new Set())
352
+ if (vid) regions[rKey].add(vid)
353
+ const cKey = `${rKey}:${sess.city || 'Unknown'}`
354
+ ;(cities[cKey] ||= new Set())
355
+ if (vid) cities[cKey].add(vid)
356
+ }
357
+ if (sess.utmCampaign) {
358
+ const c = (campaigns[sess.utmCampaign] ||= { v: new Set(), s: 0, src: sess.utmSource || '', med: sess.utmMedium || '' })
359
+ if (vid) c.v.add(vid)
360
+ c.s++
361
+ }
362
+ if (sess.entryPath) {
363
+ const e = (entries[sess.entryPath] ||= { s: 0, b: 0 })
364
+ e.s++
365
+ if (sess.isBounce) e.b++
366
+ }
367
+ if (sess.exitPath) {
368
+ const x = (exits[sess.exitPath] ||= { s: 0 })
369
+ x.s++
370
+ }
371
+ }
372
+ }
373
+
374
+ const events: Record<string, { c: number, v: Set<string>, val: number }> = {}
375
+ {
376
+ const res = await queryAllItems({
377
+ TableName: TABLE_NAME,
378
+ KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
379
+ ExpressionAttributeValues: {
380
+ ':pk': { S: `SITE#${siteId}` },
381
+ ':start': { S: `EVENT#${day}T00:00:00.000Z` },
382
+ ':end': { S: `EVENT#${day}T23:59:59.999Z` },
383
+ },
384
+ })
385
+ for (const raw of (res.Items || [])) {
386
+ const ev = unmarshall(raw)
387
+ const name = ev.name || 'unknown'
388
+ const cell = (events[name] ||= { c: 0, v: new Set(), val: 0 })
389
+ cell.c++
390
+ if (ev.visitorId) cell.v.add(ev.visitorId)
391
+ if (typeof ev.value === 'number') cell.val += ev.value
392
+ }
393
+ }
394
+
395
+ const sized = <T, U>(m: Record<string, T>, f: (t: T) => U): Record<string, U> => {
396
+ const out: Record<string, U> = {}
397
+ for (const [k, cell] of Object.entries(m)) out[k] = f(cell)
398
+ return out
399
+ }
400
+
401
+ return {
402
+ day,
403
+ pages: capDim(sized(pages, c => ({ w: c.w, v: c.v.size, e: c.e })), c => c.v, 'pages', siteId, day),
404
+ sources: capDim(sized(sources, c => ({ v: c.v.size, w: c.w })), c => c.v, 'sources', siteId, day),
405
+ devices: sized(simple.devices, set => ({ v: set.size })),
406
+ browsers: capDim(sized(simple.browsers, set => ({ v: set.size })), c => c.v, 'browsers', siteId, day),
407
+ os: sized(simple.os, set => ({ v: set.size })),
408
+ countries: capDim(sized(simple.countries, set => ({ v: set.size })), c => c.v, 'countries', siteId, day),
409
+ regions: capDim(sized(regions, set => ({ v: set.size })), c => c.v, 'regions', siteId, day),
410
+ cities: capDim(sized(cities, set => ({ v: set.size })), c => c.v, 'cities', siteId, day),
411
+ campaigns: capDim(sized(campaigns, c => ({ v: c.v.size, s: c.s, src: c.src, med: c.med })), c => c.v, 'campaigns', siteId, day),
412
+ events: capDim(sized(events, c => ({ c: c.c, v: c.v.size, val: c.val })), c => c.c, 'events', siteId, day),
413
+ entries: capDim(entries, c => c.s, 'entries', siteId, day),
414
+ exits: capDim(exits, c => c.s, 'exits', siteId, day),
415
+ }
416
+ }
417
+
418
+ /** Write a day's dimensional rollup item (no TTL — aggregates live forever). */
419
+ export async function writeDayDimRollup(siteId: string, rollup: DayDimRollup): Promise<void> {
420
+ await dynamodb.putItem({
421
+ TableName: TABLE_NAME,
422
+ Item: marshall({
423
+ pk: `SITE#${siteId}`,
424
+ sk: `ROLLUP#DIMS#${rollup.day}`,
425
+ ...rollup,
426
+ computedAt: new Date().toISOString(),
427
+ }),
428
+ })
429
+ }
430
+
431
+ /** Read dimensional rollups for [startDay, endDay] (YYYY-MM-DD), keyed by day. */
432
+ export async function readDayDimRollups(siteId: string, startDay: string, endDay: string): Promise<Map<string, DayDimRollup>> {
433
+ const result = await queryAllItems({
434
+ TableName: TABLE_NAME,
435
+ KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
436
+ ExpressionAttributeValues: {
437
+ ':pk': { S: `SITE#${siteId}` },
438
+ ':start': { S: `ROLLUP#DIMS#${startDay}` },
439
+ ':end': { S: `ROLLUP#DIMS#${endDay}~` },
440
+ },
441
+ })
442
+ const map = new Map<string, DayDimRollup>()
443
+ for (const raw of (result.Items || [])) {
444
+ const item = unmarshall(raw) as DayDimRollup & { sk?: string }
445
+ if (item.day) map.set(item.day, item)
446
+ }
447
+ return map
448
+ }
449
+
450
+ /**
451
+ * The contiguous settled-day prefix of [startDate, endDate] that dimensional
452
+ * rollups can serve, and where the raw tail begins — the same split
453
+ * handleGetStats uses for its scalar rollups.
454
+ */
455
+ export async function readDimRollupPrefix(siteId: string, startDate: Date, endDate: Date): Promise<{ days: DayDimRollup[], rawWindowStart: Date }> {
456
+ const eligible = fullyCoveredDays(startDate, endDate).filter(d => isSettledDay(d))
457
+ if (eligible.length === 0)
458
+ return { days: [], rawWindowStart: startDate }
459
+ const rollups = await readDayDimRollups(siteId, eligible[0], eligible[eligible.length - 1])
460
+ const days: DayDimRollup[] = []
461
+ for (const day of eligible) {
462
+ const r = rollups.get(day)
463
+ if (!r)
464
+ break
465
+ days.push(r)
466
+ }
467
+ if (days.length === 0)
468
+ return { days: [], rawWindowStart: startDate }
469
+ const next = new Date(`${days[days.length - 1].day}T00:00:00.000Z`)
470
+ next.setUTCDate(next.getUTCDate() + 1)
471
+ return { days, rawWindowStart: next }
472
+ }
@@ -0,0 +1,97 @@
1
+ /**
2
+ * Visitor-hash salt (#88).
3
+ *
4
+ * The cookieless visitor id is hash(ip, ua, siteId, salt). The salt used to be
5
+ * `analytics-${date}` — predictable, so anyone could reproduce/correlate
6
+ * visitor hashes. It is now HMAC-SHA256(secret, date): rotates daily, and
7
+ * cannot be reproduced without the server secret.
8
+ *
9
+ * The secret comes from ANALYTICS_SALT_SECRET. When unset, a process-random
10
+ * secret is generated so hashes stay non-reproducible — but they then differ
11
+ * across instances/restarts (unique-visitor counts reset), so production
12
+ * should always set the env var.
13
+ */
14
+ import { createHmac, randomBytes } from 'node:crypto'
15
+
16
+ let ephemeralSecret: string | null = null
17
+
18
+ function getSaltSecret(): string {
19
+ const configured = process.env.ANALYTICS_SALT_SECRET
20
+ if (configured)
21
+ return configured
22
+ if (!ephemeralSecret) {
23
+ ephemeralSecret = randomBytes(32).toString('hex')
24
+ console.warn('[analytics] ANALYTICS_SALT_SECRET is not set — using an ephemeral salt secret. Visitor hashes will not be stable across restarts/instances; set ANALYTICS_SALT_SECRET in production.')
25
+ }
26
+ return ephemeralSecret
27
+ }
28
+
29
+ /** Daily-rotating, secret-seeded salt for visitor ID hashing.
30
+ *
31
+ * NOTE: process-local — on multi-instance deployments prefer
32
+ * {@link getSharedDailySalt}, which persists the day's salt in DynamoDB so
33
+ * every instance hashes with the same value.
34
+ */
35
+ export function getDailySalt(): string {
36
+ const today = new Date().toISOString().slice(0, 10)
37
+ return createHmac('sha256', getSaltSecret()).update(today).digest('hex')
38
+ }
39
+
40
+ // In-process cache of the shared salt: one DynamoDB read per instance per day.
41
+ let sharedSaltDay: string | null = null
42
+ let sharedSaltValue: string | null = null
43
+
44
+ /**
45
+ * Cluster-consistent daily salt (#165). The process-local salt differs per
46
+ * instance when ANALYTICS_SALT_SECRET is unset (Lambda concurrency, restarts),
47
+ * which inflates unique-visitor counts — the same visitor hashes differently on
48
+ * every instance. Persist the day's salt as a SALT item via conditional put
49
+ * (first writer wins) so all instances converge, with the process-local value
50
+ * as both the candidate and the fallback when DynamoDB is unreachable.
51
+ */
52
+ export async function getSharedDailySalt(): Promise<string> {
53
+ const today = new Date().toISOString().slice(0, 10)
54
+ if (sharedSaltDay === today && sharedSaltValue)
55
+ return sharedSaltValue
56
+
57
+ const local = getDailySalt()
58
+ const key = { pk: { S: 'SALT' }, sk: { S: `DAY#${today}` } }
59
+ try {
60
+ // Deferred import so the legacy sync path never touches DynamoDB.
61
+ const { dynamodb, TABLE_NAME } = await import('./dynamodb')
62
+ try {
63
+ await dynamodb.putItem({
64
+ TableName: TABLE_NAME,
65
+ Item: {
66
+ ...key,
67
+ salt: { S: local },
68
+ // Keep a few days so the settle window can still hash consistently.
69
+ ttl: { N: String(Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60) },
70
+ },
71
+ ConditionExpression: 'attribute_not_exists(pk)',
72
+ })
73
+ sharedSaltDay = today
74
+ sharedSaltValue = local
75
+ return local
76
+ }
77
+ catch (e: any) {
78
+ const { isConditionalCheckFailed } = await import('./dynamodb')
79
+ if (!isConditionalCheckFailed(e))
80
+ throw e
81
+ // Another instance won the race — adopt its salt.
82
+ const res = await dynamodb.getItem({ TableName: TABLE_NAME, Key: key }) as { Item?: { salt?: { S?: string } } }
83
+ const winner = res.Item?.salt?.S
84
+ if (winner) {
85
+ sharedSaltDay = today
86
+ sharedSaltValue = winner
87
+ return winner
88
+ }
89
+ return local
90
+ }
91
+ }
92
+ catch {
93
+ // DynamoDB unreachable — fall back to the process-local salt rather than
94
+ // failing ingest; counts degrade to per-instance behavior until it heals.
95
+ return local
96
+ }
97
+ }
@@ -0,0 +1,97 @@
1
+ /**
2
+ * Lightweight scheduled-jobs runtime.
3
+ *
4
+ * Features register periodic jobs (alerts, uptime checks, webhook delivery,
5
+ * email digests). Last-run is persisted in DynamoDB (JOBS / JOB#{name}) so the
6
+ * schedule survives restarts and is consistent across instances. Trigger it via
7
+ * the in-process loop (startScheduler, single-instance/dev) or an external cron
8
+ * hitting POST /api/jobs/tick (Lambda schedule / EventBridge).
9
+ */
10
+ import { dynamodb, TABLE_NAME, unmarshall } from './dynamodb'
11
+
12
+ export interface Job {
13
+ name: string
14
+ intervalMs: number
15
+ run: () => Promise<void>
16
+ }
17
+
18
+ const jobs: Job[] = []
19
+
20
+ export function registerJob(job: Job): void {
21
+ if (!jobs.some(j => j.name === job.name))
22
+ jobs.push(job)
23
+ }
24
+
25
+ export function registeredJobs(): Job[] {
26
+ return jobs.slice()
27
+ }
28
+
29
+ /** Pure: whether a job is due given its last run (ms epoch, or null) + interval. */
30
+ export function isDue(lastRunMs: number | null, intervalMs: number, now: number): boolean {
31
+ return lastRunMs === null || now - lastRunMs >= intervalMs
32
+ }
33
+
34
+ async function getLastRun(name: string): Promise<number | null> {
35
+ try {
36
+ const r = await dynamodb.getItem({ TableName: TABLE_NAME, Key: { pk: { S: 'JOBS' }, sk: { S: `JOB#${name}` } } })
37
+ if (!r.Item)
38
+ return null
39
+ const v = unmarshall(r.Item).lastRun
40
+ return typeof v === 'number' ? v : null
41
+ }
42
+ catch {
43
+ return null
44
+ }
45
+ }
46
+
47
+ async function setLastRun(name: string, ts: number): Promise<void> {
48
+ try {
49
+ await dynamodb.putItem({
50
+ TableName: TABLE_NAME,
51
+ Item: { pk: { S: 'JOBS' }, sk: { S: `JOB#${name}` }, lastRun: { N: String(ts) }, updatedAt: { S: new Date(ts).toISOString() } },
52
+ })
53
+ }
54
+ catch (e) {
55
+ console.error(`[jobs] setLastRun(${name}) failed:`, (e as Error).message)
56
+ }
57
+ }
58
+
59
+ /** Run every job whose interval has elapsed. Returns which ran / were skipped / failed. */
60
+ export async function runDueJobs(now: number = Date.now()): Promise<{ ran: string[], skipped: string[], failed: string[] }> {
61
+ const ran: string[] = []
62
+ const skipped: string[] = []
63
+ const failed: string[] = []
64
+ for (const job of jobs) {
65
+ const last = await getLastRun(job.name)
66
+ if (!isDue(last, job.intervalMs, now)) {
67
+ skipped.push(job.name)
68
+ continue
69
+ }
70
+ try {
71
+ await job.run()
72
+ await setLastRun(job.name, now)
73
+ ran.push(job.name)
74
+ }
75
+ catch (e) {
76
+ failed.push(job.name)
77
+ console.error(`[jobs] ${job.name} failed:`, (e as Error).message)
78
+ }
79
+ }
80
+ return { ran, skipped, failed }
81
+ }
82
+
83
+ let timer: ReturnType<typeof setInterval> | null = null
84
+
85
+ /** Start the in-process tick loop (single-instance / dev). Idempotent. */
86
+ export function startScheduler(tickMs: number = 60_000): void {
87
+ if (timer)
88
+ return
89
+ timer = setInterval(() => { void runDueJobs() }, tickMs)
90
+ }
91
+
92
+ export function stopScheduler(): void {
93
+ if (timer) {
94
+ clearInterval(timer)
95
+ timer = null
96
+ }
97
+ }