@stacksjs/ts-analytics 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/LICENSE.md +21 -0
- package/README.md +361 -0
- package/bin/cli.ts +169 -0
- package/dist/Analytics.d.ts +558 -0
- package/dist/api.d.ts +109 -0
- package/dist/batching.d.ts +93 -0
- package/dist/chunk-2mx7fq49.js +4 -0
- package/dist/chunk-3z29508k.js +204 -0
- package/dist/chunk-deephkz6.js +56 -0
- package/dist/chunk-j261vgyp.js +305 -0
- package/dist/chunk-xga17tz7.js +207 -0
- package/dist/config.d.ts +132 -0
- package/dist/dashboard/components/index.d.ts +80 -0
- package/dist/dashboard/composables/useAnalytics.d.ts +99 -0
- package/dist/dashboard/index.d.ts +110 -0
- package/dist/dashboard/types/index.d.ts +144 -0
- package/dist/dashboard/utils/index.d.ts +81 -0
- package/dist/dynamodb.d.ts +86 -0
- package/dist/funnels.d.ts +104 -0
- package/dist/geolocation.d.ts +96 -0
- package/dist/index.d.ts +371 -0
- package/dist/index.js +10211 -0
- package/dist/infrastructure/cdk.d.ts +60 -0
- package/dist/infrastructure/cloudformation.d.ts +44 -0
- package/dist/infrastructure/index.d.ts +45 -0
- package/dist/infrastructure/setup.d.ts +133 -0
- package/dist/integrations/cloudflare.d.ts +76 -0
- package/dist/integrations/hono.d.ts +60 -0
- package/dist/integrations/index.d.ts +24 -0
- package/dist/integrations/nuxt.d.ts +7 -0
- package/dist/integrations/nuxt.js +42 -0
- package/dist/integrations/runtime/use-ts-analytics.d.ts +18 -0
- package/dist/integrations/runtime/use-ts-analytics.js +17 -0
- package/dist/integrations/stx.d.ts +88 -0
- package/dist/integrations/stx.js +18 -0
- package/dist/lib/crypto-random.d.ts +4 -0
- package/dist/lib/salt.d.ts +16 -0
- package/dist/local.d.ts +56 -0
- package/dist/model-connector.d.ts +145 -0
- package/dist/models/AggregatedStats.d.ts +9 -0
- package/dist/models/CampaignStats.d.ts +9 -0
- package/dist/models/Conversion.d.ts +11 -0
- package/dist/models/CustomEvent.d.ts +11 -0
- package/dist/models/DeviceStats.d.ts +9 -0
- package/dist/models/EventStats.d.ts +9 -0
- package/dist/models/GeoStats.d.ts +9 -0
- package/dist/models/Goal.d.ts +9 -0
- package/dist/models/GoalStats.d.ts +9 -0
- package/dist/models/PageStats.d.ts +11 -0
- package/dist/models/PageView.d.ts +13 -0
- package/dist/models/RealtimeStats.d.ts +9 -0
- package/dist/models/ReferrerStats.d.ts +9 -0
- package/dist/models/Session.d.ts +11 -0
- package/dist/models/Site.d.ts +11 -0
- package/dist/models/index.d.ts +28 -0
- package/dist/models/types.d.ts +60 -0
- package/dist/sqs-buffering.d.ts +243 -0
- package/dist/stacks-integration.d.ts +159 -0
- package/dist/tracking-script.d.ts +71 -0
- package/dist/tracking.d.ts +17 -0
- package/dist/tracking.js +26 -0
- package/dist/types.d.ts +595 -0
- package/dist/utils/geolocation.d.ts +111 -0
- package/dist/utils/user-agent.d.ts +17 -0
- package/dist/version.d.ts +7 -0
- package/package.json +119 -0
- package/src/Analytics.ts +3349 -0
- package/src/api.ts +1286 -0
- package/src/assets/crosswind.css +2220 -0
- package/src/batching.ts +452 -0
- package/src/components/dashboard/index.ts +18 -0
- package/src/config.ts +456 -0
- package/src/dashboard/Dashboard.stx +1517 -0
- package/src/dashboard/components/AlertCard.stx +177 -0
- package/src/dashboard/components/AnalyticsDashboard.stx +354 -0
- package/src/dashboard/components/AnimatedNumber.stx +86 -0
- package/src/dashboard/components/BarChart.stx +220 -0
- package/src/dashboard/components/BrowserBreakdown.stx +98 -0
- package/src/dashboard/components/BrowsersTable.stx +125 -0
- package/src/dashboard/components/CampaignBreakdown.stx +163 -0
- package/src/dashboard/components/CampaignTable.stx +238 -0
- package/src/dashboard/components/CountryList.stx +101 -0
- package/src/dashboard/components/DataTable.stx +226 -0
- package/src/dashboard/components/DateRangePicker.stx +77 -0
- package/src/dashboard/components/DeviceBreakdown.stx +94 -0
- package/src/dashboard/components/DevicesTable.stx +163 -0
- package/src/dashboard/components/DonutChart.stories.ts +55 -0
- package/src/dashboard/components/DonutChart.stx +157 -0
- package/src/dashboard/components/EmptyState.stx +90 -0
- package/src/dashboard/components/EngagementMetrics.stx +183 -0
- package/src/dashboard/components/EventsSection.stx +192 -0
- package/src/dashboard/components/FilterBar.stx +104 -0
- package/src/dashboard/components/FullAnalyticsDashboard.stx +455 -0
- package/src/dashboard/components/FunnelChart.stx +142 -0
- package/src/dashboard/components/GeoTable.stx +337 -0
- package/src/dashboard/components/GoalsPanel.stx +109 -0
- package/src/dashboard/components/Header.stx +39 -0
- package/src/dashboard/components/HeatmapChart.stx +140 -0
- package/src/dashboard/components/LiveActivityFeed.stx +172 -0
- package/src/dashboard/components/MetricComparison.stx +106 -0
- package/src/dashboard/components/MiniStats.stx +96 -0
- package/src/dashboard/components/OSBreakdown.stx +124 -0
- package/src/dashboard/components/PageDetailCard.stx +102 -0
- package/src/dashboard/components/PagesTable.stx +127 -0
- package/src/dashboard/components/ProgressRing.stx +89 -0
- package/src/dashboard/components/RealtimeCounter.stories.ts +45 -0
- package/src/dashboard/components/RealtimeCounter.stx +46 -0
- package/src/dashboard/components/ReferrersTable.stx +180 -0
- package/src/dashboard/components/SparklineChart.stx +160 -0
- package/src/dashboard/components/StatCard.stories.ts +58 -0
- package/src/dashboard/components/StatCard.stx +90 -0
- package/src/dashboard/components/SummaryStats.stx +81 -0
- package/src/dashboard/components/TabNav.stx +66 -0
- package/src/dashboard/components/ThemeSwitcher.stx +124 -0
- package/src/dashboard/components/TimeSeriesChart.stx +106 -0
- package/src/dashboard/components/TopList.stories.ts +58 -0
- package/src/dashboard/components/TopList.stx +74 -0
- package/src/dashboard/components/TrendIndicator.stx +84 -0
- package/src/dashboard/components/heatmap/ClickHeatmap.stx +264 -0
- package/src/dashboard/components/heatmap/ElementClickList.stx +125 -0
- package/src/dashboard/components/heatmap/HeatmapControls.stx +125 -0
- package/src/dashboard/components/heatmap/HeatmapLegend.stx +69 -0
- package/src/dashboard/components/heatmap/PageHeatmap.stx +264 -0
- package/src/dashboard/components/heatmap/ScrollHeatmap.stx +127 -0
- package/src/dashboard/components/heatmap/index.ts +12 -0
- package/src/dashboard/components/index.ts +86 -0
- package/src/dashboard/composables/useAnalytics.ts +465 -0
- package/src/dashboard/demo/DemoApp.stx +370 -0
- package/src/dashboard/demo/index.ts +8 -0
- package/src/dashboard/demo/mockData.ts +234 -0
- package/src/dashboard/index.ts +117 -0
- package/src/dashboard/stx-shim.d.ts +4 -0
- package/src/dashboard/types/index.ts +203 -0
- package/src/dashboard/utils/index.ts +426 -0
- package/src/dynamodb.ts +344 -0
- package/src/funnels.ts +534 -0
- package/src/geolocation.ts +515 -0
- package/src/handlers/alerts.ts +328 -0
- package/src/handlers/annotations.ts +128 -0
- package/src/handlers/api-keys.ts +240 -0
- package/src/handlers/auth.ts +1020 -0
- package/src/handlers/authz.ts +137 -0
- package/src/handlers/collect.ts +724 -0
- package/src/handlers/data.ts +625 -0
- package/src/handlers/experiments.ts +138 -0
- package/src/handlers/funnels.ts +216 -0
- package/src/handlers/goals.ts +218 -0
- package/src/handlers/heatmaps.ts +272 -0
- package/src/handlers/index.ts +56 -0
- package/src/handlers/lib/read-cache.ts +63 -0
- package/src/handlers/misc.ts +486 -0
- package/src/handlers/oauth.ts +233 -0
- package/src/handlers/performance.ts +388 -0
- package/src/handlers/sessions.ts +413 -0
- package/src/handlers/sharing.ts +198 -0
- package/src/handlers/stats.ts +1368 -0
- package/src/handlers/team.ts +161 -0
- package/src/handlers/uptime.ts +283 -0
- package/src/handlers/views.ts +390 -0
- package/src/handlers/webhooks.ts +226 -0
- package/src/heatmap/index.ts +32 -0
- package/src/heatmap/tracking-script.ts +452 -0
- package/src/heatmap/types.ts +79 -0
- package/src/index.ts +387 -0
- package/src/infrastructure/cdk.ts +496 -0
- package/src/infrastructure/cloudformation.ts +595 -0
- package/src/infrastructure/index.ts +48 -0
- package/src/infrastructure/setup.ts +611 -0
- package/src/integrations/cloudflare.ts +732 -0
- package/src/integrations/hono.ts +589 -0
- package/src/integrations/index.ts +27 -0
- package/src/integrations/nuxt.ts +78 -0
- package/src/integrations/runtime/use-ts-analytics.ts +32 -0
- package/src/integrations/stx.ts +138 -0
- package/src/jobs/index.ts +127 -0
- package/src/lib/crypto-random.ts +41 -0
- package/src/lib/ddb-errors.ts +20 -0
- package/src/lib/dynamodb.ts +216 -0
- package/src/lib/email.ts +38 -0
- package/src/lib/ga-import.ts +471 -0
- package/src/lib/ga4-api.ts +244 -0
- package/src/lib/goals.ts +205 -0
- package/src/lib/index.ts +6 -0
- package/src/lib/ingest-counters.ts +123 -0
- package/src/lib/log.ts +36 -0
- package/src/lib/membership.ts +98 -0
- package/src/lib/plans.ts +90 -0
- package/src/lib/quota.ts +46 -0
- package/src/lib/rate-limit.ts +27 -0
- package/src/lib/rollups.ts +472 -0
- package/src/lib/salt.ts +97 -0
- package/src/lib/scheduler.ts +97 -0
- package/src/lib/significance.ts +66 -0
- package/src/lib/site-retention.ts +56 -0
- package/src/local.ts +360 -0
- package/src/model-connector.ts +616 -0
- package/src/models/AggregatedStats.ts +162 -0
- package/src/models/CampaignStats.ts +141 -0
- package/src/models/Conversion.ts +135 -0
- package/src/models/CustomEvent.ts +123 -0
- package/src/models/DeviceStats.ts +105 -0
- package/src/models/EventStats.ts +108 -0
- package/src/models/GeoStats.ts +112 -0
- package/src/models/Goal.ts +105 -0
- package/src/models/GoalStats.ts +106 -0
- package/src/models/PageStats.ts +152 -0
- package/src/models/PageView.ts +277 -0
- package/src/models/RealtimeStats.ts +96 -0
- package/src/models/ReferrerStats.ts +115 -0
- package/src/models/Session.ts +235 -0
- package/src/models/Site.ts +115 -0
- package/src/models/index.ts +50 -0
- package/src/models/orm/index.ts +2043 -0
- package/src/models/types.ts +71 -0
- package/src/router.ts +521 -0
- package/src/sqs-buffering.ts +806 -0
- package/src/stacks-integration.ts +527 -0
- package/src/tracking-script.ts +944 -0
- package/src/tracking.ts +27 -0
- package/src/types/analytics.ts +219 -0
- package/src/types/api.ts +75 -0
- package/src/types/bun-router.d.ts +15 -0
- package/src/types/dashboard.ts +62 -0
- package/src/types/index.ts +66 -0
- package/src/types/stx.d.ts +27 -0
- package/src/types/window.d.ts +101 -0
- package/src/types.ts +911 -0
- package/src/utils/cache.ts +148 -0
- package/src/utils/date.ts +118 -0
- package/src/utils/filters.ts +71 -0
- package/src/utils/geolocation.ts +323 -0
- package/src/utils/index.ts +9 -0
- package/src/utils/response.ts +180 -0
- package/src/utils/timezone-country.ts +242 -0
- package/src/utils/user-agent.ts +110 -0
- package/src/version.ts +7 -0
|
@@ -0,0 +1,472 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Daily pre-aggregation rollups (#94).
|
|
3
|
+
*
|
|
4
|
+
* Dashboards used to re-scan every raw PAGEVIEW#/SESSION# item on each query.
|
|
5
|
+
* A scheduled job now writes one ROLLUP#DAY#{YYYY-MM-DD} item per site per
|
|
6
|
+
* complete day (views, unique visitors, sessions, bounces, duration, events),
|
|
7
|
+
* and the heavy read paths (/stats, /timeseries) consume those — touching raw
|
|
8
|
+
* events only for days without a rollup (today's live window, or gaps).
|
|
9
|
+
*
|
|
10
|
+
* Semantics note: a rollup stores that day's unique visitors; summing across
|
|
11
|
+
* days counts a returning visitor once per day (Fathom-style "daily uniques"),
|
|
12
|
+
* whereas the raw path deduplicates across the whole range. Counts converge
|
|
13
|
+
* for single-day ranges and diverge slightly for multi-day ones.
|
|
14
|
+
*
|
|
15
|
+
* A day with no traffic still gets a zero rollup so it reads as covered —
|
|
16
|
+
* otherwise empty days would be re-scanned forever.
|
|
17
|
+
*/
|
|
18
|
+
import { queryAllItems, querySessionItemsInRange, dynamodb, TABLE_NAME, marshall, unmarshall } from './dynamodb'
|
|
19
|
+
|
|
20
|
+
export interface DayRollup {
|
|
21
|
+
day: string // YYYY-MM-DD
|
|
22
|
+
views: number
|
|
23
|
+
visitors: number
|
|
24
|
+
sessions: number
|
|
25
|
+
bounces: number
|
|
26
|
+
totalDuration: number
|
|
27
|
+
events: number
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** All YYYY-MM-DD keys touched by [start, end], inclusive. Pure — unit tested. */
|
|
31
|
+
export function dayKeysBetween(start: Date, end: Date): string[] {
|
|
32
|
+
const days: string[] = []
|
|
33
|
+
const cur = new Date(Date.UTC(start.getUTCFullYear(), start.getUTCMonth(), start.getUTCDate()))
|
|
34
|
+
const last = end.toISOString().slice(0, 10)
|
|
35
|
+
while (true) {
|
|
36
|
+
const key = cur.toISOString().slice(0, 10)
|
|
37
|
+
days.push(key)
|
|
38
|
+
if (key >= last)
|
|
39
|
+
break
|
|
40
|
+
cur.setUTCDate(cur.getUTCDate() + 1)
|
|
41
|
+
}
|
|
42
|
+
return days
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Whether a day is complete (strictly before today, UTC). Pure — unit tested. */
|
|
46
|
+
export function isCompleteDay(day: string, now: Date = new Date()): boolean {
|
|
47
|
+
return day < now.toISOString().slice(0, 10)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* A day "settles" ROLLUP_SETTLE_DAYS after it completes. Only settled days may
|
|
52
|
+
* be SERVED from a rollup — the most-recent complete day(s) stay on the live raw
|
|
53
|
+
* path, so late-arriving events for an already-rolled day (unload beacons near
|
|
54
|
+
* the midnight boundary, retries, clock skew) are counted immediately instead of
|
|
55
|
+
* being silently dropped (#162). The rollup job recomputes the trailing window,
|
|
56
|
+
* so longer-tail late events are folded in before a day becomes settled. Pure.
|
|
57
|
+
*/
|
|
58
|
+
export const ROLLUP_SETTLE_DAYS = 1
|
|
59
|
+
export function isSettledDay(day: string, now: Date = new Date()): boolean {
|
|
60
|
+
const cutoff = new Date(now.getTime() - ROLLUP_SETTLE_DAYS * 24 * 60 * 60 * 1000)
|
|
61
|
+
return day < cutoff.toISOString().slice(0, 10)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Days whose full 24h window lies inside [start, end] — only these may be
|
|
66
|
+
* served from a full-day rollup; partial first/last days must stay on the raw
|
|
67
|
+
* path or counts would include hours outside the range. Pure — unit tested.
|
|
68
|
+
*/
|
|
69
|
+
export function fullyCoveredDays(start: Date, end: Date): string[] {
|
|
70
|
+
const startIso = start.toISOString()
|
|
71
|
+
const endIso = end.toISOString()
|
|
72
|
+
return dayKeysBetween(start, end).filter((day) => {
|
|
73
|
+
return startIso <= `${day}T00:00:00.000Z` && endIso >= `${day}T23:59:59.999Z`
|
|
74
|
+
})
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Compute a single day's rollup from raw pageviews + sessions. */
|
|
78
|
+
export async function computeDayRollup(siteId: string, day: string): Promise<DayRollup> {
|
|
79
|
+
// Pageviews are timestamp-keyed → one bounded range query (paginated).
|
|
80
|
+
const visitors = new Set<string>()
|
|
81
|
+
let views = 0
|
|
82
|
+
let lastKey: Record<string, any> | undefined
|
|
83
|
+
do {
|
|
84
|
+
const page = await dynamodb.query({
|
|
85
|
+
TableName: TABLE_NAME,
|
|
86
|
+
KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
|
|
87
|
+
ExpressionAttributeValues: {
|
|
88
|
+
':pk': { S: `SITE#${siteId}` },
|
|
89
|
+
':start': { S: `PAGEVIEW#${day}T00:00:00.000Z` },
|
|
90
|
+
':end': { S: `PAGEVIEW#${day}T23:59:59.999Z` },
|
|
91
|
+
},
|
|
92
|
+
ExclusiveStartKey: lastKey,
|
|
93
|
+
}) as { Items?: any[], LastEvaluatedKey?: Record<string, any> }
|
|
94
|
+
for (const raw of (page.Items || [])) {
|
|
95
|
+
const pv = unmarshall(raw)
|
|
96
|
+
views++
|
|
97
|
+
if (pv.visitorId)
|
|
98
|
+
visitors.add(pv.visitorId)
|
|
99
|
+
}
|
|
100
|
+
lastKey = page.LastEvaluatedKey
|
|
101
|
+
} while (lastKey)
|
|
102
|
+
|
|
103
|
+
let sessions = 0
|
|
104
|
+
let bounces = 0
|
|
105
|
+
let totalDuration = 0
|
|
106
|
+
let events = 0
|
|
107
|
+
// Day-bounded via the time-keyed GSI entry (#171): the rollup job used to
|
|
108
|
+
// scan EVERY session per site per recomputed day.
|
|
109
|
+
const dayStart = new Date(`${day}T00:00:00.000Z`)
|
|
110
|
+
const dayEnd = new Date(`${day}T23:59:59.999Z`)
|
|
111
|
+
{
|
|
112
|
+
const page = await querySessionItemsInRange(siteId, dayStart, dayEnd)
|
|
113
|
+
for (const raw of (page.Items || [])) {
|
|
114
|
+
const s = unmarshall(raw)
|
|
115
|
+
const startedDay = typeof s.startedAt === 'string' ? s.startedAt.slice(0, 10) : ''
|
|
116
|
+
if (startedDay !== day)
|
|
117
|
+
continue
|
|
118
|
+
sessions++
|
|
119
|
+
if (s.isBounce)
|
|
120
|
+
bounces++
|
|
121
|
+
// Real engaged time when departure pings exist (#167), else wall-clock.
|
|
122
|
+
totalDuration += (s.activeTime > 0 ? s.activeTime : (s.duration || 0))
|
|
123
|
+
events += s.eventCount || 0
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return { day, views, visitors: visitors.size, sessions, bounces, totalDuration, events }
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Write a day's rollup item. */
|
|
131
|
+
export async function writeDayRollup(siteId: string, rollup: DayRollup): Promise<void> {
|
|
132
|
+
await dynamodb.putItem({
|
|
133
|
+
TableName: TABLE_NAME,
|
|
134
|
+
Item: marshall({
|
|
135
|
+
pk: `SITE#${siteId}`,
|
|
136
|
+
sk: `ROLLUP#DAY#${rollup.day}`,
|
|
137
|
+
...rollup,
|
|
138
|
+
computedAt: new Date().toISOString(),
|
|
139
|
+
}),
|
|
140
|
+
})
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Read all day rollups for a site within [startDay, endDay] (YYYY-MM-DD). */
|
|
144
|
+
export async function readDayRollups(siteId: string, startDay: string, endDay: string): Promise<Map<string, DayRollup>> {
|
|
145
|
+
const result = await dynamodb.query({
|
|
146
|
+
TableName: TABLE_NAME,
|
|
147
|
+
KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
|
|
148
|
+
ExpressionAttributeValues: {
|
|
149
|
+
':pk': { S: `SITE#${siteId}` },
|
|
150
|
+
':start': { S: `ROLLUP#DAY#${startDay}` },
|
|
151
|
+
':end': { S: `ROLLUP#DAY#${endDay}` },
|
|
152
|
+
},
|
|
153
|
+
}) as { Items?: any[] }
|
|
154
|
+
|
|
155
|
+
const map = new Map<string, DayRollup>()
|
|
156
|
+
for (const raw of (result.Items || [])) {
|
|
157
|
+
const r = unmarshall(raw)
|
|
158
|
+
if (r.day)
|
|
159
|
+
map.set(r.day, r as DayRollup)
|
|
160
|
+
}
|
|
161
|
+
return map
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Recompute rollups for the last `daysBack` complete days of a site. Recomputing
|
|
166
|
+
* the whole trailing window (rather than skipping existing rollups) folds in
|
|
167
|
+
* events that arrived late for an already-rolled day, so they aren't silently
|
|
168
|
+
* dropped (#162). Rollups are deterministic from raw events, so re-runs are
|
|
169
|
+
* safe. Returns how many were written.
|
|
170
|
+
*/
|
|
171
|
+
export async function ensureDayRollups(siteId: string, daysBack = 3): Promise<number> {
|
|
172
|
+
const now = new Date()
|
|
173
|
+
const end = new Date(now.getTime() - 24 * 60 * 60 * 1000) // yesterday
|
|
174
|
+
const start = new Date(now.getTime() - daysBack * 24 * 60 * 60 * 1000)
|
|
175
|
+
// Trailing settle window is always recomputed (late events, #162).
|
|
176
|
+
const recompute = new Set(dayKeysBetween(start, end).filter(d => isCompleteDay(d, now)))
|
|
177
|
+
|
|
178
|
+
// Fill-once: any older settled day within raw retention that's missing its
|
|
179
|
+
// dimensional rollup gets computed before the raw rows TTL out (#172).
|
|
180
|
+
const retentionStart = new Date(now.getTime() - 35 * 24 * 60 * 60 * 1000)
|
|
181
|
+
const fillable = dayKeysBetween(retentionStart, end).filter(d => isSettledDay(d, now))
|
|
182
|
+
if (fillable.length > 0) {
|
|
183
|
+
const existingDims = await readDayDimRollups(siteId, fillable[0], fillable[fillable.length - 1])
|
|
184
|
+
for (const day of fillable) {
|
|
185
|
+
if (!existingDims.has(day))
|
|
186
|
+
recompute.add(day)
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
if (recompute.size === 0)
|
|
190
|
+
return 0
|
|
191
|
+
|
|
192
|
+
// Imported history (#155) must survive the trailing-window recompute: GA
|
|
193
|
+
// days predate live tracking, so recomputing them from (empty) raw rows
|
|
194
|
+
// would replace real history with zeros.
|
|
195
|
+
const guardDays = [...recompute].sort()
|
|
196
|
+
const existing = await readDayRollups(siteId, guardDays[0], guardDays[guardDays.length - 1])
|
|
197
|
+
for (const [day, rollup] of existing) {
|
|
198
|
+
if ((rollup as { source?: string }).source === 'ga-import')
|
|
199
|
+
recompute.delete(day)
|
|
200
|
+
}
|
|
201
|
+
if (recompute.size === 0)
|
|
202
|
+
return 0
|
|
203
|
+
|
|
204
|
+
let written = 0
|
|
205
|
+
for (const day of [...recompute].sort()) {
|
|
206
|
+
const rollup = await computeDayRollup(siteId, day)
|
|
207
|
+
await writeDayRollup(siteId, rollup)
|
|
208
|
+
const dims = await computeDayDimRollup(siteId, day)
|
|
209
|
+
await writeDayDimRollup(siteId, dims)
|
|
210
|
+
written++
|
|
211
|
+
}
|
|
212
|
+
return written
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Roll up every site's recent complete days (driven by the scheduled job). */
|
|
216
|
+
export async function runDailyRollups(daysBack = 3): Promise<number> {
|
|
217
|
+
const sitesResult = await dynamodb.query({
|
|
218
|
+
TableName: TABLE_NAME,
|
|
219
|
+
KeyConditionExpression: 'pk = :pk',
|
|
220
|
+
ExpressionAttributeValues: { ':pk': { S: 'SITES' } },
|
|
221
|
+
}) as { Items?: any[] }
|
|
222
|
+
|
|
223
|
+
let written = 0
|
|
224
|
+
for (const raw of (sitesResult.Items || [])) {
|
|
225
|
+
const site = unmarshall(raw)
|
|
226
|
+
const siteId = site.id || site.siteId
|
|
227
|
+
if (!siteId)
|
|
228
|
+
continue
|
|
229
|
+
try {
|
|
230
|
+
written += await ensureDayRollups(siteId, daysBack)
|
|
231
|
+
}
|
|
232
|
+
catch (e) {
|
|
233
|
+
console.error(`Rollup failed for site ${siteId}:`, (e as Error).message)
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return written
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// ============================================================================
|
|
240
|
+
// Dimensional day rollups (#172)
|
|
241
|
+
//
|
|
242
|
+
// Raw rows TTL out (30d), and every breakdown panel read them exclusively —
|
|
243
|
+
// so Pages/Referrers/Devices/… silently shrank to a 30-day window while the
|
|
244
|
+
// totals above them showed the full range, and "All Time" was structurally
|
|
245
|
+
// impossible. One ROLLUP#DIMS#{day} item per site per settled day holds
|
|
246
|
+
// per-dimension aggregates forever; read paths serve a rolled prefix and
|
|
247
|
+
// aggregate raw only for the live tail (same pattern as handleGetStats).
|
|
248
|
+
// Filtered queries can't be answered from rollups and stay raw-only.
|
|
249
|
+
// ============================================================================
|
|
250
|
+
|
|
251
|
+
/** Per-key metric cells (short field names keep the item small). */
|
|
252
|
+
export interface DimCellPage { w: number, v: number, e: number } // views, visitors, entries
|
|
253
|
+
export interface DimCellSource { v: number, w: number } // visitors, views
|
|
254
|
+
export interface DimCellVisitors { v: number }
|
|
255
|
+
export interface DimCellCampaign { v: number, s: number, src: string, med: string }
|
|
256
|
+
export interface DimCellEvent { c: number, v: number, val: number }
|
|
257
|
+
export interface DimCellEntry { s: number, b: number }
|
|
258
|
+
export interface DimCellExit { s: number }
|
|
259
|
+
|
|
260
|
+
export interface DayDimRollup {
|
|
261
|
+
day: string
|
|
262
|
+
pages: Record<string, DimCellPage>
|
|
263
|
+
sources: Record<string, DimCellSource>
|
|
264
|
+
devices: Record<string, DimCellVisitors>
|
|
265
|
+
browsers: Record<string, DimCellVisitors>
|
|
266
|
+
os: Record<string, DimCellVisitors>
|
|
267
|
+
countries: Record<string, DimCellVisitors>
|
|
268
|
+
/** Keyed `${country}:${region}` — mirrors handleGetRegions. */
|
|
269
|
+
regions: Record<string, DimCellVisitors>
|
|
270
|
+
/** Keyed `${country}:${region}:${city}` — mirrors handleGetCities. */
|
|
271
|
+
cities: Record<string, DimCellVisitors>
|
|
272
|
+
campaigns: Record<string, DimCellCampaign>
|
|
273
|
+
events: Record<string, DimCellEvent>
|
|
274
|
+
entries: Record<string, DimCellEntry>
|
|
275
|
+
exits: Record<string, DimCellExit>
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/** Cap each dimension's stored keys so the item stays far below 400KB. */
|
|
279
|
+
const DIM_CAP = 300
|
|
280
|
+
|
|
281
|
+
function capDim<T>(map: Record<string, T>, primary: (cell: T) => number, dim: string, siteId: string, day: string): Record<string, T> {
|
|
282
|
+
const keys = Object.keys(map)
|
|
283
|
+
if (keys.length <= DIM_CAP)
|
|
284
|
+
return map
|
|
285
|
+
console.warn(`[rollups] ${siteId} ${day} ${dim}: ${keys.length} keys, keeping top ${DIM_CAP}`)
|
|
286
|
+
const kept = keys.sort((a, b) => primary(map[b]) - primary(map[a])).slice(0, DIM_CAP)
|
|
287
|
+
const out: Record<string, T> = {}
|
|
288
|
+
for (const k of kept) out[k] = map[k]
|
|
289
|
+
return out
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/** Compute one day's per-dimension aggregates (three bounded passes). */
|
|
293
|
+
export async function computeDayDimRollup(siteId: string, day: string): Promise<DayDimRollup> {
|
|
294
|
+
const dayStart = new Date(`${day}T00:00:00.000Z`)
|
|
295
|
+
const dayEnd = new Date(`${day}T23:59:59.999Z`)
|
|
296
|
+
|
|
297
|
+
const pages: Record<string, { w: number, v: Set<string>, e: number }> = {}
|
|
298
|
+
{
|
|
299
|
+
const res = await queryAllItems({
|
|
300
|
+
TableName: TABLE_NAME,
|
|
301
|
+
KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
|
|
302
|
+
ExpressionAttributeValues: {
|
|
303
|
+
':pk': { S: `SITE#${siteId}` },
|
|
304
|
+
':start': { S: `PAGEVIEW#${day}T00:00:00.000Z` },
|
|
305
|
+
':end': { S: `PAGEVIEW#${day}T23:59:59.999Z` },
|
|
306
|
+
},
|
|
307
|
+
})
|
|
308
|
+
for (const raw of (res.Items || [])) {
|
|
309
|
+
const pv = unmarshall(raw)
|
|
310
|
+
const path = pv.path || '/'
|
|
311
|
+
const cell = (pages[path] ||= { w: 0, v: new Set(), e: 0 })
|
|
312
|
+
cell.w++
|
|
313
|
+
if (pv.visitorId) cell.v.add(pv.visitorId)
|
|
314
|
+
if (pv.isUnique) cell.e++
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const sources: Record<string, { v: Set<string>, w: number }> = {}
|
|
319
|
+
const simple: Record<'devices' | 'browsers' | 'os' | 'countries', Record<string, Set<string>>>
|
|
320
|
+
= { devices: {}, browsers: {}, os: {}, countries: {} }
|
|
321
|
+
const regions: Record<string, Set<string>> = {}
|
|
322
|
+
const cities: Record<string, Set<string>> = {}
|
|
323
|
+
const campaigns: Record<string, { v: Set<string>, s: number, src: string, med: string }> = {}
|
|
324
|
+
const entries: Record<string, DimCellEntry> = {}
|
|
325
|
+
const exits: Record<string, DimCellExit> = {}
|
|
326
|
+
{
|
|
327
|
+
const res = await querySessionItemsInRange(siteId, dayStart, dayEnd)
|
|
328
|
+
for (const raw of (res.Items || [])) {
|
|
329
|
+
const sess = unmarshall(raw)
|
|
330
|
+
if (typeof sess.startedAt === 'string' && sess.startedAt.slice(0, 10) !== day) continue
|
|
331
|
+
const vid = sess.visitorId
|
|
332
|
+
const source = sess.referrerSource || 'Direct'
|
|
333
|
+
;(sources[source] ||= { v: new Set(), w: 0 })
|
|
334
|
+
if (vid) sources[source].v.add(vid)
|
|
335
|
+
sources[source].w += sess.pageViewCount || 1
|
|
336
|
+
const dims: Array<['devices' | 'browsers' | 'os' | 'countries', string | undefined]> = [
|
|
337
|
+
['devices', sess.deviceType], ['browsers', sess.browser], ['os', sess.os],
|
|
338
|
+
['countries', sess.country],
|
|
339
|
+
]
|
|
340
|
+
for (const [dim, value] of dims) {
|
|
341
|
+
// Countries omit empty values entirely (the report drops the Unknown
|
|
342
|
+
// bucket); other dims keep the handlers' 'Unknown'/'unknown' fallback.
|
|
343
|
+
if (dim === 'countries' && !value) continue
|
|
344
|
+
const key = value || (dim === 'devices' ? 'unknown' : 'Unknown')
|
|
345
|
+
;(simple[dim][key] ||= new Set())
|
|
346
|
+
if (vid) simple[dim][key].add(vid)
|
|
347
|
+
}
|
|
348
|
+
{
|
|
349
|
+
// Composite keys mirror the handlers (Unknown buckets included there).
|
|
350
|
+
const rKey = `${sess.country || 'Unknown'}:${sess.region || 'Unknown'}`
|
|
351
|
+
;(regions[rKey] ||= new Set())
|
|
352
|
+
if (vid) regions[rKey].add(vid)
|
|
353
|
+
const cKey = `${rKey}:${sess.city || 'Unknown'}`
|
|
354
|
+
;(cities[cKey] ||= new Set())
|
|
355
|
+
if (vid) cities[cKey].add(vid)
|
|
356
|
+
}
|
|
357
|
+
if (sess.utmCampaign) {
|
|
358
|
+
const c = (campaigns[sess.utmCampaign] ||= { v: new Set(), s: 0, src: sess.utmSource || '', med: sess.utmMedium || '' })
|
|
359
|
+
if (vid) c.v.add(vid)
|
|
360
|
+
c.s++
|
|
361
|
+
}
|
|
362
|
+
if (sess.entryPath) {
|
|
363
|
+
const e = (entries[sess.entryPath] ||= { s: 0, b: 0 })
|
|
364
|
+
e.s++
|
|
365
|
+
if (sess.isBounce) e.b++
|
|
366
|
+
}
|
|
367
|
+
if (sess.exitPath) {
|
|
368
|
+
const x = (exits[sess.exitPath] ||= { s: 0 })
|
|
369
|
+
x.s++
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
const events: Record<string, { c: number, v: Set<string>, val: number }> = {}
|
|
375
|
+
{
|
|
376
|
+
const res = await queryAllItems({
|
|
377
|
+
TableName: TABLE_NAME,
|
|
378
|
+
KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
|
|
379
|
+
ExpressionAttributeValues: {
|
|
380
|
+
':pk': { S: `SITE#${siteId}` },
|
|
381
|
+
':start': { S: `EVENT#${day}T00:00:00.000Z` },
|
|
382
|
+
':end': { S: `EVENT#${day}T23:59:59.999Z` },
|
|
383
|
+
},
|
|
384
|
+
})
|
|
385
|
+
for (const raw of (res.Items || [])) {
|
|
386
|
+
const ev = unmarshall(raw)
|
|
387
|
+
const name = ev.name || 'unknown'
|
|
388
|
+
const cell = (events[name] ||= { c: 0, v: new Set(), val: 0 })
|
|
389
|
+
cell.c++
|
|
390
|
+
if (ev.visitorId) cell.v.add(ev.visitorId)
|
|
391
|
+
if (typeof ev.value === 'number') cell.val += ev.value
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const sized = <T, U>(m: Record<string, T>, f: (t: T) => U): Record<string, U> => {
|
|
396
|
+
const out: Record<string, U> = {}
|
|
397
|
+
for (const [k, cell] of Object.entries(m)) out[k] = f(cell)
|
|
398
|
+
return out
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
return {
|
|
402
|
+
day,
|
|
403
|
+
pages: capDim(sized(pages, c => ({ w: c.w, v: c.v.size, e: c.e })), c => c.v, 'pages', siteId, day),
|
|
404
|
+
sources: capDim(sized(sources, c => ({ v: c.v.size, w: c.w })), c => c.v, 'sources', siteId, day),
|
|
405
|
+
devices: sized(simple.devices, set => ({ v: set.size })),
|
|
406
|
+
browsers: capDim(sized(simple.browsers, set => ({ v: set.size })), c => c.v, 'browsers', siteId, day),
|
|
407
|
+
os: sized(simple.os, set => ({ v: set.size })),
|
|
408
|
+
countries: capDim(sized(simple.countries, set => ({ v: set.size })), c => c.v, 'countries', siteId, day),
|
|
409
|
+
regions: capDim(sized(regions, set => ({ v: set.size })), c => c.v, 'regions', siteId, day),
|
|
410
|
+
cities: capDim(sized(cities, set => ({ v: set.size })), c => c.v, 'cities', siteId, day),
|
|
411
|
+
campaigns: capDim(sized(campaigns, c => ({ v: c.v.size, s: c.s, src: c.src, med: c.med })), c => c.v, 'campaigns', siteId, day),
|
|
412
|
+
events: capDim(sized(events, c => ({ c: c.c, v: c.v.size, val: c.val })), c => c.c, 'events', siteId, day),
|
|
413
|
+
entries: capDim(entries, c => c.s, 'entries', siteId, day),
|
|
414
|
+
exits: capDim(exits, c => c.s, 'exits', siteId, day),
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/** Write a day's dimensional rollup item (no TTL — aggregates live forever). */
|
|
419
|
+
export async function writeDayDimRollup(siteId: string, rollup: DayDimRollup): Promise<void> {
|
|
420
|
+
await dynamodb.putItem({
|
|
421
|
+
TableName: TABLE_NAME,
|
|
422
|
+
Item: marshall({
|
|
423
|
+
pk: `SITE#${siteId}`,
|
|
424
|
+
sk: `ROLLUP#DIMS#${rollup.day}`,
|
|
425
|
+
...rollup,
|
|
426
|
+
computedAt: new Date().toISOString(),
|
|
427
|
+
}),
|
|
428
|
+
})
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** Read dimensional rollups for [startDay, endDay] (YYYY-MM-DD), keyed by day. */
|
|
432
|
+
export async function readDayDimRollups(siteId: string, startDay: string, endDay: string): Promise<Map<string, DayDimRollup>> {
|
|
433
|
+
const result = await queryAllItems({
|
|
434
|
+
TableName: TABLE_NAME,
|
|
435
|
+
KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
|
|
436
|
+
ExpressionAttributeValues: {
|
|
437
|
+
':pk': { S: `SITE#${siteId}` },
|
|
438
|
+
':start': { S: `ROLLUP#DIMS#${startDay}` },
|
|
439
|
+
':end': { S: `ROLLUP#DIMS#${endDay}~` },
|
|
440
|
+
},
|
|
441
|
+
})
|
|
442
|
+
const map = new Map<string, DayDimRollup>()
|
|
443
|
+
for (const raw of (result.Items || [])) {
|
|
444
|
+
const item = unmarshall(raw) as DayDimRollup & { sk?: string }
|
|
445
|
+
if (item.day) map.set(item.day, item)
|
|
446
|
+
}
|
|
447
|
+
return map
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* The contiguous settled-day prefix of [startDate, endDate] that dimensional
|
|
452
|
+
* rollups can serve, and where the raw tail begins — the same split
|
|
453
|
+
* handleGetStats uses for its scalar rollups.
|
|
454
|
+
*/
|
|
455
|
+
export async function readDimRollupPrefix(siteId: string, startDate: Date, endDate: Date): Promise<{ days: DayDimRollup[], rawWindowStart: Date }> {
|
|
456
|
+
const eligible = fullyCoveredDays(startDate, endDate).filter(d => isSettledDay(d))
|
|
457
|
+
if (eligible.length === 0)
|
|
458
|
+
return { days: [], rawWindowStart: startDate }
|
|
459
|
+
const rollups = await readDayDimRollups(siteId, eligible[0], eligible[eligible.length - 1])
|
|
460
|
+
const days: DayDimRollup[] = []
|
|
461
|
+
for (const day of eligible) {
|
|
462
|
+
const r = rollups.get(day)
|
|
463
|
+
if (!r)
|
|
464
|
+
break
|
|
465
|
+
days.push(r)
|
|
466
|
+
}
|
|
467
|
+
if (days.length === 0)
|
|
468
|
+
return { days: [], rawWindowStart: startDate }
|
|
469
|
+
const next = new Date(`${days[days.length - 1].day}T00:00:00.000Z`)
|
|
470
|
+
next.setUTCDate(next.getUTCDate() + 1)
|
|
471
|
+
return { days, rawWindowStart: next }
|
|
472
|
+
}
|
package/src/lib/salt.ts
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Visitor-hash salt (#88).
|
|
3
|
+
*
|
|
4
|
+
* The cookieless visitor id is hash(ip, ua, siteId, salt). The salt used to be
|
|
5
|
+
* `analytics-${date}` — predictable, so anyone could reproduce/correlate
|
|
6
|
+
* visitor hashes. It is now HMAC-SHA256(secret, date): rotates daily, and
|
|
7
|
+
* cannot be reproduced without the server secret.
|
|
8
|
+
*
|
|
9
|
+
* The secret comes from ANALYTICS_SALT_SECRET. When unset, a process-random
|
|
10
|
+
* secret is generated so hashes stay non-reproducible — but they then differ
|
|
11
|
+
* across instances/restarts (unique-visitor counts reset), so production
|
|
12
|
+
* should always set the env var.
|
|
13
|
+
*/
|
|
14
|
+
import { createHmac, randomBytes } from 'node:crypto'
|
|
15
|
+
|
|
16
|
+
let ephemeralSecret: string | null = null
|
|
17
|
+
|
|
18
|
+
function getSaltSecret(): string {
|
|
19
|
+
const configured = process.env.ANALYTICS_SALT_SECRET
|
|
20
|
+
if (configured)
|
|
21
|
+
return configured
|
|
22
|
+
if (!ephemeralSecret) {
|
|
23
|
+
ephemeralSecret = randomBytes(32).toString('hex')
|
|
24
|
+
console.warn('[analytics] ANALYTICS_SALT_SECRET is not set — using an ephemeral salt secret. Visitor hashes will not be stable across restarts/instances; set ANALYTICS_SALT_SECRET in production.')
|
|
25
|
+
}
|
|
26
|
+
return ephemeralSecret
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Daily-rotating, secret-seeded salt for visitor ID hashing.
|
|
30
|
+
*
|
|
31
|
+
* NOTE: process-local — on multi-instance deployments prefer
|
|
32
|
+
* {@link getSharedDailySalt}, which persists the day's salt in DynamoDB so
|
|
33
|
+
* every instance hashes with the same value.
|
|
34
|
+
*/
|
|
35
|
+
export function getDailySalt(): string {
|
|
36
|
+
const today = new Date().toISOString().slice(0, 10)
|
|
37
|
+
return createHmac('sha256', getSaltSecret()).update(today).digest('hex')
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// In-process cache of the shared salt: one DynamoDB read per instance per day.
|
|
41
|
+
let sharedSaltDay: string | null = null
|
|
42
|
+
let sharedSaltValue: string | null = null
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Cluster-consistent daily salt (#165). The process-local salt differs per
|
|
46
|
+
* instance when ANALYTICS_SALT_SECRET is unset (Lambda concurrency, restarts),
|
|
47
|
+
* which inflates unique-visitor counts — the same visitor hashes differently on
|
|
48
|
+
* every instance. Persist the day's salt as a SALT item via conditional put
|
|
49
|
+
* (first writer wins) so all instances converge, with the process-local value
|
|
50
|
+
* as both the candidate and the fallback when DynamoDB is unreachable.
|
|
51
|
+
*/
|
|
52
|
+
export async function getSharedDailySalt(): Promise<string> {
|
|
53
|
+
const today = new Date().toISOString().slice(0, 10)
|
|
54
|
+
if (sharedSaltDay === today && sharedSaltValue)
|
|
55
|
+
return sharedSaltValue
|
|
56
|
+
|
|
57
|
+
const local = getDailySalt()
|
|
58
|
+
const key = { pk: { S: 'SALT' }, sk: { S: `DAY#${today}` } }
|
|
59
|
+
try {
|
|
60
|
+
// Deferred import so the legacy sync path never touches DynamoDB.
|
|
61
|
+
const { dynamodb, TABLE_NAME } = await import('./dynamodb')
|
|
62
|
+
try {
|
|
63
|
+
await dynamodb.putItem({
|
|
64
|
+
TableName: TABLE_NAME,
|
|
65
|
+
Item: {
|
|
66
|
+
...key,
|
|
67
|
+
salt: { S: local },
|
|
68
|
+
// Keep a few days so the settle window can still hash consistently.
|
|
69
|
+
ttl: { N: String(Math.floor(Date.now() / 1000) + 7 * 24 * 60 * 60) },
|
|
70
|
+
},
|
|
71
|
+
ConditionExpression: 'attribute_not_exists(pk)',
|
|
72
|
+
})
|
|
73
|
+
sharedSaltDay = today
|
|
74
|
+
sharedSaltValue = local
|
|
75
|
+
return local
|
|
76
|
+
}
|
|
77
|
+
catch (e: any) {
|
|
78
|
+
const { isConditionalCheckFailed } = await import('./dynamodb')
|
|
79
|
+
if (!isConditionalCheckFailed(e))
|
|
80
|
+
throw e
|
|
81
|
+
// Another instance won the race — adopt its salt.
|
|
82
|
+
const res = await dynamodb.getItem({ TableName: TABLE_NAME, Key: key }) as { Item?: { salt?: { S?: string } } }
|
|
83
|
+
const winner = res.Item?.salt?.S
|
|
84
|
+
if (winner) {
|
|
85
|
+
sharedSaltDay = today
|
|
86
|
+
sharedSaltValue = winner
|
|
87
|
+
return winner
|
|
88
|
+
}
|
|
89
|
+
return local
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
// DynamoDB unreachable — fall back to the process-local salt rather than
|
|
94
|
+
// failing ingest; counts degrade to per-instance behavior until it heals.
|
|
95
|
+
return local
|
|
96
|
+
}
|
|
97
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight scheduled-jobs runtime.
|
|
3
|
+
*
|
|
4
|
+
* Features register periodic jobs (alerts, uptime checks, webhook delivery,
|
|
5
|
+
* email digests). Last-run is persisted in DynamoDB (JOBS / JOB#{name}) so the
|
|
6
|
+
* schedule survives restarts and is consistent across instances. Trigger it via
|
|
7
|
+
* the in-process loop (startScheduler, single-instance/dev) or an external cron
|
|
8
|
+
* hitting POST /api/jobs/tick (Lambda schedule / EventBridge).
|
|
9
|
+
*/
|
|
10
|
+
import { dynamodb, TABLE_NAME, unmarshall } from './dynamodb'
|
|
11
|
+
|
|
12
|
+
export interface Job {
|
|
13
|
+
name: string
|
|
14
|
+
intervalMs: number
|
|
15
|
+
run: () => Promise<void>
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const jobs: Job[] = []
|
|
19
|
+
|
|
20
|
+
export function registerJob(job: Job): void {
|
|
21
|
+
if (!jobs.some(j => j.name === job.name))
|
|
22
|
+
jobs.push(job)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function registeredJobs(): Job[] {
|
|
26
|
+
return jobs.slice()
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Pure: whether a job is due given its last run (ms epoch, or null) + interval. */
|
|
30
|
+
export function isDue(lastRunMs: number | null, intervalMs: number, now: number): boolean {
|
|
31
|
+
return lastRunMs === null || now - lastRunMs >= intervalMs
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
async function getLastRun(name: string): Promise<number | null> {
|
|
35
|
+
try {
|
|
36
|
+
const r = await dynamodb.getItem({ TableName: TABLE_NAME, Key: { pk: { S: 'JOBS' }, sk: { S: `JOB#${name}` } } })
|
|
37
|
+
if (!r.Item)
|
|
38
|
+
return null
|
|
39
|
+
const v = unmarshall(r.Item).lastRun
|
|
40
|
+
return typeof v === 'number' ? v : null
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
return null
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function setLastRun(name: string, ts: number): Promise<void> {
|
|
48
|
+
try {
|
|
49
|
+
await dynamodb.putItem({
|
|
50
|
+
TableName: TABLE_NAME,
|
|
51
|
+
Item: { pk: { S: 'JOBS' }, sk: { S: `JOB#${name}` }, lastRun: { N: String(ts) }, updatedAt: { S: new Date(ts).toISOString() } },
|
|
52
|
+
})
|
|
53
|
+
}
|
|
54
|
+
catch (e) {
|
|
55
|
+
console.error(`[jobs] setLastRun(${name}) failed:`, (e as Error).message)
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Run every job whose interval has elapsed. Returns which ran / were skipped / failed. */
|
|
60
|
+
export async function runDueJobs(now: number = Date.now()): Promise<{ ran: string[], skipped: string[], failed: string[] }> {
|
|
61
|
+
const ran: string[] = []
|
|
62
|
+
const skipped: string[] = []
|
|
63
|
+
const failed: string[] = []
|
|
64
|
+
for (const job of jobs) {
|
|
65
|
+
const last = await getLastRun(job.name)
|
|
66
|
+
if (!isDue(last, job.intervalMs, now)) {
|
|
67
|
+
skipped.push(job.name)
|
|
68
|
+
continue
|
|
69
|
+
}
|
|
70
|
+
try {
|
|
71
|
+
await job.run()
|
|
72
|
+
await setLastRun(job.name, now)
|
|
73
|
+
ran.push(job.name)
|
|
74
|
+
}
|
|
75
|
+
catch (e) {
|
|
76
|
+
failed.push(job.name)
|
|
77
|
+
console.error(`[jobs] ${job.name} failed:`, (e as Error).message)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return { ran, skipped, failed }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
let timer: ReturnType<typeof setInterval> | null = null
|
|
84
|
+
|
|
85
|
+
/** Start the in-process tick loop (single-instance / dev). Idempotent. */
|
|
86
|
+
export function startScheduler(tickMs: number = 60_000): void {
|
|
87
|
+
if (timer)
|
|
88
|
+
return
|
|
89
|
+
timer = setInterval(() => { void runDueJobs() }, tickMs)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function stopScheduler(): void {
|
|
93
|
+
if (timer) {
|
|
94
|
+
clearInterval(timer)
|
|
95
|
+
timer = null
|
|
96
|
+
}
|
|
97
|
+
}
|