@clipin/convex-wearables 0.0.3 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,14 +1,148 @@
1
1
  import { v } from "convex/values";
2
- import { internalMutation, query } from "./_generated/server";
2
+ import { internal } from "./_generated/api";
3
+ import type { Doc, Id } from "./_generated/dataModel";
4
+ import {
5
+ type DatabaseReader,
6
+ type DatabaseWriter,
7
+ internalMutation,
8
+ type MutationCtx,
9
+ mutation,
10
+ query,
11
+ } from "./_generated/server";
12
+ import { providerName, timeSeriesAggregation } from "./schema";
13
+ import {
14
+ buildBuiltinFullTiers,
15
+ comparePolicyScopes,
16
+ createTimeSeriesPolicyScopeKey,
17
+ DEFAULT_MAINTENANCE_INTERVAL_MS,
18
+ DEFAULT_POLICY_SET_KEY,
19
+ DEFAULT_TIME_SERIES_AGGREGATIONS,
20
+ findTierForAge,
21
+ getBucketEnd,
22
+ getBucketStart,
23
+ getRawTier,
24
+ inferTimeSeriesPolicyScope,
25
+ type NormalizedTimeSeriesTier,
26
+ normalizeAggregations,
27
+ normalizeTimeSeriesPolicyRuleInputs,
28
+ parseDurationInput,
29
+ resolveScopedPolicyRule,
30
+ type TimeSeriesAggregation,
31
+ type TimeSeriesPolicyRuleInput,
32
+ } from "./timeSeriesPolicyUtils";
33
+
34
+ const DAY_MS = 24 * 60 * 60 * 1000;
35
+ const MAX_QUERY_LIMIT = 2000;
36
+ const MAINTENANCE_SETTINGS_KEY = "default";
37
+ const MAINTENANCE_BATCH_SIZE = 20;
38
+ const MAINTENANCE_POINT_BATCH_SIZE = 2000;
39
+ const MAINTENANCE_ROLLUP_BATCH_SIZE = 500;
40
+ const LONG_IDLE_MAINTENANCE_MS = 30 * DAY_MS;
41
+
42
+ type StoredPolicyRule = Doc<"timeSeriesPolicyRules">;
43
+ type StoredRollup = Doc<"timeSeriesRollups">;
44
+ type StoredSeriesState = Doc<"timeSeriesSeriesState">;
45
+
46
+ type DurationInput = string | number;
47
+
48
+ type TimeSeriesPoint = {
49
+ timestamp: number;
50
+ value: number;
51
+ resolution?: "raw" | "rollup";
52
+ bucketMinutes?: number;
53
+ avg?: number;
54
+ min?: number;
55
+ max?: number;
56
+ last?: number;
57
+ count?: number;
58
+ };
59
+
60
+ type StoredPointInput = {
61
+ recordedAt: number;
62
+ value: number;
63
+ externalId?: string;
64
+ };
65
+
66
+ type AggregatedStats = {
67
+ avg: number;
68
+ min: number;
69
+ max: number;
70
+ last: number;
71
+ lastRecordedAt: number;
72
+ count: number;
73
+ };
74
+
75
+ type EffectivePolicy = {
76
+ tiers: NormalizedTimeSeriesTier[];
77
+ sourceKind: "preset" | "default" | "builtin";
78
+ sourceKey: string | null;
79
+ matchedScope: "default" | "global" | "provider" | "series" | "provider_series";
80
+ provider?: string;
81
+ seriesType?: string;
82
+ };
83
+
84
+ type PolicySettingsDoc =
85
+ | Doc<"timeSeriesPolicySettings">
86
+ | {
87
+ key: string;
88
+ maintenanceEnabled: boolean;
89
+ maintenanceIntervalMs: number;
90
+ updatedAt: number;
91
+ };
92
+
93
+ type TimeSeriesReadDb = Pick<DatabaseReader, "get" | "query">;
94
+ type TimeSeriesWriteDb = Pick<DatabaseWriter, "delete" | "get" | "insert" | "patch" | "query">;
95
+ type TimeSeriesMutationContext = Pick<MutationCtx, "db" | "scheduler">;
96
+
97
+ const durationInputValidator = v.union(v.string(), v.number());
98
+
99
+ const tierInputValidator = v.union(
100
+ v.object({
101
+ kind: v.literal("raw"),
102
+ fromAge: durationInputValidator,
103
+ toAge: v.union(durationInputValidator, v.null()),
104
+ }),
105
+ v.object({
106
+ kind: v.literal("rollup"),
107
+ fromAge: durationInputValidator,
108
+ toAge: v.union(durationInputValidator, v.null()),
109
+ bucket: durationInputValidator,
110
+ aggregations: v.optional(v.array(timeSeriesAggregation)),
111
+ }),
112
+ );
113
+
114
+ const policyRuleInputValidator = v.object({
115
+ provider: v.optional(providerName),
116
+ seriesType: v.optional(v.string()),
117
+ tiers: v.array(tierInputValidator),
118
+ });
119
+
120
+ const policyPresetInputValidator = v.object({
121
+ key: v.string(),
122
+ rules: v.array(policyRuleInputValidator),
123
+ });
124
+
125
+ const maintenanceInputValidator = v.object({
126
+ enabled: v.optional(v.boolean()),
127
+ interval: v.optional(durationInputValidator),
128
+ });
129
+
130
+ const timeSeriesPointValidator = v.object({
131
+ timestamp: v.number(),
132
+ value: v.number(),
133
+ resolution: v.optional(v.union(v.literal("raw"), v.literal("rollup"))),
134
+ bucketMinutes: v.optional(v.number()),
135
+ avg: v.optional(v.number()),
136
+ min: v.optional(v.number()),
137
+ max: v.optional(v.number()),
138
+ last: v.optional(v.number()),
139
+ count: v.optional(v.number()),
140
+ });
3
141
 
4
142
  // ---------------------------------------------------------------------------
5
143
  // Queries
6
144
  // ---------------------------------------------------------------------------
7
145
 
8
- /**
9
- * Get time-series data points with cursor-based pagination.
10
- * Uses the by_source_type_time index for efficient range queries.
11
- */
12
146
  export const getTimeSeries = query({
13
147
  args: {
14
148
  dataSourceId: v.id("dataSources"),
@@ -20,56 +154,51 @@ export const getTimeSeries = query({
20
154
  order: v.optional(v.union(v.literal("asc"), v.literal("desc"))),
21
155
  },
22
156
  returns: v.object({
23
- points: v.array(
24
- v.object({
25
- timestamp: v.number(),
26
- value: v.number(),
27
- }),
28
- ),
157
+ points: v.array(timeSeriesPointValidator),
29
158
  nextCursor: v.union(v.string(), v.null()),
30
159
  hasMore: v.boolean(),
31
160
  }),
32
161
  handler: async (ctx, args) => {
33
- const limit = Math.min(args.limit ?? 500, 2000);
162
+ const source = await ctx.db.get(args.dataSourceId);
163
+ if (!source) {
164
+ return { points: [], nextCursor: null, hasMore: false };
165
+ }
166
+
167
+ const limit = Math.min(args.limit ?? 500, MAX_QUERY_LIMIT);
34
168
  const order = args.order ?? "asc";
169
+ const cursor = args.cursor ? Number(args.cursor) : undefined;
35
170
 
36
- const startDate = args.cursor ? Number(args.cursor) : args.startDate;
171
+ const points = await getPolicyAwarePointsForSource(ctx.db, {
172
+ dataSourceId: source._id,
173
+ userId: source.userId,
174
+ provider: source.provider,
175
+ seriesType: args.seriesType,
176
+ startDate: args.startDate,
177
+ endDate: args.endDate,
178
+ limit: limit + 2,
179
+ order,
180
+ });
37
181
 
38
- const results = await ctx.db
39
- .query("dataPoints")
40
- .withIndex("by_source_type_time", (idx) =>
41
- order === "asc"
42
- ? idx
43
- .eq("dataSourceId", args.dataSourceId)
44
- .eq("seriesType", args.seriesType)
45
- .gte("recordedAt", startDate)
46
- .lte("recordedAt", args.endDate)
47
- : idx
48
- .eq("dataSourceId", args.dataSourceId)
49
- .eq("seriesType", args.seriesType)
50
- .gte("recordedAt", args.startDate)
51
- .lte("recordedAt", startDate),
52
- )
53
- .order(order)
54
- .take(limit + 1);
55
-
56
- const hasMore = results.length > limit;
57
- const items = hasMore ? results.slice(0, limit) : results;
58
- const points = items.map((dp) => ({
59
- timestamp: dp.recordedAt,
60
- value: dp.value,
61
- }));
182
+ const filtered =
183
+ cursor === undefined
184
+ ? points
185
+ : points.filter((point) =>
186
+ order === "asc" ? point.timestamp > cursor : point.timestamp < cursor,
187
+ );
188
+
189
+ const hasMore = filtered.length > limit;
190
+ const items = hasMore ? filtered.slice(0, limit) : filtered;
62
191
  const nextCursor =
63
- hasMore && items.length > 0 ? String(items[items.length - 1].recordedAt) : null;
192
+ hasMore && items.length > 0 ? String(items[items.length - 1].timestamp) : null;
64
193
 
65
- return { points, nextCursor, hasMore };
194
+ return {
195
+ points: items,
196
+ nextCursor,
197
+ hasMore,
198
+ };
66
199
  },
67
200
  });
68
201
 
69
- /**
70
- * Get time-series data for a user across all their data sources for a given series type.
71
- * This is a user-facing query that resolves data sources internally.
72
- */
73
202
  export const getTimeSeriesForUser = query({
74
203
  args: {
75
204
  userId: v.string(),
@@ -78,49 +207,36 @@ export const getTimeSeriesForUser = query({
78
207
  endDate: v.number(),
79
208
  limit: v.optional(v.number()),
80
209
  },
81
- returns: v.array(
82
- v.object({
83
- timestamp: v.number(),
84
- value: v.number(),
85
- }),
86
- ),
210
+ returns: v.array(timeSeriesPointValidator),
87
211
  handler: async (ctx, args) => {
88
- const limit = Math.min(args.limit ?? 500, 2000);
212
+ const limit = Math.min(args.limit ?? 500, MAX_QUERY_LIMIT);
89
213
 
90
- // Get all data sources for this user
91
214
  const sources = await ctx.db
92
215
  .query("dataSources")
93
216
  .withIndex("by_user_provider", (idx) => idx.eq("userId", args.userId))
94
217
  .collect();
95
218
 
96
- // Collect data from all sources (merge and sort)
97
- const allPoints: { timestamp: number; value: number }[] = [];
219
+ const points: TimeSeriesPoint[] = [];
98
220
  for (const source of sources) {
99
- const points = await ctx.db
100
- .query("dataPoints")
101
- .withIndex("by_source_type_time", (idx) =>
102
- idx
103
- .eq("dataSourceId", source._id)
104
- .eq("seriesType", args.seriesType)
105
- .gte("recordedAt", args.startDate)
106
- .lte("recordedAt", args.endDate),
107
- )
108
- .take(limit);
109
-
110
- for (const dp of points) {
111
- allPoints.push({ timestamp: dp.recordedAt, value: dp.value });
112
- }
221
+ points.push(
222
+ ...(await getPolicyAwarePointsForSource(ctx.db, {
223
+ dataSourceId: source._id,
224
+ userId: source.userId,
225
+ provider: source.provider,
226
+ seriesType: args.seriesType,
227
+ startDate: args.startDate,
228
+ endDate: args.endDate,
229
+ limit,
230
+ order: "asc",
231
+ })),
232
+ );
113
233
  }
114
234
 
115
- // Sort by timestamp and limit
116
- allPoints.sort((a, b) => a.timestamp - b.timestamp);
117
- return allPoints.slice(0, limit);
235
+ points.sort((a, b) => a.timestamp - b.timestamp);
236
+ return points.slice(0, limit);
118
237
  },
119
238
  });
120
239
 
121
- /**
122
- * Get the latest data point for a user and series type.
123
- */
124
240
  export const getLatestDataPoint = query({
125
241
  args: {
126
242
  userId: v.string(),
@@ -143,7 +259,7 @@ export const getLatestDataPoint = query({
143
259
  let latest: { timestamp: number; value: number; provider: string } | null = null;
144
260
 
145
261
  for (const source of sources) {
146
- const point = await ctx.db
262
+ const rawPoint = await ctx.db
147
263
  .query("dataPoints")
148
264
  .withIndex("by_source_type_time", (idx) =>
149
265
  idx.eq("dataSourceId", source._id).eq("seriesType", args.seriesType),
@@ -151,12 +267,39 @@ export const getLatestDataPoint = query({
151
267
  .order("desc")
152
268
  .first();
153
269
 
154
- if (point && (latest === null || point.recordedAt > latest.timestamp)) {
155
- latest = {
156
- timestamp: point.recordedAt,
157
- value: point.value,
158
- provider: source.provider,
159
- };
270
+ const rollupPoint = await ctx.db
271
+ .query("timeSeriesRollups")
272
+ .withIndex("by_source_type_bucket", (idx) =>
273
+ idx.eq("dataSourceId", source._id).eq("seriesType", args.seriesType),
274
+ )
275
+ .order("desc")
276
+ .first();
277
+
278
+ const rawCandidate = rawPoint
279
+ ? {
280
+ timestamp: rawPoint.recordedAt,
281
+ value: rawPoint.value,
282
+ provider: source.provider,
283
+ }
284
+ : null;
285
+ const rollupCandidate = rollupPoint
286
+ ? {
287
+ timestamp: rollupPoint.lastRecordedAt,
288
+ value: rollupPoint.last,
289
+ provider: source.provider,
290
+ }
291
+ : null;
292
+ const candidate =
293
+ rawCandidate === null
294
+ ? rollupCandidate
295
+ : rollupCandidate === null
296
+ ? rawCandidate
297
+ : rawCandidate.timestamp >= rollupCandidate.timestamp
298
+ ? rawCandidate
299
+ : rollupCandidate;
300
+
301
+ if (candidate && (latest === null || candidate.timestamp > latest.timestamp)) {
302
+ latest = candidate;
160
303
  }
161
304
  }
162
305
 
@@ -164,34 +307,95 @@ export const getLatestDataPoint = query({
164
307
  },
165
308
  });
166
309
 
167
- /**
168
- * Get all available series types for a user (i.e., types that have at least one data point).
169
- */
170
310
  export const getAvailableSeriesTypes = query({
171
311
  args: { userId: v.string() },
172
312
  returns: v.array(v.string()),
173
313
  handler: async (ctx, args) => {
174
- const sources = await ctx.db
175
- .query("dataSources")
176
- .withIndex("by_user_provider", (idx) => idx.eq("userId", args.userId))
314
+ const states = await ctx.db
315
+ .query("timeSeriesSeriesState")
316
+ .withIndex("by_user", (idx) => idx.eq("userId", args.userId))
177
317
  .collect();
178
318
 
179
- const types = new Set<string>();
319
+ return Array.from(new Set(states.map((state) => state.seriesType))).sort();
320
+ },
321
+ });
180
322
 
181
- for (const source of sources) {
182
- // Sample a few data points to discover types
183
- // This is efficient because we only need one per type
184
- const points = await ctx.db
185
- .query("dataPoints")
186
- .withIndex("by_source_type_time", (idx) => idx.eq("dataSourceId", source._id))
187
- .take(200);
323
+ export const getTimeSeriesPolicyConfiguration = query({
324
+ args: {},
325
+ returns: v.any(),
326
+ handler: async (ctx) => {
327
+ const settings = await getTimeSeriesPolicySettings(ctx.db);
328
+ const rules = await ctx.db.query("timeSeriesPolicyRules").collect();
188
329
 
189
- for (const point of points) {
190
- types.add(point.seriesType);
191
- }
330
+ return {
331
+ maintenance: {
332
+ enabled: settings.maintenanceEnabled,
333
+ intervalMs: settings.maintenanceIntervalMs,
334
+ },
335
+ defaultRules: formatPolicyRulesForResponse(
336
+ rules.filter(
337
+ (rule) =>
338
+ rule.policySetKind === "default" && rule.policySetKey === DEFAULT_POLICY_SET_KEY,
339
+ ),
340
+ ),
341
+ presets: groupPolicyRulesByPreset(rules.filter((rule) => rule.policySetKind === "preset")),
342
+ };
343
+ },
344
+ });
345
+
346
+ export const getUserTimeSeriesPolicyPreset = query({
347
+ args: {
348
+ userId: v.string(),
349
+ },
350
+ returns: v.union(
351
+ v.object({
352
+ userId: v.string(),
353
+ presetKey: v.string(),
354
+ updatedAt: v.number(),
355
+ }),
356
+ v.null(),
357
+ ),
358
+ handler: async (ctx, args) => {
359
+ const assignment = await ctx.db
360
+ .query("timeSeriesPolicyAssignments")
361
+ .withIndex("by_user", (idx) => idx.eq("userId", args.userId))
362
+ .first();
363
+
364
+ if (!assignment) {
365
+ return null;
192
366
  }
193
367
 
194
- return Array.from(types).sort();
368
+ return {
369
+ userId: assignment.userId,
370
+ presetKey: assignment.presetKey,
371
+ updatedAt: assignment.updatedAt,
372
+ };
373
+ },
374
+ });
375
+
376
+ export const getEffectiveTimeSeriesPolicy = query({
377
+ args: {
378
+ userId: v.string(),
379
+ provider: providerName,
380
+ seriesType: v.string(),
381
+ },
382
+ returns: v.any(),
383
+ handler: async (ctx, args) => {
384
+ const effective = await resolveEffectivePolicy(
385
+ ctx.db,
386
+ args.userId,
387
+ args.provider,
388
+ args.seriesType,
389
+ );
390
+
391
+ return {
392
+ provider: args.provider,
393
+ seriesType: args.seriesType,
394
+ sourceKind: effective.sourceKind,
395
+ sourceKey: effective.sourceKey,
396
+ matchedScope: effective.matchedScope,
397
+ tiers: effective.tiers.map(formatTierForResponse),
398
+ };
195
399
  },
196
400
  });
197
401
 
@@ -199,9 +403,130 @@ export const getAvailableSeriesTypes = query({
199
403
  // Mutations
200
404
  // ---------------------------------------------------------------------------
201
405
 
202
- /**
203
- * Store a single data point. Deduplicates by (source, type, time).
204
- */
406
+ export const replaceTimeSeriesPolicyConfiguration = mutation({
407
+ args: {
408
+ defaultRules: v.array(policyRuleInputValidator),
409
+ presets: v.optional(v.array(policyPresetInputValidator)),
410
+ maintenance: v.optional(maintenanceInputValidator),
411
+ },
412
+ returns: v.object({
413
+ defaultRulesStored: v.number(),
414
+ presetsStored: v.number(),
415
+ }),
416
+ handler: async (ctx, args) => {
417
+ const normalizedDefaultRules = normalizeTimeSeriesPolicyRuleInputs(
418
+ args.defaultRules as TimeSeriesPolicyRuleInput[],
419
+ );
420
+ const presetKeys = new Set<string>();
421
+ const normalizedPresets = (args.presets ?? []).map((preset) => {
422
+ if (!preset.key.trim()) {
423
+ throw new Error("Preset keys must not be empty");
424
+ }
425
+ if (preset.key === DEFAULT_POLICY_SET_KEY) {
426
+ throw new Error(`Preset key "${DEFAULT_POLICY_SET_KEY}" is reserved`);
427
+ }
428
+ if (presetKeys.has(preset.key)) {
429
+ throw new Error(`Duplicate time-series policy preset key "${preset.key}"`);
430
+ }
431
+ presetKeys.add(preset.key);
432
+
433
+ return {
434
+ key: preset.key,
435
+ rules: normalizeTimeSeriesPolicyRuleInputs(preset.rules as TimeSeriesPolicyRuleInput[]),
436
+ };
437
+ });
438
+
439
+ const existingRules = await ctx.db.query("timeSeriesPolicyRules").collect();
440
+ for (const rule of existingRules) {
441
+ await ctx.db.delete(rule._id);
442
+ }
443
+
444
+ const updatedAt = Date.now();
445
+ for (const rule of normalizedDefaultRules) {
446
+ await ctx.db.insert("timeSeriesPolicyRules", {
447
+ policySetKind: "default",
448
+ policySetKey: DEFAULT_POLICY_SET_KEY,
449
+ scopeKey: createTimeSeriesPolicyScopeKey(rule.provider, rule.seriesType),
450
+ provider: rule.provider,
451
+ seriesType: rule.seriesType,
452
+ tiers: rule.tiers,
453
+ updatedAt,
454
+ });
455
+ }
456
+
457
+ for (const preset of normalizedPresets) {
458
+ for (const rule of preset.rules) {
459
+ await ctx.db.insert("timeSeriesPolicyRules", {
460
+ policySetKind: "preset",
461
+ policySetKey: preset.key,
462
+ scopeKey: createTimeSeriesPolicyScopeKey(rule.provider, rule.seriesType),
463
+ provider: rule.provider,
464
+ seriesType: rule.seriesType,
465
+ tiers: rule.tiers,
466
+ updatedAt,
467
+ });
468
+ }
469
+ }
470
+
471
+ await upsertTimeSeriesPolicySettings(ctx, args.maintenance);
472
+ await markAllSeriesStateDue(ctx, updatedAt);
473
+ await ensureTimeSeriesMaintenanceScheduled(ctx);
474
+
475
+ return {
476
+ defaultRulesStored: normalizedDefaultRules.length,
477
+ presetsStored: normalizedPresets.reduce((sum, preset) => sum + preset.rules.length, 0),
478
+ };
479
+ },
480
+ });
481
+
482
+ export const setUserTimeSeriesPolicyPreset = mutation({
483
+ args: {
484
+ userId: v.string(),
485
+ presetKey: v.union(v.string(), v.null()),
486
+ },
487
+ returns: v.null(),
488
+ handler: async (ctx, args) => {
489
+ const existing = await ctx.db
490
+ .query("timeSeriesPolicyAssignments")
491
+ .withIndex("by_user", (idx) => idx.eq("userId", args.userId))
492
+ .first();
493
+
494
+ if (args.presetKey === null) {
495
+ if (existing) {
496
+ await ctx.db.delete(existing._id);
497
+ }
498
+ await markSeriesStateDueForUser(ctx, args.userId, Date.now());
499
+ await ensureTimeSeriesMaintenanceScheduled(ctx);
500
+ return null;
501
+ }
502
+
503
+ const presetKey = args.presetKey;
504
+ const presetRules = await ctx.db
505
+ .query("timeSeriesPolicyRules")
506
+ .withIndex("by_set", (idx) => idx.eq("policySetKind", "preset").eq("policySetKey", presetKey))
507
+ .take(1);
508
+ if (presetRules.length === 0) {
509
+ throw new Error(`Time-series policy preset "${presetKey}" does not exist`);
510
+ }
511
+
512
+ const patch = {
513
+ userId: args.userId,
514
+ presetKey,
515
+ updatedAt: Date.now(),
516
+ };
517
+
518
+ if (existing) {
519
+ await ctx.db.patch(existing._id, patch);
520
+ } else {
521
+ await ctx.db.insert("timeSeriesPolicyAssignments", patch);
522
+ }
523
+
524
+ await markSeriesStateDueForUser(ctx, args.userId, Date.now());
525
+ await ensureTimeSeriesMaintenanceScheduled(ctx);
526
+ return null;
527
+ },
528
+ });
529
+
205
530
  export const storeDataPoint = internalMutation({
206
531
  args: {
207
532
  dataSourceId: v.id("dataSources"),
@@ -210,32 +535,24 @@ export const storeDataPoint = internalMutation({
210
535
  value: v.number(),
211
536
  externalId: v.optional(v.string()),
212
537
  },
213
- returns: v.id("dataPoints"),
538
+ returns: v.union(v.id("dataPoints"), v.null()),
214
539
  handler: async (ctx, args) => {
215
- // Deduplicate by (source, type, time)
216
- const existing = await ctx.db
217
- .query("dataPoints")
218
- .withIndex("by_source_type_time", (idx) =>
219
- idx
220
- .eq("dataSourceId", args.dataSourceId)
221
- .eq("seriesType", args.seriesType)
222
- .eq("recordedAt", args.recordedAt),
223
- )
224
- .first();
225
-
226
- if (existing) {
227
- await ctx.db.patch(existing._id, { value: args.value });
228
- return existing._id;
229
- }
540
+ const result = await storePointsWithPolicy(ctx, {
541
+ dataSourceId: args.dataSourceId,
542
+ seriesType: args.seriesType,
543
+ points: [
544
+ {
545
+ recordedAt: args.recordedAt,
546
+ value: args.value,
547
+ externalId: args.externalId,
548
+ },
549
+ ],
550
+ });
230
551
 
231
- return await ctx.db.insert("dataPoints", args);
552
+ return result.lastRawId;
232
553
  },
233
554
  });
234
555
 
235
- /**
236
- * Store a batch of data points. Used by sync workflows.
237
- * Must complete within 1 second — caller is responsible for batch sizing.
238
- */
239
556
  export const storeBatch = internalMutation({
240
557
  args: {
241
558
  dataSourceId: v.id("dataSources"),
@@ -250,56 +567,1110 @@ export const storeBatch = internalMutation({
250
567
  },
251
568
  returns: v.number(),
252
569
  handler: async (ctx, args) => {
253
- let count = 0;
254
- for (const point of args.points) {
255
- // Deduplicate by (source, type, time)
256
- const existing = await ctx.db
257
- .query("dataPoints")
258
- .withIndex("by_source_type_time", (idx) =>
259
- idx
260
- .eq("dataSourceId", args.dataSourceId)
261
- .eq("seriesType", args.seriesType)
262
- .eq("recordedAt", point.recordedAt),
263
- )
264
- .first();
570
+ const result = await storePointsWithPolicy(ctx, args);
571
+ return result.processedCount;
572
+ },
573
+ });
265
574
 
266
- if (existing) {
267
- await ctx.db.patch(existing._id, { value: point.value });
268
- } else {
269
- await ctx.db.insert("dataPoints", {
270
- dataSourceId: args.dataSourceId,
271
- seriesType: args.seriesType,
272
- recordedAt: point.recordedAt,
273
- value: point.value,
274
- externalId: point.externalId,
275
- });
575
+ export const runTimeSeriesMaintenance = internalMutation({
576
+ args: {},
577
+ returns: v.null(),
578
+ handler: async (ctx) => {
579
+ const settingsDoc = await ensureTimeSeriesPolicySettingsDoc(ctx.db);
580
+ const now = Date.now();
581
+
582
+ if (!settingsDoc.maintenanceEnabled) {
583
+ await ctx.db.patch(settingsDoc._id, {
584
+ scheduledAt: undefined,
585
+ lastRunAt: now,
586
+ lastError: undefined,
587
+ updatedAt: now,
588
+ });
589
+ return null;
590
+ }
591
+
592
+ await ctx.db.patch(settingsDoc._id, {
593
+ scheduledAt: undefined,
594
+ lastRunAt: now,
595
+ lastError: undefined,
596
+ updatedAt: now,
597
+ });
598
+
599
+ let lastError: string | undefined;
600
+ const dueStates = await ctx.db
601
+ .query("timeSeriesSeriesState")
602
+ .withIndex("by_next_maintenance", (idx) => idx.lte("nextMaintenanceAt", now))
603
+ .take(MAINTENANCE_BATCH_SIZE);
604
+
605
+ for (const state of dueStates) {
606
+ try {
607
+ await maintainSeriesState(ctx.db, state, now);
608
+ } catch (error) {
609
+ lastError = error instanceof Error ? error.message : String(error);
276
610
  }
277
- count++;
278
611
  }
279
- return count;
612
+
613
+ const refreshed = await ensureTimeSeriesPolicySettingsDoc(ctx.db);
614
+ const hasBacklog = dueStates.length === MAINTENANCE_BATCH_SIZE;
615
+ const delayMs = hasBacklog
616
+ ? Math.min(refreshed.maintenanceIntervalMs, 60 * 1000)
617
+ : refreshed.maintenanceIntervalMs;
618
+ const scheduledAt = now + delayMs;
619
+
620
+ await ctx.scheduler.runAfter(delayMs, internal.dataPoints.runTimeSeriesMaintenance, {});
621
+ await ctx.db.patch(refreshed._id, {
622
+ scheduledAt,
623
+ lastRunAt: now,
624
+ lastError,
625
+ updatedAt: now,
626
+ });
627
+
628
+ return null;
280
629
  },
281
630
  });
282
631
 
283
- /**
284
- * Delete all data points for a data source. Used during user/connection cleanup.
285
- */
286
632
  export const deleteByDataSource = internalMutation({
287
633
  args: { dataSourceId: v.id("dataSources") },
634
+ returns: v.null(),
288
635
  handler: async (ctx, args) => {
289
- // Delete in batches to avoid hitting limits
290
- let batch = await ctx.db
636
+ let rawBatch = await ctx.db
291
637
  .query("dataPoints")
292
638
  .withIndex("by_source_type_time", (idx) => idx.eq("dataSourceId", args.dataSourceId))
293
639
  .take(1000);
294
640
 
295
- while (batch.length > 0) {
296
- for (const dp of batch) {
297
- await ctx.db.delete(dp._id);
641
+ while (rawBatch.length > 0) {
642
+ for (const point of rawBatch) {
643
+ await ctx.db.delete(point._id);
298
644
  }
299
- batch = await ctx.db
645
+ rawBatch = await ctx.db
300
646
  .query("dataPoints")
301
647
  .withIndex("by_source_type_time", (idx) => idx.eq("dataSourceId", args.dataSourceId))
302
648
  .take(1000);
303
649
  }
650
+
651
+ let rollupBatch = await ctx.db
652
+ .query("timeSeriesRollups")
653
+ .withIndex("by_source_type_bucket", (idx) => idx.eq("dataSourceId", args.dataSourceId))
654
+ .take(1000);
655
+
656
+ while (rollupBatch.length > 0) {
657
+ for (const rollup of rollupBatch) {
658
+ await ctx.db.delete(rollup._id);
659
+ }
660
+ rollupBatch = await ctx.db
661
+ .query("timeSeriesRollups")
662
+ .withIndex("by_source_type_bucket", (idx) => idx.eq("dataSourceId", args.dataSourceId))
663
+ .take(1000);
664
+ }
665
+
666
+ const state = await ctx.db
667
+ .query("timeSeriesSeriesState")
668
+ .withIndex("by_source_series", (idx) => idx.eq("dataSourceId", args.dataSourceId))
669
+ .collect();
670
+ for (const doc of state) {
671
+ await ctx.db.delete(doc._id);
672
+ }
673
+
674
+ return null;
304
675
  },
305
676
  });
677
+
678
+ // ---------------------------------------------------------------------------
679
+ // Policy helpers
680
+ // ---------------------------------------------------------------------------
681
+
682
+ async function getTimeSeriesPolicySettings(db: {
683
+ query: DatabaseReader["query"];
684
+ }): Promise<PolicySettingsDoc> {
685
+ const existing = await db
686
+ .query("timeSeriesPolicySettings")
687
+ .withIndex("by_key", (idx) => idx.eq("key", MAINTENANCE_SETTINGS_KEY))
688
+ .first();
689
+
690
+ return (
691
+ existing ?? {
692
+ key: MAINTENANCE_SETTINGS_KEY,
693
+ maintenanceEnabled: true,
694
+ maintenanceIntervalMs: DEFAULT_MAINTENANCE_INTERVAL_MS,
695
+ updatedAt: 0,
696
+ }
697
+ );
698
+ }
699
+
700
+ async function ensureTimeSeriesPolicySettingsDoc(db: {
701
+ query: DatabaseWriter["query"];
702
+ insert: DatabaseWriter["insert"];
703
+ get: DatabaseWriter["get"];
704
+ }) {
705
+ const existing = await db
706
+ .query("timeSeriesPolicySettings")
707
+ .withIndex("by_key", (idx) => idx.eq("key", MAINTENANCE_SETTINGS_KEY))
708
+ .first();
709
+ if (existing) {
710
+ return existing;
711
+ }
712
+
713
+ const id = await db.insert("timeSeriesPolicySettings", {
714
+ key: MAINTENANCE_SETTINGS_KEY,
715
+ maintenanceEnabled: true,
716
+ maintenanceIntervalMs: DEFAULT_MAINTENANCE_INTERVAL_MS,
717
+ updatedAt: Date.now(),
718
+ });
719
+ const inserted = await db.get(id);
720
+ if (!inserted) {
721
+ throw new Error("Failed to create time-series policy settings");
722
+ }
723
+ return inserted;
724
+ }
725
+
726
+ async function upsertTimeSeriesPolicySettings(
727
+ ctx: {
728
+ db: Pick<DatabaseWriter, "get" | "insert" | "patch" | "query">;
729
+ },
730
+ maintenance?: {
731
+ enabled?: boolean;
732
+ interval?: DurationInput;
733
+ },
734
+ ) {
735
+ const existing = await ensureTimeSeriesPolicySettingsDoc(ctx.db);
736
+ const patch = {
737
+ maintenanceEnabled: maintenance?.enabled ?? existing.maintenanceEnabled,
738
+ maintenanceIntervalMs:
739
+ maintenance?.interval !== undefined
740
+ ? parseDurationInput(maintenance.interval, "maintenance.interval")
741
+ : existing.maintenanceIntervalMs,
742
+ updatedAt: Date.now(),
743
+ };
744
+
745
+ await ctx.db.patch(existing._id, patch);
746
+ }
747
+
748
+ async function resolveEffectivePolicy(
749
+ db: {
750
+ query: TimeSeriesReadDb["query"];
751
+ },
752
+ userId: string,
753
+ provider: string,
754
+ seriesType: string,
755
+ ): Promise<EffectivePolicy> {
756
+ const assignment = await db
757
+ .query("timeSeriesPolicyAssignments")
758
+ .withIndex("by_user", (idx) => idx.eq("userId", userId))
759
+ .first();
760
+
761
+ if (assignment) {
762
+ const presetRules = (await db
763
+ .query("timeSeriesPolicyRules")
764
+ .withIndex("by_set", (idx) =>
765
+ idx.eq("policySetKind", "preset").eq("policySetKey", assignment.presetKey),
766
+ )
767
+ .collect()) as StoredPolicyRule[];
768
+ const presetMatch = resolveScopedPolicyRule(presetRules, provider, seriesType);
769
+
770
+ if (presetMatch) {
771
+ return {
772
+ provider: presetMatch.provider,
773
+ seriesType: presetMatch.seriesType,
774
+ tiers: presetMatch.tiers,
775
+ sourceKind: "preset",
776
+ sourceKey: assignment.presetKey,
777
+ matchedScope: inferTimeSeriesPolicyScope(presetMatch.provider, presetMatch.seriesType),
778
+ };
779
+ }
780
+ }
781
+
782
+ const defaultRules = (await db
783
+ .query("timeSeriesPolicyRules")
784
+ .withIndex("by_set", (idx) =>
785
+ idx.eq("policySetKind", "default").eq("policySetKey", DEFAULT_POLICY_SET_KEY),
786
+ )
787
+ .collect()) as StoredPolicyRule[];
788
+ const defaultMatch = resolveScopedPolicyRule(defaultRules, provider, seriesType);
789
+
790
+ if (defaultMatch) {
791
+ return {
792
+ provider: defaultMatch.provider,
793
+ seriesType: defaultMatch.seriesType,
794
+ tiers: defaultMatch.tiers,
795
+ sourceKind: "default",
796
+ sourceKey: DEFAULT_POLICY_SET_KEY,
797
+ matchedScope: inferTimeSeriesPolicyScope(defaultMatch.provider, defaultMatch.seriesType),
798
+ };
799
+ }
800
+
801
+ return {
802
+ tiers: buildBuiltinFullTiers(),
803
+ sourceKind: "builtin",
804
+ sourceKey: null,
805
+ matchedScope: "default",
806
+ };
807
+ }
808
+
809
+ function formatTierForResponse(tier: NormalizedTimeSeriesTier) {
810
+ if (tier.kind === "raw") {
811
+ return {
812
+ kind: "raw",
813
+ fromAgeMs: tier.fromAgeMs,
814
+ toAgeMs: tier.toAgeMs,
815
+ };
816
+ }
817
+
818
+ return {
819
+ kind: "rollup",
820
+ fromAgeMs: tier.fromAgeMs,
821
+ toAgeMs: tier.toAgeMs,
822
+ bucketMs: tier.bucketMs,
823
+ aggregations: tier.aggregations,
824
+ };
825
+ }
826
+
827
+ function formatPolicyRulesForResponse(rules: StoredPolicyRule[]) {
828
+ return rules
829
+ .map((rule) => ({
830
+ _id: rule._id,
831
+ provider: rule.provider,
832
+ seriesType: rule.seriesType,
833
+ scope: inferTimeSeriesPolicyScope(rule.provider, rule.seriesType),
834
+ policySetKind: rule.policySetKind,
835
+ policySetKey: rule.policySetKey,
836
+ tiers: rule.tiers.map(formatTierForResponse),
837
+ updatedAt: rule.updatedAt,
838
+ }))
839
+ .sort(comparePolicyScopes);
840
+ }
841
+
842
+ function groupPolicyRulesByPreset(rules: StoredPolicyRule[]) {
843
+ const grouped = new Map<string, StoredPolicyRule[]>();
844
+ for (const rule of rules) {
845
+ const existing = grouped.get(rule.policySetKey) ?? [];
846
+ existing.push(rule);
847
+ grouped.set(rule.policySetKey, existing);
848
+ }
849
+
850
+ return Array.from(grouped.entries())
851
+ .sort((a, b) => a[0].localeCompare(b[0]))
852
+ .map(([key, groupedRules]) => ({
853
+ key,
854
+ rules: formatPolicyRulesForResponse(groupedRules),
855
+ }));
856
+ }
857
+
858
+ function policyRequiresMaintenance(tiers: NormalizedTimeSeriesTier[]) {
859
+ const rawTier = getRawTier(tiers);
860
+ if (!rawTier) {
861
+ return true;
862
+ }
863
+ if (tiers.length > 1) {
864
+ return true;
865
+ }
866
+ return rawTier.toAgeMs !== null;
867
+ }
868
+
869
+ // ---------------------------------------------------------------------------
870
+ // Write path
871
+ // ---------------------------------------------------------------------------
872
+
873
+ async function storePointsWithPolicy(
874
+ ctx: TimeSeriesMutationContext,
875
+ args: {
876
+ dataSourceId: Id<"dataSources">;
877
+ seriesType: string;
878
+ points: StoredPointInput[];
879
+ },
880
+ ) {
881
+ const dataSource = await ctx.db.get(args.dataSourceId);
882
+ if (!dataSource) {
883
+ throw new Error(`Data source ${args.dataSourceId} not found`);
884
+ }
885
+
886
+ const effectivePolicy = await resolveEffectivePolicy(
887
+ ctx.db,
888
+ dataSource.userId,
889
+ dataSource.provider,
890
+ args.seriesType,
891
+ );
892
+ const settings = await getTimeSeriesPolicySettings(ctx.db);
893
+ const now = Date.now();
894
+ const points = dedupeIncomingPoints(args.points);
895
+
896
+ const rawPoints: StoredPointInput[] = [];
897
+ const rollupGroups = new Map<
898
+ string,
899
+ { bucketMs: number; bucketStart: number; stats: AggregatedStats }
900
+ >();
901
+
902
+ for (const point of points) {
903
+ const ageMs = Math.max(0, now - point.recordedAt);
904
+ const destinationTier = findTierForAge(effectivePolicy.tiers, ageMs);
905
+ if (!destinationTier) {
906
+ continue;
907
+ }
908
+
909
+ if (destinationTier.kind === "raw") {
910
+ rawPoints.push(point);
911
+ continue;
912
+ }
913
+
914
+ addRawPointToRollupGroup(rollupGroups, point, destinationTier.bucketMs);
915
+ }
916
+
917
+ let lastRawId: Id<"dataPoints"> | null = null;
918
+ if (rawPoints.length > 0) {
919
+ lastRawId = await upsertRawPoints(ctx.db, args.dataSourceId, args.seriesType, rawPoints);
920
+ }
921
+ if (rollupGroups.size > 0) {
922
+ await upsertRollupStatsGroups(
923
+ ctx.db,
924
+ args.dataSourceId,
925
+ args.seriesType,
926
+ rollupGroups.values(),
927
+ );
928
+ }
929
+
930
+ await upsertSeriesState(ctx.db, {
931
+ dataSourceId: dataSource._id,
932
+ connectionId: dataSource.connectionId,
933
+ userId: dataSource.userId,
934
+ provider: dataSource.provider,
935
+ seriesType: args.seriesType,
936
+ latestRecordedAt: points.length > 0 ? points[points.length - 1].recordedAt : now,
937
+ nextMaintenanceAt: policyRequiresMaintenance(effectivePolicy.tiers)
938
+ ? now + settings.maintenanceIntervalMs
939
+ : now + LONG_IDLE_MAINTENANCE_MS,
940
+ });
941
+
942
+ if (policyRequiresMaintenance(effectivePolicy.tiers)) {
943
+ await maintainSeriesState(
944
+ ctx.db,
945
+ {
946
+ dataSourceId: dataSource._id,
947
+ connectionId: dataSource.connectionId,
948
+ userId: dataSource.userId,
949
+ provider: dataSource.provider,
950
+ seriesType: args.seriesType,
951
+ },
952
+ now,
953
+ );
954
+ await ensureTimeSeriesMaintenanceScheduled(ctx);
955
+ }
956
+
957
+ return {
958
+ processedCount: points.length,
959
+ lastRawId,
960
+ };
961
+ }
962
+
963
+ function dedupeIncomingPoints(points: StoredPointInput[]) {
964
+ const deduped = new Map<number, StoredPointInput>();
965
+ for (const point of points) {
966
+ deduped.set(point.recordedAt, point);
967
+ }
968
+ return Array.from(deduped.values()).sort((a, b) => a.recordedAt - b.recordedAt);
969
+ }
970
+
971
+ async function upsertRawPoints(
972
+ db: TimeSeriesWriteDb,
973
+ dataSourceId: Id<"dataSources">,
974
+ seriesType: string,
975
+ points: StoredPointInput[],
976
+ ) {
977
+ let lastId: Id<"dataPoints"> | null = null;
978
+
979
+ for (const point of points) {
980
+ const existing = await db
981
+ .query("dataPoints")
982
+ .withIndex("by_source_type_time", (idx) =>
983
+ idx
984
+ .eq("dataSourceId", dataSourceId)
985
+ .eq("seriesType", seriesType)
986
+ .eq("recordedAt", point.recordedAt),
987
+ )
988
+ .first();
989
+
990
+ if (existing) {
991
+ await db.patch(existing._id, {
992
+ value: point.value,
993
+ externalId: point.externalId,
994
+ });
995
+ lastId = existing._id;
996
+ } else {
997
+ lastId = await db.insert("dataPoints", {
998
+ dataSourceId,
999
+ seriesType,
1000
+ recordedAt: point.recordedAt,
1001
+ value: point.value,
1002
+ externalId: point.externalId,
1003
+ });
1004
+ }
1005
+ }
1006
+
1007
+ return lastId;
1008
+ }
1009
+
1010
+ function addRawPointToRollupGroup(
1011
+ groups: Map<string, { bucketMs: number; bucketStart: number; stats: AggregatedStats }>,
1012
+ point: StoredPointInput,
1013
+ bucketMs: number,
1014
+ ) {
1015
+ const bucketStart = getBucketStart(point.recordedAt, bucketMs);
1016
+ const key = `${bucketMs}:${bucketStart}`;
1017
+ const existing = groups.get(key);
1018
+ const stats = aggregateRawPoints([
1019
+ {
1020
+ recordedAt: point.recordedAt,
1021
+ value: point.value,
1022
+ },
1023
+ ]);
1024
+
1025
+ if (existing) {
1026
+ existing.stats = combineAggregatedStats([existing.stats, stats]);
1027
+ } else {
1028
+ groups.set(key, {
1029
+ bucketMs,
1030
+ bucketStart,
1031
+ stats,
1032
+ });
1033
+ }
1034
+ }
1035
+
1036
+ async function upsertRollupStatsGroups(
1037
+ db: TimeSeriesWriteDb,
1038
+ dataSourceId: Id<"dataSources">,
1039
+ seriesType: string,
1040
+ groups: Iterable<{ bucketMs: number; bucketStart: number; stats: AggregatedStats }>,
1041
+ ) {
1042
+ for (const group of groups) {
1043
+ const existing = await db
1044
+ .query("timeSeriesRollups")
1045
+ .withIndex("by_source_type_bucket_size", (idx) =>
1046
+ idx
1047
+ .eq("dataSourceId", dataSourceId)
1048
+ .eq("seriesType", seriesType)
1049
+ .eq("bucketMs", group.bucketMs)
1050
+ .eq("bucketStart", group.bucketStart),
1051
+ )
1052
+ .first();
1053
+
1054
+ const combined = existing
1055
+ ? combineAggregatedStats([rollupDocToStats(existing), group.stats])
1056
+ : group.stats;
1057
+ const patch = {
1058
+ dataSourceId,
1059
+ seriesType,
1060
+ bucketMs: group.bucketMs,
1061
+ bucketStart: group.bucketStart,
1062
+ bucketEnd: getBucketEnd(group.bucketStart, group.bucketMs),
1063
+ avg: combined.avg,
1064
+ min: combined.min,
1065
+ max: combined.max,
1066
+ last: combined.last,
1067
+ lastRecordedAt: combined.lastRecordedAt,
1068
+ count: combined.count,
1069
+ updatedAt: Date.now(),
1070
+ };
1071
+
1072
+ if (existing) {
1073
+ await db.patch(existing._id, patch);
1074
+ } else {
1075
+ await db.insert("timeSeriesRollups", patch);
1076
+ }
1077
+ }
1078
+ }
1079
+
1080
+ async function upsertSeriesState(
1081
+ db: TimeSeriesWriteDb,
1082
+ args: {
1083
+ dataSourceId: Id<"dataSources">;
1084
+ connectionId?: Id<"connections">;
1085
+ userId: string;
1086
+ provider: Doc<"dataSources">["provider"];
1087
+ seriesType: string;
1088
+ latestRecordedAt: number;
1089
+ nextMaintenanceAt: number;
1090
+ },
1091
+ ) {
1092
+ const existing = await db
1093
+ .query("timeSeriesSeriesState")
1094
+ .withIndex("by_source_series", (idx) =>
1095
+ idx.eq("dataSourceId", args.dataSourceId).eq("seriesType", args.seriesType),
1096
+ )
1097
+ .first();
1098
+
1099
+ const patch = {
1100
+ dataSourceId: args.dataSourceId,
1101
+ connectionId: args.connectionId,
1102
+ userId: args.userId,
1103
+ provider: args.provider,
1104
+ seriesType: args.seriesType,
1105
+ latestRecordedAt: existing
1106
+ ? Math.max(existing.latestRecordedAt, args.latestRecordedAt)
1107
+ : args.latestRecordedAt,
1108
+ lastIngestedAt: Date.now(),
1109
+ nextMaintenanceAt: args.nextMaintenanceAt,
1110
+ updatedAt: Date.now(),
1111
+ };
1112
+
1113
+ if (existing) {
1114
+ await db.patch(existing._id, patch);
1115
+ return existing._id;
1116
+ }
1117
+
1118
+ return await db.insert("timeSeriesSeriesState", patch);
1119
+ }
1120
+
1121
+ // ---------------------------------------------------------------------------
1122
+ // Maintenance
1123
+ // ---------------------------------------------------------------------------
1124
+
1125
+ async function ensureTimeSeriesMaintenanceScheduled(ctx: TimeSeriesMutationContext) {
1126
+ const settings = await ensureTimeSeriesPolicySettingsDoc(ctx.db);
1127
+ if (!settings.maintenanceEnabled) {
1128
+ return;
1129
+ }
1130
+
1131
+ const now = Date.now();
1132
+ if (settings.scheduledAt !== undefined && settings.scheduledAt > now) {
1133
+ return;
1134
+ }
1135
+
1136
+ const delayMs = settings.maintenanceIntervalMs;
1137
+ const scheduledAt = now + delayMs;
1138
+ await ctx.scheduler.runAfter(delayMs, internal.dataPoints.runTimeSeriesMaintenance, {});
1139
+ await ctx.db.patch(settings._id, {
1140
+ scheduledAt,
1141
+ updatedAt: now,
1142
+ });
1143
+ }
1144
+
1145
+ async function markAllSeriesStateDue(ctx: TimeSeriesMutationContext, dueAt: number) {
1146
+ const states = await ctx.db.query("timeSeriesSeriesState").collect();
1147
+
1148
+ for (const state of states) {
1149
+ await ctx.db.patch(state._id, {
1150
+ nextMaintenanceAt: dueAt,
1151
+ updatedAt: Date.now(),
1152
+ });
1153
+ }
1154
+ }
1155
+
1156
+ async function markSeriesStateDueForUser(
1157
+ ctx: TimeSeriesMutationContext,
1158
+ userId: string,
1159
+ dueAt: number,
1160
+ ) {
1161
+ const batch = await ctx.db
1162
+ .query("timeSeriesSeriesState")
1163
+ .withIndex("by_user", (idx) => idx.eq("userId", userId))
1164
+ .collect();
1165
+
1166
+ for (const state of batch) {
1167
+ await ctx.db.patch(state._id, {
1168
+ nextMaintenanceAt: dueAt,
1169
+ updatedAt: Date.now(),
1170
+ });
1171
+ }
1172
+ }
1173
+
1174
+ async function maintainSeriesState(
1175
+ db: TimeSeriesWriteDb,
1176
+ state: Pick<
1177
+ StoredSeriesState,
1178
+ "connectionId" | "dataSourceId" | "userId" | "provider" | "seriesType"
1179
+ >,
1180
+ now: number,
1181
+ ) {
1182
+ const source = await db.get(state.dataSourceId);
1183
+ const storedState = await db
1184
+ .query("timeSeriesSeriesState")
1185
+ .withIndex("by_source_series", (idx) =>
1186
+ idx.eq("dataSourceId", state.dataSourceId).eq("seriesType", state.seriesType),
1187
+ )
1188
+ .first();
1189
+
1190
+ if (!source || !storedState) {
1191
+ if (storedState) {
1192
+ await db.delete(storedState._id);
1193
+ }
1194
+ return;
1195
+ }
1196
+
1197
+ const effective = await resolveEffectivePolicy(
1198
+ db,
1199
+ storedState.userId,
1200
+ storedState.provider,
1201
+ storedState.seriesType,
1202
+ );
1203
+ await compactRawPoints(db, storedState, effective.tiers, now);
1204
+ await compactRollupPoints(db, storedState, effective.tiers, now);
1205
+
1206
+ const stillHasData = await sourceSeriesHasData(
1207
+ db,
1208
+ storedState.dataSourceId,
1209
+ storedState.seriesType,
1210
+ );
1211
+ if (!stillHasData) {
1212
+ await db.delete(storedState._id);
1213
+ return;
1214
+ }
1215
+
1216
+ const settings = await getTimeSeriesPolicySettings(db);
1217
+ await db.patch(storedState._id, {
1218
+ nextMaintenanceAt: policyRequiresMaintenance(effective.tiers)
1219
+ ? now + settings.maintenanceIntervalMs
1220
+ : now + LONG_IDLE_MAINTENANCE_MS,
1221
+ lastMaintenanceAt: now,
1222
+ updatedAt: now,
1223
+ });
1224
+ }
1225
+
1226
+ async function compactRawPoints(
1227
+ db: TimeSeriesWriteDb,
1228
+ state: StoredSeriesState,
1229
+ tiers: NormalizedTimeSeriesTier[],
1230
+ now: number,
1231
+ ) {
1232
+ const rawTier = getRawTier(tiers);
1233
+ const rawCutoff = rawTier?.toAgeMs ?? null;
1234
+ const query =
1235
+ rawCutoff === null
1236
+ ? rawTier
1237
+ ? []
1238
+ : await db
1239
+ .query("dataPoints")
1240
+ .withIndex("by_source_type_time", (idx) =>
1241
+ idx.eq("dataSourceId", state.dataSourceId).eq("seriesType", state.seriesType),
1242
+ )
1243
+ .take(MAINTENANCE_POINT_BATCH_SIZE)
1244
+ : await db
1245
+ .query("dataPoints")
1246
+ .withIndex("by_source_type_time", (idx) =>
1247
+ idx
1248
+ .eq("dataSourceId", state.dataSourceId)
1249
+ .eq("seriesType", state.seriesType)
1250
+ .lt("recordedAt", now - rawCutoff),
1251
+ )
1252
+ .take(MAINTENANCE_POINT_BATCH_SIZE);
1253
+
1254
+ if (query.length === 0) {
1255
+ return;
1256
+ }
1257
+
1258
+ const rollupGroups = new Map<
1259
+ string,
1260
+ { bucketMs: number; bucketStart: number; stats: AggregatedStats }
1261
+ >();
1262
+ const toDelete: Id<"dataPoints">[] = [];
1263
+
1264
+ for (const point of query as Doc<"dataPoints">[]) {
1265
+ const ageMs = Math.max(0, now - point.recordedAt);
1266
+ const destinationTier = findTierForAge(tiers, ageMs);
1267
+ if (!destinationTier) {
1268
+ toDelete.push(point._id);
1269
+ continue;
1270
+ }
1271
+
1272
+ if (destinationTier.kind === "raw") {
1273
+ continue;
1274
+ }
1275
+
1276
+ const bucketStart = getBucketStart(point.recordedAt, destinationTier.bucketMs);
1277
+ const bucketEnd = getBucketEnd(bucketStart, destinationTier.bucketMs);
1278
+ if (bucketEnd > now - destinationTier.fromAgeMs) {
1279
+ continue;
1280
+ }
1281
+
1282
+ addRawPointToRollupGroup(rollupGroups, point, destinationTier.bucketMs);
1283
+ toDelete.push(point._id);
1284
+ }
1285
+
1286
+ if (rollupGroups.size > 0) {
1287
+ await upsertRollupStatsGroups(db, state.dataSourceId, state.seriesType, rollupGroups.values());
1288
+ }
1289
+ for (const pointId of toDelete) {
1290
+ await db.delete(pointId);
1291
+ }
1292
+ }
1293
+
1294
+ async function compactRollupPoints(
1295
+ db: TimeSeriesWriteDb,
1296
+ state: StoredSeriesState,
1297
+ tiers: NormalizedTimeSeriesTier[],
1298
+ now: number,
1299
+ ) {
1300
+ const rollups = await db
1301
+ .query("timeSeriesRollups")
1302
+ .withIndex("by_source_type_bucket", (idx) =>
1303
+ idx.eq("dataSourceId", state.dataSourceId).eq("seriesType", state.seriesType),
1304
+ )
1305
+ .take(MAINTENANCE_ROLLUP_BATCH_SIZE);
1306
+
1307
+ if (rollups.length === 0) {
1308
+ return;
1309
+ }
1310
+
1311
+ const destinationGroups = new Map<
1312
+ string,
1313
+ { bucketMs: number; bucketStart: number; stats: AggregatedStats }
1314
+ >();
1315
+ const toDelete: Id<"timeSeriesRollups">[] = [];
1316
+
1317
+ for (const rollup of rollups as StoredRollup[]) {
1318
+ const ageMs = Math.max(0, now - rollup.bucketEnd);
1319
+ const destinationTier = findTierForAge(tiers, ageMs);
1320
+
1321
+ if (!destinationTier) {
1322
+ toDelete.push(rollup._id);
1323
+ continue;
1324
+ }
1325
+
1326
+ if (destinationTier.kind === "raw") {
1327
+ continue;
1328
+ }
1329
+
1330
+ const destinationBucketStart = getBucketStart(rollup.bucketStart, destinationTier.bucketMs);
1331
+ const destinationBucketEnd = getBucketEnd(destinationBucketStart, destinationTier.bucketMs);
1332
+
1333
+ if (destinationTier.bucketMs === rollup.bucketMs) {
1334
+ continue;
1335
+ }
1336
+ if (destinationBucketEnd > now - destinationTier.fromAgeMs) {
1337
+ continue;
1338
+ }
1339
+
1340
+ addStatsToRollupGroup(
1341
+ destinationGroups,
1342
+ destinationTier.bucketMs,
1343
+ destinationBucketStart,
1344
+ rollupDocToStats(rollup),
1345
+ );
1346
+ toDelete.push(rollup._id);
1347
+ }
1348
+
1349
+ if (destinationGroups.size > 0) {
1350
+ await upsertRollupStatsGroups(
1351
+ db,
1352
+ state.dataSourceId,
1353
+ state.seriesType,
1354
+ destinationGroups.values(),
1355
+ );
1356
+ }
1357
+ for (const rollupId of toDelete) {
1358
+ await db.delete(rollupId);
1359
+ }
1360
+ }
1361
+
1362
+ async function sourceSeriesHasData(
1363
+ db: Pick<TimeSeriesWriteDb, "query">,
1364
+ dataSourceId: Id<"dataSources">,
1365
+ seriesType: string,
1366
+ ) {
1367
+ const raw = await db
1368
+ .query("dataPoints")
1369
+ .withIndex("by_source_type_time", (idx) =>
1370
+ idx.eq("dataSourceId", dataSourceId).eq("seriesType", seriesType),
1371
+ )
1372
+ .first();
1373
+ if (raw) {
1374
+ return true;
1375
+ }
1376
+
1377
+ const rollup = await db
1378
+ .query("timeSeriesRollups")
1379
+ .withIndex("by_source_type_bucket", (idx) =>
1380
+ idx.eq("dataSourceId", dataSourceId).eq("seriesType", seriesType),
1381
+ )
1382
+ .first();
1383
+ return rollup !== null;
1384
+ }
1385
+
1386
+ function aggregateRawPoints(points: Array<{ recordedAt: number; value: number }>): AggregatedStats {
1387
+ const values = points.map((point) => point.value);
1388
+ const lastPoint = points.reduce((latest, point) =>
1389
+ point.recordedAt >= latest.recordedAt ? point : latest,
1390
+ );
1391
+
1392
+ return {
1393
+ avg: values.reduce((sum, value) => sum + value, 0) / values.length,
1394
+ min: Math.min(...values),
1395
+ max: Math.max(...values),
1396
+ last: lastPoint.value,
1397
+ lastRecordedAt: lastPoint.recordedAt,
1398
+ count: points.length,
1399
+ };
1400
+ }
1401
+
1402
+ function combineAggregatedStats(stats: AggregatedStats[]) {
1403
+ const totalCount = stats.reduce((sum, item) => sum + item.count, 0);
1404
+ const latest = stats.reduce((current, item) =>
1405
+ item.lastRecordedAt >= current.lastRecordedAt ? item : current,
1406
+ );
1407
+
1408
+ return {
1409
+ avg: stats.reduce((sum, item) => sum + item.avg * item.count, 0) / totalCount,
1410
+ min: Math.min(...stats.map((item) => item.min)),
1411
+ max: Math.max(...stats.map((item) => item.max)),
1412
+ last: latest.last,
1413
+ lastRecordedAt: latest.lastRecordedAt,
1414
+ count: totalCount,
1415
+ };
1416
+ }
1417
+
1418
+ function rollupDocToStats(rollup: StoredRollup): AggregatedStats {
1419
+ return {
1420
+ avg: rollup.avg,
1421
+ min: rollup.min,
1422
+ max: rollup.max,
1423
+ last: rollup.last,
1424
+ lastRecordedAt: rollup.lastRecordedAt,
1425
+ count: rollup.count,
1426
+ };
1427
+ }
1428
+
1429
+ function addStatsToRollupGroup(
1430
+ groups: Map<string, { bucketMs: number; bucketStart: number; stats: AggregatedStats }>,
1431
+ bucketMs: number,
1432
+ bucketStart: number,
1433
+ stats: AggregatedStats,
1434
+ ) {
1435
+ const key = `${bucketMs}:${bucketStart}`;
1436
+ const existing = groups.get(key);
1437
+ if (existing) {
1438
+ existing.stats = combineAggregatedStats([existing.stats, stats]);
1439
+ return;
1440
+ }
1441
+
1442
+ groups.set(key, {
1443
+ bucketMs,
1444
+ bucketStart,
1445
+ stats,
1446
+ });
1447
+ }
1448
+
1449
+ // ---------------------------------------------------------------------------
1450
+ // Read path
1451
+ // ---------------------------------------------------------------------------
1452
+
1453
+ async function getPolicyAwarePointsForSource(
1454
+ db: TimeSeriesReadDb,
1455
+ args: {
1456
+ dataSourceId: Id<"dataSources">;
1457
+ userId: string;
1458
+ provider: Doc<"dataSources">["provider"];
1459
+ seriesType: string;
1460
+ startDate: number;
1461
+ endDate: number;
1462
+ limit: number;
1463
+ order: "asc" | "desc";
1464
+ },
1465
+ ) {
1466
+ const effective = await resolveEffectivePolicy(db, args.userId, args.provider, args.seriesType);
1467
+ const now = Date.now();
1468
+ const points: TimeSeriesPoint[] = [];
1469
+
1470
+ for (const tier of effective.tiers) {
1471
+ const interval = getAbsoluteTimeRangeForTier(tier, now, args.startDate, args.endDate);
1472
+ if (!interval) {
1473
+ continue;
1474
+ }
1475
+
1476
+ const preferred =
1477
+ tier.kind === "raw"
1478
+ ? await queryRawPoints(db, {
1479
+ dataSourceId: args.dataSourceId,
1480
+ seriesType: args.seriesType,
1481
+ startDate: interval.start,
1482
+ endDate: interval.end,
1483
+ limit: args.limit + 1,
1484
+ order: args.order,
1485
+ })
1486
+ : await queryRollupPoints(db, {
1487
+ dataSourceId: args.dataSourceId,
1488
+ seriesType: args.seriesType,
1489
+ bucketMs: tier.bucketMs,
1490
+ startDate: interval.start,
1491
+ endDate: interval.end,
1492
+ limit: args.limit + 1,
1493
+ order: args.order,
1494
+ aggregations: tier.aggregations,
1495
+ });
1496
+
1497
+ if (preferred.length > 0) {
1498
+ points.push(...preferred);
1499
+ continue;
1500
+ }
1501
+
1502
+ points.push(
1503
+ ...(await queryFallbackPoints(db, {
1504
+ dataSourceId: args.dataSourceId,
1505
+ seriesType: args.seriesType,
1506
+ startDate: interval.start,
1507
+ endDate: interval.end,
1508
+ limit: args.limit + 1,
1509
+ order: args.order,
1510
+ })),
1511
+ );
1512
+ }
1513
+
1514
+ points.sort((a, b) =>
1515
+ args.order === "asc" ? a.timestamp - b.timestamp : b.timestamp - a.timestamp,
1516
+ );
1517
+ return points.slice(0, args.limit);
1518
+ }
1519
+
1520
+ function getAbsoluteTimeRangeForTier(
1521
+ tier: NormalizedTimeSeriesTier,
1522
+ now: number,
1523
+ startDate: number,
1524
+ endDate: number,
1525
+ ) {
1526
+ const absoluteStart =
1527
+ tier.toAgeMs === null ? startDate : Math.max(startDate, now - tier.toAgeMs + 1);
1528
+ const absoluteEnd = Math.min(endDate, now - tier.fromAgeMs);
1529
+
1530
+ if (absoluteStart > absoluteEnd) {
1531
+ return null;
1532
+ }
1533
+
1534
+ return {
1535
+ start: absoluteStart,
1536
+ end: absoluteEnd,
1537
+ };
1538
+ }
1539
+
1540
+ async function queryRawPoints(
1541
+ db: Pick<TimeSeriesReadDb, "query">,
1542
+ args: {
1543
+ dataSourceId: Id<"dataSources">;
1544
+ seriesType: string;
1545
+ startDate: number;
1546
+ endDate: number;
1547
+ limit: number;
1548
+ order: "asc" | "desc";
1549
+ },
1550
+ ) {
1551
+ const raw = await db
1552
+ .query("dataPoints")
1553
+ .withIndex("by_source_type_time", (idx) =>
1554
+ idx
1555
+ .eq("dataSourceId", args.dataSourceId)
1556
+ .eq("seriesType", args.seriesType)
1557
+ .gte("recordedAt", args.startDate)
1558
+ .lte("recordedAt", args.endDate),
1559
+ )
1560
+ .order(args.order)
1561
+ .take(args.limit);
1562
+
1563
+ return raw.map((point: Doc<"dataPoints">) => ({
1564
+ timestamp: point.recordedAt,
1565
+ value: point.value,
1566
+ resolution: "raw" as const,
1567
+ }));
1568
+ }
1569
+
1570
+ async function queryRollupPoints(
1571
+ db: Pick<TimeSeriesReadDb, "query">,
1572
+ args: {
1573
+ dataSourceId: Id<"dataSources">;
1574
+ seriesType: string;
1575
+ bucketMs: number;
1576
+ startDate: number;
1577
+ endDate: number;
1578
+ limit: number;
1579
+ order: "asc" | "desc";
1580
+ aggregations: readonly TimeSeriesAggregation[];
1581
+ },
1582
+ ) {
1583
+ const rows = await db
1584
+ .query("timeSeriesRollups")
1585
+ .withIndex("by_source_type_bucket_size", (idx) =>
1586
+ idx
1587
+ .eq("dataSourceId", args.dataSourceId)
1588
+ .eq("seriesType", args.seriesType)
1589
+ .eq("bucketMs", args.bucketMs)
1590
+ .gte("bucketStart", Math.max(0, args.startDate - args.bucketMs))
1591
+ .lte("bucketStart", args.endDate),
1592
+ )
1593
+ .order(args.order)
1594
+ .take(args.limit);
1595
+
1596
+ return rows
1597
+ .filter(
1598
+ (row: StoredRollup) => row.bucketEnd >= args.startDate && row.bucketStart <= args.endDate,
1599
+ )
1600
+ .map((row: StoredRollup) => toRollupPoint(row, args.aggregations));
1601
+ }
1602
+
1603
+ async function queryFallbackPoints(
1604
+ db: Pick<TimeSeriesReadDb, "query">,
1605
+ args: {
1606
+ dataSourceId: Id<"dataSources">;
1607
+ seriesType: string;
1608
+ startDate: number;
1609
+ endDate: number;
1610
+ limit: number;
1611
+ order: "asc" | "desc";
1612
+ },
1613
+ ) {
1614
+ const raw = await queryRawPoints(db, args);
1615
+ if (raw.length > 0) {
1616
+ return raw;
1617
+ }
1618
+
1619
+ const rows = await db
1620
+ .query("timeSeriesRollups")
1621
+ .withIndex("by_source_type_bucket", (idx) =>
1622
+ idx
1623
+ .eq("dataSourceId", args.dataSourceId)
1624
+ .eq("seriesType", args.seriesType)
1625
+ .gte("bucketStart", Math.max(0, args.startDate - DAY_MS))
1626
+ .lte("bucketStart", args.endDate),
1627
+ )
1628
+ .take(args.limit * 4);
1629
+
1630
+ rows.sort((a: StoredRollup, b: StoredRollup) =>
1631
+ a.bucketMs === b.bucketMs ? a.bucketStart - b.bucketStart : a.bucketMs - b.bucketMs,
1632
+ );
1633
+
1634
+ const filtered = rows.filter(
1635
+ (row: StoredRollup) => row.bucketEnd >= args.startDate && row.bucketStart <= args.endDate,
1636
+ );
1637
+ return filtered.map((row: StoredRollup) =>
1638
+ toRollupPoint(row, normalizeAggregations(DEFAULT_TIME_SERIES_AGGREGATIONS)),
1639
+ );
1640
+ }
1641
+
1642
+ function toRollupPoint(
1643
+ rollup: StoredRollup,
1644
+ aggregations: readonly TimeSeriesAggregation[],
1645
+ ): TimeSeriesPoint {
1646
+ return {
1647
+ timestamp: rollup.bucketStart,
1648
+ value: selectPrimaryRollupValue(rollup, aggregations),
1649
+ resolution: "rollup",
1650
+ bucketMinutes: Math.floor(rollup.bucketMs / (60 * 1000)),
1651
+ avg: rollup.avg,
1652
+ min: rollup.min,
1653
+ max: rollup.max,
1654
+ last: rollup.last,
1655
+ count: rollup.count,
1656
+ };
1657
+ }
1658
+
1659
+ function selectPrimaryRollupValue(
1660
+ rollup: StoredRollup,
1661
+ aggregations: readonly TimeSeriesAggregation[],
1662
+ ) {
1663
+ if (aggregations.includes("avg")) {
1664
+ return rollup.avg;
1665
+ }
1666
+ if (aggregations.includes("last")) {
1667
+ return rollup.last;
1668
+ }
1669
+ if (aggregations.includes("max")) {
1670
+ return rollup.max;
1671
+ }
1672
+ if (aggregations.includes("min")) {
1673
+ return rollup.min;
1674
+ }
1675
+ return rollup.count;
1676
+ }