@clipin/convex-wearables 0.0.3 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +395 -0
- package/dist/client/index.d.ts +38 -2
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +34 -3
- package/dist/client/index.js.map +1 -1
- package/dist/client/types.d.ts +83 -0
- package/dist/client/types.d.ts.map +1 -1
- package/dist/client/types.js.map +1 -1
- package/dist/component/dataPoints.d.ts +148 -34
- package/dist/component/dataPoints.d.ts.map +1 -1
- package/dist/component/dataPoints.js +1048 -139
- package/dist/component/dataPoints.js.map +1 -1
- package/dist/component/lifecycle.d.ts.map +1 -1
- package/dist/component/lifecycle.js +37 -1
- package/dist/component/lifecycle.js.map +1 -1
- package/dist/component/schema.d.ts +166 -2
- package/dist/component/schema.d.ts.map +1 -1
- package/dist/component/schema.js +89 -0
- package/dist/component/schema.js.map +1 -1
- package/dist/component/timeSeriesPolicyUtils.d.ts +97 -0
- package/dist/component/timeSeriesPolicyUtils.d.ts.map +1 -0
- package/dist/component/timeSeriesPolicyUtils.js +163 -0
- package/dist/component/timeSeriesPolicyUtils.js.map +1 -0
- package/dist/test.d.ts +162 -2
- package/dist/test.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/client/index.test.ts +179 -0
- package/src/client/index.ts +87 -7
- package/src/client/types.ts +99 -0
- package/src/component/dataPoints.test.ts +546 -1
- package/src/component/dataPoints.ts +1526 -155
- package/src/component/lifecycle.ts +42 -1
- package/src/component/schema.ts +106 -0
- package/src/component/timeSeriesPolicyUtils.ts +243 -0
|
@@ -1,14 +1,148 @@
|
|
|
1
1
|
import { v } from "convex/values";
|
|
2
|
-
import {
|
|
2
|
+
import { internal } from "./_generated/api";
|
|
3
|
+
import type { Doc, Id } from "./_generated/dataModel";
|
|
4
|
+
import {
|
|
5
|
+
type DatabaseReader,
|
|
6
|
+
type DatabaseWriter,
|
|
7
|
+
internalMutation,
|
|
8
|
+
type MutationCtx,
|
|
9
|
+
mutation,
|
|
10
|
+
query,
|
|
11
|
+
} from "./_generated/server";
|
|
12
|
+
import { providerName, timeSeriesAggregation } from "./schema";
|
|
13
|
+
import {
|
|
14
|
+
buildBuiltinFullTiers,
|
|
15
|
+
comparePolicyScopes,
|
|
16
|
+
createTimeSeriesPolicyScopeKey,
|
|
17
|
+
DEFAULT_MAINTENANCE_INTERVAL_MS,
|
|
18
|
+
DEFAULT_POLICY_SET_KEY,
|
|
19
|
+
DEFAULT_TIME_SERIES_AGGREGATIONS,
|
|
20
|
+
findTierForAge,
|
|
21
|
+
getBucketEnd,
|
|
22
|
+
getBucketStart,
|
|
23
|
+
getRawTier,
|
|
24
|
+
inferTimeSeriesPolicyScope,
|
|
25
|
+
type NormalizedTimeSeriesTier,
|
|
26
|
+
normalizeAggregations,
|
|
27
|
+
normalizeTimeSeriesPolicyRuleInputs,
|
|
28
|
+
parseDurationInput,
|
|
29
|
+
resolveScopedPolicyRule,
|
|
30
|
+
type TimeSeriesAggregation,
|
|
31
|
+
type TimeSeriesPolicyRuleInput,
|
|
32
|
+
} from "./timeSeriesPolicyUtils";
|
|
33
|
+
|
|
34
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
35
|
+
const MAX_QUERY_LIMIT = 2000;
|
|
36
|
+
const MAINTENANCE_SETTINGS_KEY = "default";
|
|
37
|
+
const MAINTENANCE_BATCH_SIZE = 20;
|
|
38
|
+
const MAINTENANCE_POINT_BATCH_SIZE = 2000;
|
|
39
|
+
const MAINTENANCE_ROLLUP_BATCH_SIZE = 500;
|
|
40
|
+
const LONG_IDLE_MAINTENANCE_MS = 30 * DAY_MS;
|
|
41
|
+
|
|
42
|
+
type StoredPolicyRule = Doc<"timeSeriesPolicyRules">;
|
|
43
|
+
type StoredRollup = Doc<"timeSeriesRollups">;
|
|
44
|
+
type StoredSeriesState = Doc<"timeSeriesSeriesState">;
|
|
45
|
+
|
|
46
|
+
type DurationInput = string | number;
|
|
47
|
+
|
|
48
|
+
type TimeSeriesPoint = {
|
|
49
|
+
timestamp: number;
|
|
50
|
+
value: number;
|
|
51
|
+
resolution?: "raw" | "rollup";
|
|
52
|
+
bucketMinutes?: number;
|
|
53
|
+
avg?: number;
|
|
54
|
+
min?: number;
|
|
55
|
+
max?: number;
|
|
56
|
+
last?: number;
|
|
57
|
+
count?: number;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
type StoredPointInput = {
|
|
61
|
+
recordedAt: number;
|
|
62
|
+
value: number;
|
|
63
|
+
externalId?: string;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
type AggregatedStats = {
|
|
67
|
+
avg: number;
|
|
68
|
+
min: number;
|
|
69
|
+
max: number;
|
|
70
|
+
last: number;
|
|
71
|
+
lastRecordedAt: number;
|
|
72
|
+
count: number;
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
type EffectivePolicy = {
|
|
76
|
+
tiers: NormalizedTimeSeriesTier[];
|
|
77
|
+
sourceKind: "preset" | "default" | "builtin";
|
|
78
|
+
sourceKey: string | null;
|
|
79
|
+
matchedScope: "default" | "global" | "provider" | "series" | "provider_series";
|
|
80
|
+
provider?: string;
|
|
81
|
+
seriesType?: string;
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
type PolicySettingsDoc =
|
|
85
|
+
| Doc<"timeSeriesPolicySettings">
|
|
86
|
+
| {
|
|
87
|
+
key: string;
|
|
88
|
+
maintenanceEnabled: boolean;
|
|
89
|
+
maintenanceIntervalMs: number;
|
|
90
|
+
updatedAt: number;
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
type TimeSeriesReadDb = Pick<DatabaseReader, "get" | "query">;
|
|
94
|
+
type TimeSeriesWriteDb = Pick<DatabaseWriter, "delete" | "get" | "insert" | "patch" | "query">;
|
|
95
|
+
type TimeSeriesMutationContext = Pick<MutationCtx, "db" | "scheduler">;
|
|
96
|
+
|
|
97
|
+
const durationInputValidator = v.union(v.string(), v.number());
|
|
98
|
+
|
|
99
|
+
const tierInputValidator = v.union(
|
|
100
|
+
v.object({
|
|
101
|
+
kind: v.literal("raw"),
|
|
102
|
+
fromAge: durationInputValidator,
|
|
103
|
+
toAge: v.union(durationInputValidator, v.null()),
|
|
104
|
+
}),
|
|
105
|
+
v.object({
|
|
106
|
+
kind: v.literal("rollup"),
|
|
107
|
+
fromAge: durationInputValidator,
|
|
108
|
+
toAge: v.union(durationInputValidator, v.null()),
|
|
109
|
+
bucket: durationInputValidator,
|
|
110
|
+
aggregations: v.optional(v.array(timeSeriesAggregation)),
|
|
111
|
+
}),
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
const policyRuleInputValidator = v.object({
|
|
115
|
+
provider: v.optional(providerName),
|
|
116
|
+
seriesType: v.optional(v.string()),
|
|
117
|
+
tiers: v.array(tierInputValidator),
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
const policyPresetInputValidator = v.object({
|
|
121
|
+
key: v.string(),
|
|
122
|
+
rules: v.array(policyRuleInputValidator),
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
const maintenanceInputValidator = v.object({
|
|
126
|
+
enabled: v.optional(v.boolean()),
|
|
127
|
+
interval: v.optional(durationInputValidator),
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
const timeSeriesPointValidator = v.object({
|
|
131
|
+
timestamp: v.number(),
|
|
132
|
+
value: v.number(),
|
|
133
|
+
resolution: v.optional(v.union(v.literal("raw"), v.literal("rollup"))),
|
|
134
|
+
bucketMinutes: v.optional(v.number()),
|
|
135
|
+
avg: v.optional(v.number()),
|
|
136
|
+
min: v.optional(v.number()),
|
|
137
|
+
max: v.optional(v.number()),
|
|
138
|
+
last: v.optional(v.number()),
|
|
139
|
+
count: v.optional(v.number()),
|
|
140
|
+
});
|
|
3
141
|
|
|
4
142
|
// ---------------------------------------------------------------------------
|
|
5
143
|
// Queries
|
|
6
144
|
// ---------------------------------------------------------------------------
|
|
7
145
|
|
|
8
|
-
/**
|
|
9
|
-
* Get time-series data points with cursor-based pagination.
|
|
10
|
-
* Uses the by_source_type_time index for efficient range queries.
|
|
11
|
-
*/
|
|
12
146
|
export const getTimeSeries = query({
|
|
13
147
|
args: {
|
|
14
148
|
dataSourceId: v.id("dataSources"),
|
|
@@ -20,56 +154,51 @@ export const getTimeSeries = query({
|
|
|
20
154
|
order: v.optional(v.union(v.literal("asc"), v.literal("desc"))),
|
|
21
155
|
},
|
|
22
156
|
returns: v.object({
|
|
23
|
-
points: v.array(
|
|
24
|
-
v.object({
|
|
25
|
-
timestamp: v.number(),
|
|
26
|
-
value: v.number(),
|
|
27
|
-
}),
|
|
28
|
-
),
|
|
157
|
+
points: v.array(timeSeriesPointValidator),
|
|
29
158
|
nextCursor: v.union(v.string(), v.null()),
|
|
30
159
|
hasMore: v.boolean(),
|
|
31
160
|
}),
|
|
32
161
|
handler: async (ctx, args) => {
|
|
33
|
-
const
|
|
162
|
+
const source = await ctx.db.get(args.dataSourceId);
|
|
163
|
+
if (!source) {
|
|
164
|
+
return { points: [], nextCursor: null, hasMore: false };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
const limit = Math.min(args.limit ?? 500, MAX_QUERY_LIMIT);
|
|
34
168
|
const order = args.order ?? "asc";
|
|
169
|
+
const cursor = args.cursor ? Number(args.cursor) : undefined;
|
|
35
170
|
|
|
36
|
-
const
|
|
171
|
+
const points = await getPolicyAwarePointsForSource(ctx.db, {
|
|
172
|
+
dataSourceId: source._id,
|
|
173
|
+
userId: source.userId,
|
|
174
|
+
provider: source.provider,
|
|
175
|
+
seriesType: args.seriesType,
|
|
176
|
+
startDate: args.startDate,
|
|
177
|
+
endDate: args.endDate,
|
|
178
|
+
limit: limit + 2,
|
|
179
|
+
order,
|
|
180
|
+
});
|
|
37
181
|
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
: idx
|
|
48
|
-
.eq("dataSourceId", args.dataSourceId)
|
|
49
|
-
.eq("seriesType", args.seriesType)
|
|
50
|
-
.gte("recordedAt", args.startDate)
|
|
51
|
-
.lte("recordedAt", startDate),
|
|
52
|
-
)
|
|
53
|
-
.order(order)
|
|
54
|
-
.take(limit + 1);
|
|
55
|
-
|
|
56
|
-
const hasMore = results.length > limit;
|
|
57
|
-
const items = hasMore ? results.slice(0, limit) : results;
|
|
58
|
-
const points = items.map((dp) => ({
|
|
59
|
-
timestamp: dp.recordedAt,
|
|
60
|
-
value: dp.value,
|
|
61
|
-
}));
|
|
182
|
+
const filtered =
|
|
183
|
+
cursor === undefined
|
|
184
|
+
? points
|
|
185
|
+
: points.filter((point) =>
|
|
186
|
+
order === "asc" ? point.timestamp > cursor : point.timestamp < cursor,
|
|
187
|
+
);
|
|
188
|
+
|
|
189
|
+
const hasMore = filtered.length > limit;
|
|
190
|
+
const items = hasMore ? filtered.slice(0, limit) : filtered;
|
|
62
191
|
const nextCursor =
|
|
63
|
-
hasMore && items.length > 0 ? String(items[items.length - 1].
|
|
192
|
+
hasMore && items.length > 0 ? String(items[items.length - 1].timestamp) : null;
|
|
64
193
|
|
|
65
|
-
return {
|
|
194
|
+
return {
|
|
195
|
+
points: items,
|
|
196
|
+
nextCursor,
|
|
197
|
+
hasMore,
|
|
198
|
+
};
|
|
66
199
|
},
|
|
67
200
|
});
|
|
68
201
|
|
|
69
|
-
/**
|
|
70
|
-
* Get time-series data for a user across all their data sources for a given series type.
|
|
71
|
-
* This is a user-facing query that resolves data sources internally.
|
|
72
|
-
*/
|
|
73
202
|
export const getTimeSeriesForUser = query({
|
|
74
203
|
args: {
|
|
75
204
|
userId: v.string(),
|
|
@@ -78,49 +207,36 @@ export const getTimeSeriesForUser = query({
|
|
|
78
207
|
endDate: v.number(),
|
|
79
208
|
limit: v.optional(v.number()),
|
|
80
209
|
},
|
|
81
|
-
returns: v.array(
|
|
82
|
-
v.object({
|
|
83
|
-
timestamp: v.number(),
|
|
84
|
-
value: v.number(),
|
|
85
|
-
}),
|
|
86
|
-
),
|
|
210
|
+
returns: v.array(timeSeriesPointValidator),
|
|
87
211
|
handler: async (ctx, args) => {
|
|
88
|
-
const limit = Math.min(args.limit ?? 500,
|
|
212
|
+
const limit = Math.min(args.limit ?? 500, MAX_QUERY_LIMIT);
|
|
89
213
|
|
|
90
|
-
// Get all data sources for this user
|
|
91
214
|
const sources = await ctx.db
|
|
92
215
|
.query("dataSources")
|
|
93
216
|
.withIndex("by_user_provider", (idx) => idx.eq("userId", args.userId))
|
|
94
217
|
.collect();
|
|
95
218
|
|
|
96
|
-
|
|
97
|
-
const allPoints: { timestamp: number; value: number }[] = [];
|
|
219
|
+
const points: TimeSeriesPoint[] = [];
|
|
98
220
|
for (const source of sources) {
|
|
99
|
-
|
|
100
|
-
.
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
allPoints.push({ timestamp: dp.recordedAt, value: dp.value });
|
|
112
|
-
}
|
|
221
|
+
points.push(
|
|
222
|
+
...(await getPolicyAwarePointsForSource(ctx.db, {
|
|
223
|
+
dataSourceId: source._id,
|
|
224
|
+
userId: source.userId,
|
|
225
|
+
provider: source.provider,
|
|
226
|
+
seriesType: args.seriesType,
|
|
227
|
+
startDate: args.startDate,
|
|
228
|
+
endDate: args.endDate,
|
|
229
|
+
limit,
|
|
230
|
+
order: "asc",
|
|
231
|
+
})),
|
|
232
|
+
);
|
|
113
233
|
}
|
|
114
234
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
return allPoints.slice(0, limit);
|
|
235
|
+
points.sort((a, b) => a.timestamp - b.timestamp);
|
|
236
|
+
return points.slice(0, limit);
|
|
118
237
|
},
|
|
119
238
|
});
|
|
120
239
|
|
|
121
|
-
/**
|
|
122
|
-
* Get the latest data point for a user and series type.
|
|
123
|
-
*/
|
|
124
240
|
export const getLatestDataPoint = query({
|
|
125
241
|
args: {
|
|
126
242
|
userId: v.string(),
|
|
@@ -143,7 +259,7 @@ export const getLatestDataPoint = query({
|
|
|
143
259
|
let latest: { timestamp: number; value: number; provider: string } | null = null;
|
|
144
260
|
|
|
145
261
|
for (const source of sources) {
|
|
146
|
-
const
|
|
262
|
+
const rawPoint = await ctx.db
|
|
147
263
|
.query("dataPoints")
|
|
148
264
|
.withIndex("by_source_type_time", (idx) =>
|
|
149
265
|
idx.eq("dataSourceId", source._id).eq("seriesType", args.seriesType),
|
|
@@ -151,12 +267,39 @@ export const getLatestDataPoint = query({
|
|
|
151
267
|
.order("desc")
|
|
152
268
|
.first();
|
|
153
269
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
270
|
+
const rollupPoint = await ctx.db
|
|
271
|
+
.query("timeSeriesRollups")
|
|
272
|
+
.withIndex("by_source_type_bucket", (idx) =>
|
|
273
|
+
idx.eq("dataSourceId", source._id).eq("seriesType", args.seriesType),
|
|
274
|
+
)
|
|
275
|
+
.order("desc")
|
|
276
|
+
.first();
|
|
277
|
+
|
|
278
|
+
const rawCandidate = rawPoint
|
|
279
|
+
? {
|
|
280
|
+
timestamp: rawPoint.recordedAt,
|
|
281
|
+
value: rawPoint.value,
|
|
282
|
+
provider: source.provider,
|
|
283
|
+
}
|
|
284
|
+
: null;
|
|
285
|
+
const rollupCandidate = rollupPoint
|
|
286
|
+
? {
|
|
287
|
+
timestamp: rollupPoint.lastRecordedAt,
|
|
288
|
+
value: rollupPoint.last,
|
|
289
|
+
provider: source.provider,
|
|
290
|
+
}
|
|
291
|
+
: null;
|
|
292
|
+
const candidate =
|
|
293
|
+
rawCandidate === null
|
|
294
|
+
? rollupCandidate
|
|
295
|
+
: rollupCandidate === null
|
|
296
|
+
? rawCandidate
|
|
297
|
+
: rawCandidate.timestamp >= rollupCandidate.timestamp
|
|
298
|
+
? rawCandidate
|
|
299
|
+
: rollupCandidate;
|
|
300
|
+
|
|
301
|
+
if (candidate && (latest === null || candidate.timestamp > latest.timestamp)) {
|
|
302
|
+
latest = candidate;
|
|
160
303
|
}
|
|
161
304
|
}
|
|
162
305
|
|
|
@@ -164,34 +307,95 @@ export const getLatestDataPoint = query({
|
|
|
164
307
|
},
|
|
165
308
|
});
|
|
166
309
|
|
|
167
|
-
/**
|
|
168
|
-
* Get all available series types for a user (i.e., types that have at least one data point).
|
|
169
|
-
*/
|
|
170
310
|
export const getAvailableSeriesTypes = query({
|
|
171
311
|
args: { userId: v.string() },
|
|
172
312
|
returns: v.array(v.string()),
|
|
173
313
|
handler: async (ctx, args) => {
|
|
174
|
-
const
|
|
175
|
-
.query("
|
|
176
|
-
.withIndex("
|
|
314
|
+
const states = await ctx.db
|
|
315
|
+
.query("timeSeriesSeriesState")
|
|
316
|
+
.withIndex("by_user", (idx) => idx.eq("userId", args.userId))
|
|
177
317
|
.collect();
|
|
178
318
|
|
|
179
|
-
|
|
319
|
+
return Array.from(new Set(states.map((state) => state.seriesType))).sort();
|
|
320
|
+
},
|
|
321
|
+
});
|
|
180
322
|
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
.take(200);
|
|
323
|
+
export const getTimeSeriesPolicyConfiguration = query({
|
|
324
|
+
args: {},
|
|
325
|
+
returns: v.any(),
|
|
326
|
+
handler: async (ctx) => {
|
|
327
|
+
const settings = await getTimeSeriesPolicySettings(ctx.db);
|
|
328
|
+
const rules = await ctx.db.query("timeSeriesPolicyRules").collect();
|
|
188
329
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
330
|
+
return {
|
|
331
|
+
maintenance: {
|
|
332
|
+
enabled: settings.maintenanceEnabled,
|
|
333
|
+
intervalMs: settings.maintenanceIntervalMs,
|
|
334
|
+
},
|
|
335
|
+
defaultRules: formatPolicyRulesForResponse(
|
|
336
|
+
rules.filter(
|
|
337
|
+
(rule) =>
|
|
338
|
+
rule.policySetKind === "default" && rule.policySetKey === DEFAULT_POLICY_SET_KEY,
|
|
339
|
+
),
|
|
340
|
+
),
|
|
341
|
+
presets: groupPolicyRulesByPreset(rules.filter((rule) => rule.policySetKind === "preset")),
|
|
342
|
+
};
|
|
343
|
+
},
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
export const getUserTimeSeriesPolicyPreset = query({
|
|
347
|
+
args: {
|
|
348
|
+
userId: v.string(),
|
|
349
|
+
},
|
|
350
|
+
returns: v.union(
|
|
351
|
+
v.object({
|
|
352
|
+
userId: v.string(),
|
|
353
|
+
presetKey: v.string(),
|
|
354
|
+
updatedAt: v.number(),
|
|
355
|
+
}),
|
|
356
|
+
v.null(),
|
|
357
|
+
),
|
|
358
|
+
handler: async (ctx, args) => {
|
|
359
|
+
const assignment = await ctx.db
|
|
360
|
+
.query("timeSeriesPolicyAssignments")
|
|
361
|
+
.withIndex("by_user", (idx) => idx.eq("userId", args.userId))
|
|
362
|
+
.first();
|
|
363
|
+
|
|
364
|
+
if (!assignment) {
|
|
365
|
+
return null;
|
|
192
366
|
}
|
|
193
367
|
|
|
194
|
-
return
|
|
368
|
+
return {
|
|
369
|
+
userId: assignment.userId,
|
|
370
|
+
presetKey: assignment.presetKey,
|
|
371
|
+
updatedAt: assignment.updatedAt,
|
|
372
|
+
};
|
|
373
|
+
},
|
|
374
|
+
});
|
|
375
|
+
|
|
376
|
+
export const getEffectiveTimeSeriesPolicy = query({
|
|
377
|
+
args: {
|
|
378
|
+
userId: v.string(),
|
|
379
|
+
provider: providerName,
|
|
380
|
+
seriesType: v.string(),
|
|
381
|
+
},
|
|
382
|
+
returns: v.any(),
|
|
383
|
+
handler: async (ctx, args) => {
|
|
384
|
+
const effective = await resolveEffectivePolicy(
|
|
385
|
+
ctx.db,
|
|
386
|
+
args.userId,
|
|
387
|
+
args.provider,
|
|
388
|
+
args.seriesType,
|
|
389
|
+
);
|
|
390
|
+
|
|
391
|
+
return {
|
|
392
|
+
provider: args.provider,
|
|
393
|
+
seriesType: args.seriesType,
|
|
394
|
+
sourceKind: effective.sourceKind,
|
|
395
|
+
sourceKey: effective.sourceKey,
|
|
396
|
+
matchedScope: effective.matchedScope,
|
|
397
|
+
tiers: effective.tiers.map(formatTierForResponse),
|
|
398
|
+
};
|
|
195
399
|
},
|
|
196
400
|
});
|
|
197
401
|
|
|
@@ -199,9 +403,130 @@ export const getAvailableSeriesTypes = query({
|
|
|
199
403
|
// Mutations
|
|
200
404
|
// ---------------------------------------------------------------------------
|
|
201
405
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
406
|
+
export const replaceTimeSeriesPolicyConfiguration = mutation({
|
|
407
|
+
args: {
|
|
408
|
+
defaultRules: v.array(policyRuleInputValidator),
|
|
409
|
+
presets: v.optional(v.array(policyPresetInputValidator)),
|
|
410
|
+
maintenance: v.optional(maintenanceInputValidator),
|
|
411
|
+
},
|
|
412
|
+
returns: v.object({
|
|
413
|
+
defaultRulesStored: v.number(),
|
|
414
|
+
presetsStored: v.number(),
|
|
415
|
+
}),
|
|
416
|
+
handler: async (ctx, args) => {
|
|
417
|
+
const normalizedDefaultRules = normalizeTimeSeriesPolicyRuleInputs(
|
|
418
|
+
args.defaultRules as TimeSeriesPolicyRuleInput[],
|
|
419
|
+
);
|
|
420
|
+
const presetKeys = new Set<string>();
|
|
421
|
+
const normalizedPresets = (args.presets ?? []).map((preset) => {
|
|
422
|
+
if (!preset.key.trim()) {
|
|
423
|
+
throw new Error("Preset keys must not be empty");
|
|
424
|
+
}
|
|
425
|
+
if (preset.key === DEFAULT_POLICY_SET_KEY) {
|
|
426
|
+
throw new Error(`Preset key "${DEFAULT_POLICY_SET_KEY}" is reserved`);
|
|
427
|
+
}
|
|
428
|
+
if (presetKeys.has(preset.key)) {
|
|
429
|
+
throw new Error(`Duplicate time-series policy preset key "${preset.key}"`);
|
|
430
|
+
}
|
|
431
|
+
presetKeys.add(preset.key);
|
|
432
|
+
|
|
433
|
+
return {
|
|
434
|
+
key: preset.key,
|
|
435
|
+
rules: normalizeTimeSeriesPolicyRuleInputs(preset.rules as TimeSeriesPolicyRuleInput[]),
|
|
436
|
+
};
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
const existingRules = await ctx.db.query("timeSeriesPolicyRules").collect();
|
|
440
|
+
for (const rule of existingRules) {
|
|
441
|
+
await ctx.db.delete(rule._id);
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
const updatedAt = Date.now();
|
|
445
|
+
for (const rule of normalizedDefaultRules) {
|
|
446
|
+
await ctx.db.insert("timeSeriesPolicyRules", {
|
|
447
|
+
policySetKind: "default",
|
|
448
|
+
policySetKey: DEFAULT_POLICY_SET_KEY,
|
|
449
|
+
scopeKey: createTimeSeriesPolicyScopeKey(rule.provider, rule.seriesType),
|
|
450
|
+
provider: rule.provider,
|
|
451
|
+
seriesType: rule.seriesType,
|
|
452
|
+
tiers: rule.tiers,
|
|
453
|
+
updatedAt,
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
for (const preset of normalizedPresets) {
|
|
458
|
+
for (const rule of preset.rules) {
|
|
459
|
+
await ctx.db.insert("timeSeriesPolicyRules", {
|
|
460
|
+
policySetKind: "preset",
|
|
461
|
+
policySetKey: preset.key,
|
|
462
|
+
scopeKey: createTimeSeriesPolicyScopeKey(rule.provider, rule.seriesType),
|
|
463
|
+
provider: rule.provider,
|
|
464
|
+
seriesType: rule.seriesType,
|
|
465
|
+
tiers: rule.tiers,
|
|
466
|
+
updatedAt,
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
await upsertTimeSeriesPolicySettings(ctx, args.maintenance);
|
|
472
|
+
await markAllSeriesStateDue(ctx, updatedAt);
|
|
473
|
+
await ensureTimeSeriesMaintenanceScheduled(ctx);
|
|
474
|
+
|
|
475
|
+
return {
|
|
476
|
+
defaultRulesStored: normalizedDefaultRules.length,
|
|
477
|
+
presetsStored: normalizedPresets.reduce((sum, preset) => sum + preset.rules.length, 0),
|
|
478
|
+
};
|
|
479
|
+
},
|
|
480
|
+
});
|
|
481
|
+
|
|
482
|
+
export const setUserTimeSeriesPolicyPreset = mutation({
|
|
483
|
+
args: {
|
|
484
|
+
userId: v.string(),
|
|
485
|
+
presetKey: v.union(v.string(), v.null()),
|
|
486
|
+
},
|
|
487
|
+
returns: v.null(),
|
|
488
|
+
handler: async (ctx, args) => {
|
|
489
|
+
const existing = await ctx.db
|
|
490
|
+
.query("timeSeriesPolicyAssignments")
|
|
491
|
+
.withIndex("by_user", (idx) => idx.eq("userId", args.userId))
|
|
492
|
+
.first();
|
|
493
|
+
|
|
494
|
+
if (args.presetKey === null) {
|
|
495
|
+
if (existing) {
|
|
496
|
+
await ctx.db.delete(existing._id);
|
|
497
|
+
}
|
|
498
|
+
await markSeriesStateDueForUser(ctx, args.userId, Date.now());
|
|
499
|
+
await ensureTimeSeriesMaintenanceScheduled(ctx);
|
|
500
|
+
return null;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
const presetKey = args.presetKey;
|
|
504
|
+
const presetRules = await ctx.db
|
|
505
|
+
.query("timeSeriesPolicyRules")
|
|
506
|
+
.withIndex("by_set", (idx) => idx.eq("policySetKind", "preset").eq("policySetKey", presetKey))
|
|
507
|
+
.take(1);
|
|
508
|
+
if (presetRules.length === 0) {
|
|
509
|
+
throw new Error(`Time-series policy preset "${presetKey}" does not exist`);
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
const patch = {
|
|
513
|
+
userId: args.userId,
|
|
514
|
+
presetKey,
|
|
515
|
+
updatedAt: Date.now(),
|
|
516
|
+
};
|
|
517
|
+
|
|
518
|
+
if (existing) {
|
|
519
|
+
await ctx.db.patch(existing._id, patch);
|
|
520
|
+
} else {
|
|
521
|
+
await ctx.db.insert("timeSeriesPolicyAssignments", patch);
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
await markSeriesStateDueForUser(ctx, args.userId, Date.now());
|
|
525
|
+
await ensureTimeSeriesMaintenanceScheduled(ctx);
|
|
526
|
+
return null;
|
|
527
|
+
},
|
|
528
|
+
});
|
|
529
|
+
|
|
205
530
|
export const storeDataPoint = internalMutation({
|
|
206
531
|
args: {
|
|
207
532
|
dataSourceId: v.id("dataSources"),
|
|
@@ -210,32 +535,24 @@ export const storeDataPoint = internalMutation({
|
|
|
210
535
|
value: v.number(),
|
|
211
536
|
externalId: v.optional(v.string()),
|
|
212
537
|
},
|
|
213
|
-
returns: v.id("dataPoints"),
|
|
538
|
+
returns: v.union(v.id("dataPoints"), v.null()),
|
|
214
539
|
handler: async (ctx, args) => {
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
.
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
if (existing) {
|
|
227
|
-
await ctx.db.patch(existing._id, { value: args.value });
|
|
228
|
-
return existing._id;
|
|
229
|
-
}
|
|
540
|
+
const result = await storePointsWithPolicy(ctx, {
|
|
541
|
+
dataSourceId: args.dataSourceId,
|
|
542
|
+
seriesType: args.seriesType,
|
|
543
|
+
points: [
|
|
544
|
+
{
|
|
545
|
+
recordedAt: args.recordedAt,
|
|
546
|
+
value: args.value,
|
|
547
|
+
externalId: args.externalId,
|
|
548
|
+
},
|
|
549
|
+
],
|
|
550
|
+
});
|
|
230
551
|
|
|
231
|
-
return
|
|
552
|
+
return result.lastRawId;
|
|
232
553
|
},
|
|
233
554
|
});
|
|
234
555
|
|
|
235
|
-
/**
|
|
236
|
-
* Store a batch of data points. Used by sync workflows.
|
|
237
|
-
* Must complete within 1 second — caller is responsible for batch sizing.
|
|
238
|
-
*/
|
|
239
556
|
export const storeBatch = internalMutation({
|
|
240
557
|
args: {
|
|
241
558
|
dataSourceId: v.id("dataSources"),
|
|
@@ -250,56 +567,1110 @@ export const storeBatch = internalMutation({
|
|
|
250
567
|
},
|
|
251
568
|
returns: v.number(),
|
|
252
569
|
handler: async (ctx, args) => {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
.query("dataPoints")
|
|
258
|
-
.withIndex("by_source_type_time", (idx) =>
|
|
259
|
-
idx
|
|
260
|
-
.eq("dataSourceId", args.dataSourceId)
|
|
261
|
-
.eq("seriesType", args.seriesType)
|
|
262
|
-
.eq("recordedAt", point.recordedAt),
|
|
263
|
-
)
|
|
264
|
-
.first();
|
|
570
|
+
const result = await storePointsWithPolicy(ctx, args);
|
|
571
|
+
return result.processedCount;
|
|
572
|
+
},
|
|
573
|
+
});
|
|
265
574
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
575
|
+
export const runTimeSeriesMaintenance = internalMutation({
|
|
576
|
+
args: {},
|
|
577
|
+
returns: v.null(),
|
|
578
|
+
handler: async (ctx) => {
|
|
579
|
+
const settingsDoc = await ensureTimeSeriesPolicySettingsDoc(ctx.db);
|
|
580
|
+
const now = Date.now();
|
|
581
|
+
|
|
582
|
+
if (!settingsDoc.maintenanceEnabled) {
|
|
583
|
+
await ctx.db.patch(settingsDoc._id, {
|
|
584
|
+
scheduledAt: undefined,
|
|
585
|
+
lastRunAt: now,
|
|
586
|
+
lastError: undefined,
|
|
587
|
+
updatedAt: now,
|
|
588
|
+
});
|
|
589
|
+
return null;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
await ctx.db.patch(settingsDoc._id, {
|
|
593
|
+
scheduledAt: undefined,
|
|
594
|
+
lastRunAt: now,
|
|
595
|
+
lastError: undefined,
|
|
596
|
+
updatedAt: now,
|
|
597
|
+
});
|
|
598
|
+
|
|
599
|
+
let lastError: string | undefined;
|
|
600
|
+
const dueStates = await ctx.db
|
|
601
|
+
.query("timeSeriesSeriesState")
|
|
602
|
+
.withIndex("by_next_maintenance", (idx) => idx.lte("nextMaintenanceAt", now))
|
|
603
|
+
.take(MAINTENANCE_BATCH_SIZE);
|
|
604
|
+
|
|
605
|
+
for (const state of dueStates) {
|
|
606
|
+
try {
|
|
607
|
+
await maintainSeriesState(ctx.db, state, now);
|
|
608
|
+
} catch (error) {
|
|
609
|
+
lastError = error instanceof Error ? error.message : String(error);
|
|
276
610
|
}
|
|
277
|
-
count++;
|
|
278
611
|
}
|
|
279
|
-
|
|
612
|
+
|
|
613
|
+
const refreshed = await ensureTimeSeriesPolicySettingsDoc(ctx.db);
|
|
614
|
+
const hasBacklog = dueStates.length === MAINTENANCE_BATCH_SIZE;
|
|
615
|
+
const delayMs = hasBacklog
|
|
616
|
+
? Math.min(refreshed.maintenanceIntervalMs, 60 * 1000)
|
|
617
|
+
: refreshed.maintenanceIntervalMs;
|
|
618
|
+
const scheduledAt = now + delayMs;
|
|
619
|
+
|
|
620
|
+
await ctx.scheduler.runAfter(delayMs, internal.dataPoints.runTimeSeriesMaintenance, {});
|
|
621
|
+
await ctx.db.patch(refreshed._id, {
|
|
622
|
+
scheduledAt,
|
|
623
|
+
lastRunAt: now,
|
|
624
|
+
lastError,
|
|
625
|
+
updatedAt: now,
|
|
626
|
+
});
|
|
627
|
+
|
|
628
|
+
return null;
|
|
280
629
|
},
|
|
281
630
|
});
|
|
282
631
|
|
|
283
|
-
/**
|
|
284
|
-
* Delete all data points for a data source. Used during user/connection cleanup.
|
|
285
|
-
*/
|
|
286
632
|
export const deleteByDataSource = internalMutation({
|
|
287
633
|
args: { dataSourceId: v.id("dataSources") },
|
|
634
|
+
returns: v.null(),
|
|
288
635
|
handler: async (ctx, args) => {
|
|
289
|
-
|
|
290
|
-
let batch = await ctx.db
|
|
636
|
+
let rawBatch = await ctx.db
|
|
291
637
|
.query("dataPoints")
|
|
292
638
|
.withIndex("by_source_type_time", (idx) => idx.eq("dataSourceId", args.dataSourceId))
|
|
293
639
|
.take(1000);
|
|
294
640
|
|
|
295
|
-
while (
|
|
296
|
-
for (const
|
|
297
|
-
await ctx.db.delete(
|
|
641
|
+
while (rawBatch.length > 0) {
|
|
642
|
+
for (const point of rawBatch) {
|
|
643
|
+
await ctx.db.delete(point._id);
|
|
298
644
|
}
|
|
299
|
-
|
|
645
|
+
rawBatch = await ctx.db
|
|
300
646
|
.query("dataPoints")
|
|
301
647
|
.withIndex("by_source_type_time", (idx) => idx.eq("dataSourceId", args.dataSourceId))
|
|
302
648
|
.take(1000);
|
|
303
649
|
}
|
|
650
|
+
|
|
651
|
+
let rollupBatch = await ctx.db
|
|
652
|
+
.query("timeSeriesRollups")
|
|
653
|
+
.withIndex("by_source_type_bucket", (idx) => idx.eq("dataSourceId", args.dataSourceId))
|
|
654
|
+
.take(1000);
|
|
655
|
+
|
|
656
|
+
while (rollupBatch.length > 0) {
|
|
657
|
+
for (const rollup of rollupBatch) {
|
|
658
|
+
await ctx.db.delete(rollup._id);
|
|
659
|
+
}
|
|
660
|
+
rollupBatch = await ctx.db
|
|
661
|
+
.query("timeSeriesRollups")
|
|
662
|
+
.withIndex("by_source_type_bucket", (idx) => idx.eq("dataSourceId", args.dataSourceId))
|
|
663
|
+
.take(1000);
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
const state = await ctx.db
|
|
667
|
+
.query("timeSeriesSeriesState")
|
|
668
|
+
.withIndex("by_source_series", (idx) => idx.eq("dataSourceId", args.dataSourceId))
|
|
669
|
+
.collect();
|
|
670
|
+
for (const doc of state) {
|
|
671
|
+
await ctx.db.delete(doc._id);
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
return null;
|
|
304
675
|
},
|
|
305
676
|
});
|
|
677
|
+
|
|
678
|
+
// ---------------------------------------------------------------------------
|
|
679
|
+
// Policy helpers
|
|
680
|
+
// ---------------------------------------------------------------------------
|
|
681
|
+
|
|
682
|
+
async function getTimeSeriesPolicySettings(db: {
|
|
683
|
+
query: DatabaseReader["query"];
|
|
684
|
+
}): Promise<PolicySettingsDoc> {
|
|
685
|
+
const existing = await db
|
|
686
|
+
.query("timeSeriesPolicySettings")
|
|
687
|
+
.withIndex("by_key", (idx) => idx.eq("key", MAINTENANCE_SETTINGS_KEY))
|
|
688
|
+
.first();
|
|
689
|
+
|
|
690
|
+
return (
|
|
691
|
+
existing ?? {
|
|
692
|
+
key: MAINTENANCE_SETTINGS_KEY,
|
|
693
|
+
maintenanceEnabled: true,
|
|
694
|
+
maintenanceIntervalMs: DEFAULT_MAINTENANCE_INTERVAL_MS,
|
|
695
|
+
updatedAt: 0,
|
|
696
|
+
}
|
|
697
|
+
);
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
async function ensureTimeSeriesPolicySettingsDoc(db: {
|
|
701
|
+
query: DatabaseWriter["query"];
|
|
702
|
+
insert: DatabaseWriter["insert"];
|
|
703
|
+
get: DatabaseWriter["get"];
|
|
704
|
+
}) {
|
|
705
|
+
const existing = await db
|
|
706
|
+
.query("timeSeriesPolicySettings")
|
|
707
|
+
.withIndex("by_key", (idx) => idx.eq("key", MAINTENANCE_SETTINGS_KEY))
|
|
708
|
+
.first();
|
|
709
|
+
if (existing) {
|
|
710
|
+
return existing;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
const id = await db.insert("timeSeriesPolicySettings", {
|
|
714
|
+
key: MAINTENANCE_SETTINGS_KEY,
|
|
715
|
+
maintenanceEnabled: true,
|
|
716
|
+
maintenanceIntervalMs: DEFAULT_MAINTENANCE_INTERVAL_MS,
|
|
717
|
+
updatedAt: Date.now(),
|
|
718
|
+
});
|
|
719
|
+
const inserted = await db.get(id);
|
|
720
|
+
if (!inserted) {
|
|
721
|
+
throw new Error("Failed to create time-series policy settings");
|
|
722
|
+
}
|
|
723
|
+
return inserted;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
async function upsertTimeSeriesPolicySettings(
|
|
727
|
+
ctx: {
|
|
728
|
+
db: Pick<DatabaseWriter, "get" | "insert" | "patch" | "query">;
|
|
729
|
+
},
|
|
730
|
+
maintenance?: {
|
|
731
|
+
enabled?: boolean;
|
|
732
|
+
interval?: DurationInput;
|
|
733
|
+
},
|
|
734
|
+
) {
|
|
735
|
+
const existing = await ensureTimeSeriesPolicySettingsDoc(ctx.db);
|
|
736
|
+
const patch = {
|
|
737
|
+
maintenanceEnabled: maintenance?.enabled ?? existing.maintenanceEnabled,
|
|
738
|
+
maintenanceIntervalMs:
|
|
739
|
+
maintenance?.interval !== undefined
|
|
740
|
+
? parseDurationInput(maintenance.interval, "maintenance.interval")
|
|
741
|
+
: existing.maintenanceIntervalMs,
|
|
742
|
+
updatedAt: Date.now(),
|
|
743
|
+
};
|
|
744
|
+
|
|
745
|
+
await ctx.db.patch(existing._id, patch);
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
async function resolveEffectivePolicy(
|
|
749
|
+
db: {
|
|
750
|
+
query: TimeSeriesReadDb["query"];
|
|
751
|
+
},
|
|
752
|
+
userId: string,
|
|
753
|
+
provider: string,
|
|
754
|
+
seriesType: string,
|
|
755
|
+
): Promise<EffectivePolicy> {
|
|
756
|
+
const assignment = await db
|
|
757
|
+
.query("timeSeriesPolicyAssignments")
|
|
758
|
+
.withIndex("by_user", (idx) => idx.eq("userId", userId))
|
|
759
|
+
.first();
|
|
760
|
+
|
|
761
|
+
if (assignment) {
|
|
762
|
+
const presetRules = (await db
|
|
763
|
+
.query("timeSeriesPolicyRules")
|
|
764
|
+
.withIndex("by_set", (idx) =>
|
|
765
|
+
idx.eq("policySetKind", "preset").eq("policySetKey", assignment.presetKey),
|
|
766
|
+
)
|
|
767
|
+
.collect()) as StoredPolicyRule[];
|
|
768
|
+
const presetMatch = resolveScopedPolicyRule(presetRules, provider, seriesType);
|
|
769
|
+
|
|
770
|
+
if (presetMatch) {
|
|
771
|
+
return {
|
|
772
|
+
provider: presetMatch.provider,
|
|
773
|
+
seriesType: presetMatch.seriesType,
|
|
774
|
+
tiers: presetMatch.tiers,
|
|
775
|
+
sourceKind: "preset",
|
|
776
|
+
sourceKey: assignment.presetKey,
|
|
777
|
+
matchedScope: inferTimeSeriesPolicyScope(presetMatch.provider, presetMatch.seriesType),
|
|
778
|
+
};
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
const defaultRules = (await db
|
|
783
|
+
.query("timeSeriesPolicyRules")
|
|
784
|
+
.withIndex("by_set", (idx) =>
|
|
785
|
+
idx.eq("policySetKind", "default").eq("policySetKey", DEFAULT_POLICY_SET_KEY),
|
|
786
|
+
)
|
|
787
|
+
.collect()) as StoredPolicyRule[];
|
|
788
|
+
const defaultMatch = resolveScopedPolicyRule(defaultRules, provider, seriesType);
|
|
789
|
+
|
|
790
|
+
if (defaultMatch) {
|
|
791
|
+
return {
|
|
792
|
+
provider: defaultMatch.provider,
|
|
793
|
+
seriesType: defaultMatch.seriesType,
|
|
794
|
+
tiers: defaultMatch.tiers,
|
|
795
|
+
sourceKind: "default",
|
|
796
|
+
sourceKey: DEFAULT_POLICY_SET_KEY,
|
|
797
|
+
matchedScope: inferTimeSeriesPolicyScope(defaultMatch.provider, defaultMatch.seriesType),
|
|
798
|
+
};
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
return {
|
|
802
|
+
tiers: buildBuiltinFullTiers(),
|
|
803
|
+
sourceKind: "builtin",
|
|
804
|
+
sourceKey: null,
|
|
805
|
+
matchedScope: "default",
|
|
806
|
+
};
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
function formatTierForResponse(tier: NormalizedTimeSeriesTier) {
|
|
810
|
+
if (tier.kind === "raw") {
|
|
811
|
+
return {
|
|
812
|
+
kind: "raw",
|
|
813
|
+
fromAgeMs: tier.fromAgeMs,
|
|
814
|
+
toAgeMs: tier.toAgeMs,
|
|
815
|
+
};
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
return {
|
|
819
|
+
kind: "rollup",
|
|
820
|
+
fromAgeMs: tier.fromAgeMs,
|
|
821
|
+
toAgeMs: tier.toAgeMs,
|
|
822
|
+
bucketMs: tier.bucketMs,
|
|
823
|
+
aggregations: tier.aggregations,
|
|
824
|
+
};
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
function formatPolicyRulesForResponse(rules: StoredPolicyRule[]) {
|
|
828
|
+
return rules
|
|
829
|
+
.map((rule) => ({
|
|
830
|
+
_id: rule._id,
|
|
831
|
+
provider: rule.provider,
|
|
832
|
+
seriesType: rule.seriesType,
|
|
833
|
+
scope: inferTimeSeriesPolicyScope(rule.provider, rule.seriesType),
|
|
834
|
+
policySetKind: rule.policySetKind,
|
|
835
|
+
policySetKey: rule.policySetKey,
|
|
836
|
+
tiers: rule.tiers.map(formatTierForResponse),
|
|
837
|
+
updatedAt: rule.updatedAt,
|
|
838
|
+
}))
|
|
839
|
+
.sort(comparePolicyScopes);
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
function groupPolicyRulesByPreset(rules: StoredPolicyRule[]) {
|
|
843
|
+
const grouped = new Map<string, StoredPolicyRule[]>();
|
|
844
|
+
for (const rule of rules) {
|
|
845
|
+
const existing = grouped.get(rule.policySetKey) ?? [];
|
|
846
|
+
existing.push(rule);
|
|
847
|
+
grouped.set(rule.policySetKey, existing);
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
return Array.from(grouped.entries())
|
|
851
|
+
.sort((a, b) => a[0].localeCompare(b[0]))
|
|
852
|
+
.map(([key, groupedRules]) => ({
|
|
853
|
+
key,
|
|
854
|
+
rules: formatPolicyRulesForResponse(groupedRules),
|
|
855
|
+
}));
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
function policyRequiresMaintenance(tiers: NormalizedTimeSeriesTier[]) {
|
|
859
|
+
const rawTier = getRawTier(tiers);
|
|
860
|
+
if (!rawTier) {
|
|
861
|
+
return true;
|
|
862
|
+
}
|
|
863
|
+
if (tiers.length > 1) {
|
|
864
|
+
return true;
|
|
865
|
+
}
|
|
866
|
+
return rawTier.toAgeMs !== null;
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
// ---------------------------------------------------------------------------
|
|
870
|
+
// Write path
|
|
871
|
+
// ---------------------------------------------------------------------------
|
|
872
|
+
|
|
873
|
+
async function storePointsWithPolicy(
|
|
874
|
+
ctx: TimeSeriesMutationContext,
|
|
875
|
+
args: {
|
|
876
|
+
dataSourceId: Id<"dataSources">;
|
|
877
|
+
seriesType: string;
|
|
878
|
+
points: StoredPointInput[];
|
|
879
|
+
},
|
|
880
|
+
) {
|
|
881
|
+
const dataSource = await ctx.db.get(args.dataSourceId);
|
|
882
|
+
if (!dataSource) {
|
|
883
|
+
throw new Error(`Data source ${args.dataSourceId} not found`);
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
const effectivePolicy = await resolveEffectivePolicy(
|
|
887
|
+
ctx.db,
|
|
888
|
+
dataSource.userId,
|
|
889
|
+
dataSource.provider,
|
|
890
|
+
args.seriesType,
|
|
891
|
+
);
|
|
892
|
+
const settings = await getTimeSeriesPolicySettings(ctx.db);
|
|
893
|
+
const now = Date.now();
|
|
894
|
+
const points = dedupeIncomingPoints(args.points);
|
|
895
|
+
|
|
896
|
+
const rawPoints: StoredPointInput[] = [];
|
|
897
|
+
const rollupGroups = new Map<
|
|
898
|
+
string,
|
|
899
|
+
{ bucketMs: number; bucketStart: number; stats: AggregatedStats }
|
|
900
|
+
>();
|
|
901
|
+
|
|
902
|
+
for (const point of points) {
|
|
903
|
+
const ageMs = Math.max(0, now - point.recordedAt);
|
|
904
|
+
const destinationTier = findTierForAge(effectivePolicy.tiers, ageMs);
|
|
905
|
+
if (!destinationTier) {
|
|
906
|
+
continue;
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
if (destinationTier.kind === "raw") {
|
|
910
|
+
rawPoints.push(point);
|
|
911
|
+
continue;
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
addRawPointToRollupGroup(rollupGroups, point, destinationTier.bucketMs);
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
let lastRawId: Id<"dataPoints"> | null = null;
|
|
918
|
+
if (rawPoints.length > 0) {
|
|
919
|
+
lastRawId = await upsertRawPoints(ctx.db, args.dataSourceId, args.seriesType, rawPoints);
|
|
920
|
+
}
|
|
921
|
+
if (rollupGroups.size > 0) {
|
|
922
|
+
await upsertRollupStatsGroups(
|
|
923
|
+
ctx.db,
|
|
924
|
+
args.dataSourceId,
|
|
925
|
+
args.seriesType,
|
|
926
|
+
rollupGroups.values(),
|
|
927
|
+
);
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
await upsertSeriesState(ctx.db, {
|
|
931
|
+
dataSourceId: dataSource._id,
|
|
932
|
+
connectionId: dataSource.connectionId,
|
|
933
|
+
userId: dataSource.userId,
|
|
934
|
+
provider: dataSource.provider,
|
|
935
|
+
seriesType: args.seriesType,
|
|
936
|
+
latestRecordedAt: points.length > 0 ? points[points.length - 1].recordedAt : now,
|
|
937
|
+
nextMaintenanceAt: policyRequiresMaintenance(effectivePolicy.tiers)
|
|
938
|
+
? now + settings.maintenanceIntervalMs
|
|
939
|
+
: now + LONG_IDLE_MAINTENANCE_MS,
|
|
940
|
+
});
|
|
941
|
+
|
|
942
|
+
if (policyRequiresMaintenance(effectivePolicy.tiers)) {
|
|
943
|
+
await maintainSeriesState(
|
|
944
|
+
ctx.db,
|
|
945
|
+
{
|
|
946
|
+
dataSourceId: dataSource._id,
|
|
947
|
+
connectionId: dataSource.connectionId,
|
|
948
|
+
userId: dataSource.userId,
|
|
949
|
+
provider: dataSource.provider,
|
|
950
|
+
seriesType: args.seriesType,
|
|
951
|
+
},
|
|
952
|
+
now,
|
|
953
|
+
);
|
|
954
|
+
await ensureTimeSeriesMaintenanceScheduled(ctx);
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
return {
|
|
958
|
+
processedCount: points.length,
|
|
959
|
+
lastRawId,
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
function dedupeIncomingPoints(points: StoredPointInput[]) {
|
|
964
|
+
const deduped = new Map<number, StoredPointInput>();
|
|
965
|
+
for (const point of points) {
|
|
966
|
+
deduped.set(point.recordedAt, point);
|
|
967
|
+
}
|
|
968
|
+
return Array.from(deduped.values()).sort((a, b) => a.recordedAt - b.recordedAt);
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
async function upsertRawPoints(
|
|
972
|
+
db: TimeSeriesWriteDb,
|
|
973
|
+
dataSourceId: Id<"dataSources">,
|
|
974
|
+
seriesType: string,
|
|
975
|
+
points: StoredPointInput[],
|
|
976
|
+
) {
|
|
977
|
+
let lastId: Id<"dataPoints"> | null = null;
|
|
978
|
+
|
|
979
|
+
for (const point of points) {
|
|
980
|
+
const existing = await db
|
|
981
|
+
.query("dataPoints")
|
|
982
|
+
.withIndex("by_source_type_time", (idx) =>
|
|
983
|
+
idx
|
|
984
|
+
.eq("dataSourceId", dataSourceId)
|
|
985
|
+
.eq("seriesType", seriesType)
|
|
986
|
+
.eq("recordedAt", point.recordedAt),
|
|
987
|
+
)
|
|
988
|
+
.first();
|
|
989
|
+
|
|
990
|
+
if (existing) {
|
|
991
|
+
await db.patch(existing._id, {
|
|
992
|
+
value: point.value,
|
|
993
|
+
externalId: point.externalId,
|
|
994
|
+
});
|
|
995
|
+
lastId = existing._id;
|
|
996
|
+
} else {
|
|
997
|
+
lastId = await db.insert("dataPoints", {
|
|
998
|
+
dataSourceId,
|
|
999
|
+
seriesType,
|
|
1000
|
+
recordedAt: point.recordedAt,
|
|
1001
|
+
value: point.value,
|
|
1002
|
+
externalId: point.externalId,
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
return lastId;
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
function addRawPointToRollupGroup(
|
|
1011
|
+
groups: Map<string, { bucketMs: number; bucketStart: number; stats: AggregatedStats }>,
|
|
1012
|
+
point: StoredPointInput,
|
|
1013
|
+
bucketMs: number,
|
|
1014
|
+
) {
|
|
1015
|
+
const bucketStart = getBucketStart(point.recordedAt, bucketMs);
|
|
1016
|
+
const key = `${bucketMs}:${bucketStart}`;
|
|
1017
|
+
const existing = groups.get(key);
|
|
1018
|
+
const stats = aggregateRawPoints([
|
|
1019
|
+
{
|
|
1020
|
+
recordedAt: point.recordedAt,
|
|
1021
|
+
value: point.value,
|
|
1022
|
+
},
|
|
1023
|
+
]);
|
|
1024
|
+
|
|
1025
|
+
if (existing) {
|
|
1026
|
+
existing.stats = combineAggregatedStats([existing.stats, stats]);
|
|
1027
|
+
} else {
|
|
1028
|
+
groups.set(key, {
|
|
1029
|
+
bucketMs,
|
|
1030
|
+
bucketStart,
|
|
1031
|
+
stats,
|
|
1032
|
+
});
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
async function upsertRollupStatsGroups(
|
|
1037
|
+
db: TimeSeriesWriteDb,
|
|
1038
|
+
dataSourceId: Id<"dataSources">,
|
|
1039
|
+
seriesType: string,
|
|
1040
|
+
groups: Iterable<{ bucketMs: number; bucketStart: number; stats: AggregatedStats }>,
|
|
1041
|
+
) {
|
|
1042
|
+
for (const group of groups) {
|
|
1043
|
+
const existing = await db
|
|
1044
|
+
.query("timeSeriesRollups")
|
|
1045
|
+
.withIndex("by_source_type_bucket_size", (idx) =>
|
|
1046
|
+
idx
|
|
1047
|
+
.eq("dataSourceId", dataSourceId)
|
|
1048
|
+
.eq("seriesType", seriesType)
|
|
1049
|
+
.eq("bucketMs", group.bucketMs)
|
|
1050
|
+
.eq("bucketStart", group.bucketStart),
|
|
1051
|
+
)
|
|
1052
|
+
.first();
|
|
1053
|
+
|
|
1054
|
+
const combined = existing
|
|
1055
|
+
? combineAggregatedStats([rollupDocToStats(existing), group.stats])
|
|
1056
|
+
: group.stats;
|
|
1057
|
+
const patch = {
|
|
1058
|
+
dataSourceId,
|
|
1059
|
+
seriesType,
|
|
1060
|
+
bucketMs: group.bucketMs,
|
|
1061
|
+
bucketStart: group.bucketStart,
|
|
1062
|
+
bucketEnd: getBucketEnd(group.bucketStart, group.bucketMs),
|
|
1063
|
+
avg: combined.avg,
|
|
1064
|
+
min: combined.min,
|
|
1065
|
+
max: combined.max,
|
|
1066
|
+
last: combined.last,
|
|
1067
|
+
lastRecordedAt: combined.lastRecordedAt,
|
|
1068
|
+
count: combined.count,
|
|
1069
|
+
updatedAt: Date.now(),
|
|
1070
|
+
};
|
|
1071
|
+
|
|
1072
|
+
if (existing) {
|
|
1073
|
+
await db.patch(existing._id, patch);
|
|
1074
|
+
} else {
|
|
1075
|
+
await db.insert("timeSeriesRollups", patch);
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
async function upsertSeriesState(
|
|
1081
|
+
db: TimeSeriesWriteDb,
|
|
1082
|
+
args: {
|
|
1083
|
+
dataSourceId: Id<"dataSources">;
|
|
1084
|
+
connectionId?: Id<"connections">;
|
|
1085
|
+
userId: string;
|
|
1086
|
+
provider: Doc<"dataSources">["provider"];
|
|
1087
|
+
seriesType: string;
|
|
1088
|
+
latestRecordedAt: number;
|
|
1089
|
+
nextMaintenanceAt: number;
|
|
1090
|
+
},
|
|
1091
|
+
) {
|
|
1092
|
+
const existing = await db
|
|
1093
|
+
.query("timeSeriesSeriesState")
|
|
1094
|
+
.withIndex("by_source_series", (idx) =>
|
|
1095
|
+
idx.eq("dataSourceId", args.dataSourceId).eq("seriesType", args.seriesType),
|
|
1096
|
+
)
|
|
1097
|
+
.first();
|
|
1098
|
+
|
|
1099
|
+
const patch = {
|
|
1100
|
+
dataSourceId: args.dataSourceId,
|
|
1101
|
+
connectionId: args.connectionId,
|
|
1102
|
+
userId: args.userId,
|
|
1103
|
+
provider: args.provider,
|
|
1104
|
+
seriesType: args.seriesType,
|
|
1105
|
+
latestRecordedAt: existing
|
|
1106
|
+
? Math.max(existing.latestRecordedAt, args.latestRecordedAt)
|
|
1107
|
+
: args.latestRecordedAt,
|
|
1108
|
+
lastIngestedAt: Date.now(),
|
|
1109
|
+
nextMaintenanceAt: args.nextMaintenanceAt,
|
|
1110
|
+
updatedAt: Date.now(),
|
|
1111
|
+
};
|
|
1112
|
+
|
|
1113
|
+
if (existing) {
|
|
1114
|
+
await db.patch(existing._id, patch);
|
|
1115
|
+
return existing._id;
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
return await db.insert("timeSeriesSeriesState", patch);
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
// ---------------------------------------------------------------------------
|
|
1122
|
+
// Maintenance
|
|
1123
|
+
// ---------------------------------------------------------------------------
|
|
1124
|
+
|
|
1125
|
+
async function ensureTimeSeriesMaintenanceScheduled(ctx: TimeSeriesMutationContext) {
|
|
1126
|
+
const settings = await ensureTimeSeriesPolicySettingsDoc(ctx.db);
|
|
1127
|
+
if (!settings.maintenanceEnabled) {
|
|
1128
|
+
return;
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
const now = Date.now();
|
|
1132
|
+
if (settings.scheduledAt !== undefined && settings.scheduledAt > now) {
|
|
1133
|
+
return;
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
const delayMs = settings.maintenanceIntervalMs;
|
|
1137
|
+
const scheduledAt = now + delayMs;
|
|
1138
|
+
await ctx.scheduler.runAfter(delayMs, internal.dataPoints.runTimeSeriesMaintenance, {});
|
|
1139
|
+
await ctx.db.patch(settings._id, {
|
|
1140
|
+
scheduledAt,
|
|
1141
|
+
updatedAt: now,
|
|
1142
|
+
});
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
async function markAllSeriesStateDue(ctx: TimeSeriesMutationContext, dueAt: number) {
|
|
1146
|
+
const states = await ctx.db.query("timeSeriesSeriesState").collect();
|
|
1147
|
+
|
|
1148
|
+
for (const state of states) {
|
|
1149
|
+
await ctx.db.patch(state._id, {
|
|
1150
|
+
nextMaintenanceAt: dueAt,
|
|
1151
|
+
updatedAt: Date.now(),
|
|
1152
|
+
});
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
async function markSeriesStateDueForUser(
|
|
1157
|
+
ctx: TimeSeriesMutationContext,
|
|
1158
|
+
userId: string,
|
|
1159
|
+
dueAt: number,
|
|
1160
|
+
) {
|
|
1161
|
+
const batch = await ctx.db
|
|
1162
|
+
.query("timeSeriesSeriesState")
|
|
1163
|
+
.withIndex("by_user", (idx) => idx.eq("userId", userId))
|
|
1164
|
+
.collect();
|
|
1165
|
+
|
|
1166
|
+
for (const state of batch) {
|
|
1167
|
+
await ctx.db.patch(state._id, {
|
|
1168
|
+
nextMaintenanceAt: dueAt,
|
|
1169
|
+
updatedAt: Date.now(),
|
|
1170
|
+
});
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
async function maintainSeriesState(
|
|
1175
|
+
db: TimeSeriesWriteDb,
|
|
1176
|
+
state: Pick<
|
|
1177
|
+
StoredSeriesState,
|
|
1178
|
+
"connectionId" | "dataSourceId" | "userId" | "provider" | "seriesType"
|
|
1179
|
+
>,
|
|
1180
|
+
now: number,
|
|
1181
|
+
) {
|
|
1182
|
+
const source = await db.get(state.dataSourceId);
|
|
1183
|
+
const storedState = await db
|
|
1184
|
+
.query("timeSeriesSeriesState")
|
|
1185
|
+
.withIndex("by_source_series", (idx) =>
|
|
1186
|
+
idx.eq("dataSourceId", state.dataSourceId).eq("seriesType", state.seriesType),
|
|
1187
|
+
)
|
|
1188
|
+
.first();
|
|
1189
|
+
|
|
1190
|
+
if (!source || !storedState) {
|
|
1191
|
+
if (storedState) {
|
|
1192
|
+
await db.delete(storedState._id);
|
|
1193
|
+
}
|
|
1194
|
+
return;
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
const effective = await resolveEffectivePolicy(
|
|
1198
|
+
db,
|
|
1199
|
+
storedState.userId,
|
|
1200
|
+
storedState.provider,
|
|
1201
|
+
storedState.seriesType,
|
|
1202
|
+
);
|
|
1203
|
+
await compactRawPoints(db, storedState, effective.tiers, now);
|
|
1204
|
+
await compactRollupPoints(db, storedState, effective.tiers, now);
|
|
1205
|
+
|
|
1206
|
+
const stillHasData = await sourceSeriesHasData(
|
|
1207
|
+
db,
|
|
1208
|
+
storedState.dataSourceId,
|
|
1209
|
+
storedState.seriesType,
|
|
1210
|
+
);
|
|
1211
|
+
if (!stillHasData) {
|
|
1212
|
+
await db.delete(storedState._id);
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
const settings = await getTimeSeriesPolicySettings(db);
|
|
1217
|
+
await db.patch(storedState._id, {
|
|
1218
|
+
nextMaintenanceAt: policyRequiresMaintenance(effective.tiers)
|
|
1219
|
+
? now + settings.maintenanceIntervalMs
|
|
1220
|
+
: now + LONG_IDLE_MAINTENANCE_MS,
|
|
1221
|
+
lastMaintenanceAt: now,
|
|
1222
|
+
updatedAt: now,
|
|
1223
|
+
});
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
async function compactRawPoints(
|
|
1227
|
+
db: TimeSeriesWriteDb,
|
|
1228
|
+
state: StoredSeriesState,
|
|
1229
|
+
tiers: NormalizedTimeSeriesTier[],
|
|
1230
|
+
now: number,
|
|
1231
|
+
) {
|
|
1232
|
+
const rawTier = getRawTier(tiers);
|
|
1233
|
+
const rawCutoff = rawTier?.toAgeMs ?? null;
|
|
1234
|
+
const query =
|
|
1235
|
+
rawCutoff === null
|
|
1236
|
+
? rawTier
|
|
1237
|
+
? []
|
|
1238
|
+
: await db
|
|
1239
|
+
.query("dataPoints")
|
|
1240
|
+
.withIndex("by_source_type_time", (idx) =>
|
|
1241
|
+
idx.eq("dataSourceId", state.dataSourceId).eq("seriesType", state.seriesType),
|
|
1242
|
+
)
|
|
1243
|
+
.take(MAINTENANCE_POINT_BATCH_SIZE)
|
|
1244
|
+
: await db
|
|
1245
|
+
.query("dataPoints")
|
|
1246
|
+
.withIndex("by_source_type_time", (idx) =>
|
|
1247
|
+
idx
|
|
1248
|
+
.eq("dataSourceId", state.dataSourceId)
|
|
1249
|
+
.eq("seriesType", state.seriesType)
|
|
1250
|
+
.lt("recordedAt", now - rawCutoff),
|
|
1251
|
+
)
|
|
1252
|
+
.take(MAINTENANCE_POINT_BATCH_SIZE);
|
|
1253
|
+
|
|
1254
|
+
if (query.length === 0) {
|
|
1255
|
+
return;
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
const rollupGroups = new Map<
|
|
1259
|
+
string,
|
|
1260
|
+
{ bucketMs: number; bucketStart: number; stats: AggregatedStats }
|
|
1261
|
+
>();
|
|
1262
|
+
const toDelete: Id<"dataPoints">[] = [];
|
|
1263
|
+
|
|
1264
|
+
for (const point of query as Doc<"dataPoints">[]) {
|
|
1265
|
+
const ageMs = Math.max(0, now - point.recordedAt);
|
|
1266
|
+
const destinationTier = findTierForAge(tiers, ageMs);
|
|
1267
|
+
if (!destinationTier) {
|
|
1268
|
+
toDelete.push(point._id);
|
|
1269
|
+
continue;
|
|
1270
|
+
}
|
|
1271
|
+
|
|
1272
|
+
if (destinationTier.kind === "raw") {
|
|
1273
|
+
continue;
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
const bucketStart = getBucketStart(point.recordedAt, destinationTier.bucketMs);
|
|
1277
|
+
const bucketEnd = getBucketEnd(bucketStart, destinationTier.bucketMs);
|
|
1278
|
+
if (bucketEnd > now - destinationTier.fromAgeMs) {
|
|
1279
|
+
continue;
|
|
1280
|
+
}
|
|
1281
|
+
|
|
1282
|
+
addRawPointToRollupGroup(rollupGroups, point, destinationTier.bucketMs);
|
|
1283
|
+
toDelete.push(point._id);
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
if (rollupGroups.size > 0) {
|
|
1287
|
+
await upsertRollupStatsGroups(db, state.dataSourceId, state.seriesType, rollupGroups.values());
|
|
1288
|
+
}
|
|
1289
|
+
for (const pointId of toDelete) {
|
|
1290
|
+
await db.delete(pointId);
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
async function compactRollupPoints(
|
|
1295
|
+
db: TimeSeriesWriteDb,
|
|
1296
|
+
state: StoredSeriesState,
|
|
1297
|
+
tiers: NormalizedTimeSeriesTier[],
|
|
1298
|
+
now: number,
|
|
1299
|
+
) {
|
|
1300
|
+
const rollups = await db
|
|
1301
|
+
.query("timeSeriesRollups")
|
|
1302
|
+
.withIndex("by_source_type_bucket", (idx) =>
|
|
1303
|
+
idx.eq("dataSourceId", state.dataSourceId).eq("seriesType", state.seriesType),
|
|
1304
|
+
)
|
|
1305
|
+
.take(MAINTENANCE_ROLLUP_BATCH_SIZE);
|
|
1306
|
+
|
|
1307
|
+
if (rollups.length === 0) {
|
|
1308
|
+
return;
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
const destinationGroups = new Map<
|
|
1312
|
+
string,
|
|
1313
|
+
{ bucketMs: number; bucketStart: number; stats: AggregatedStats }
|
|
1314
|
+
>();
|
|
1315
|
+
const toDelete: Id<"timeSeriesRollups">[] = [];
|
|
1316
|
+
|
|
1317
|
+
for (const rollup of rollups as StoredRollup[]) {
|
|
1318
|
+
const ageMs = Math.max(0, now - rollup.bucketEnd);
|
|
1319
|
+
const destinationTier = findTierForAge(tiers, ageMs);
|
|
1320
|
+
|
|
1321
|
+
if (!destinationTier) {
|
|
1322
|
+
toDelete.push(rollup._id);
|
|
1323
|
+
continue;
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
if (destinationTier.kind === "raw") {
|
|
1327
|
+
continue;
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
const destinationBucketStart = getBucketStart(rollup.bucketStart, destinationTier.bucketMs);
|
|
1331
|
+
const destinationBucketEnd = getBucketEnd(destinationBucketStart, destinationTier.bucketMs);
|
|
1332
|
+
|
|
1333
|
+
if (destinationTier.bucketMs === rollup.bucketMs) {
|
|
1334
|
+
continue;
|
|
1335
|
+
}
|
|
1336
|
+
if (destinationBucketEnd > now - destinationTier.fromAgeMs) {
|
|
1337
|
+
continue;
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
addStatsToRollupGroup(
|
|
1341
|
+
destinationGroups,
|
|
1342
|
+
destinationTier.bucketMs,
|
|
1343
|
+
destinationBucketStart,
|
|
1344
|
+
rollupDocToStats(rollup),
|
|
1345
|
+
);
|
|
1346
|
+
toDelete.push(rollup._id);
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
if (destinationGroups.size > 0) {
|
|
1350
|
+
await upsertRollupStatsGroups(
|
|
1351
|
+
db,
|
|
1352
|
+
state.dataSourceId,
|
|
1353
|
+
state.seriesType,
|
|
1354
|
+
destinationGroups.values(),
|
|
1355
|
+
);
|
|
1356
|
+
}
|
|
1357
|
+
for (const rollupId of toDelete) {
|
|
1358
|
+
await db.delete(rollupId);
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1362
|
+
async function sourceSeriesHasData(
|
|
1363
|
+
db: Pick<TimeSeriesWriteDb, "query">,
|
|
1364
|
+
dataSourceId: Id<"dataSources">,
|
|
1365
|
+
seriesType: string,
|
|
1366
|
+
) {
|
|
1367
|
+
const raw = await db
|
|
1368
|
+
.query("dataPoints")
|
|
1369
|
+
.withIndex("by_source_type_time", (idx) =>
|
|
1370
|
+
idx.eq("dataSourceId", dataSourceId).eq("seriesType", seriesType),
|
|
1371
|
+
)
|
|
1372
|
+
.first();
|
|
1373
|
+
if (raw) {
|
|
1374
|
+
return true;
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
const rollup = await db
|
|
1378
|
+
.query("timeSeriesRollups")
|
|
1379
|
+
.withIndex("by_source_type_bucket", (idx) =>
|
|
1380
|
+
idx.eq("dataSourceId", dataSourceId).eq("seriesType", seriesType),
|
|
1381
|
+
)
|
|
1382
|
+
.first();
|
|
1383
|
+
return rollup !== null;
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
function aggregateRawPoints(points: Array<{ recordedAt: number; value: number }>): AggregatedStats {
|
|
1387
|
+
const values = points.map((point) => point.value);
|
|
1388
|
+
const lastPoint = points.reduce((latest, point) =>
|
|
1389
|
+
point.recordedAt >= latest.recordedAt ? point : latest,
|
|
1390
|
+
);
|
|
1391
|
+
|
|
1392
|
+
return {
|
|
1393
|
+
avg: values.reduce((sum, value) => sum + value, 0) / values.length,
|
|
1394
|
+
min: Math.min(...values),
|
|
1395
|
+
max: Math.max(...values),
|
|
1396
|
+
last: lastPoint.value,
|
|
1397
|
+
lastRecordedAt: lastPoint.recordedAt,
|
|
1398
|
+
count: points.length,
|
|
1399
|
+
};
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
function combineAggregatedStats(stats: AggregatedStats[]) {
|
|
1403
|
+
const totalCount = stats.reduce((sum, item) => sum + item.count, 0);
|
|
1404
|
+
const latest = stats.reduce((current, item) =>
|
|
1405
|
+
item.lastRecordedAt >= current.lastRecordedAt ? item : current,
|
|
1406
|
+
);
|
|
1407
|
+
|
|
1408
|
+
return {
|
|
1409
|
+
avg: stats.reduce((sum, item) => sum + item.avg * item.count, 0) / totalCount,
|
|
1410
|
+
min: Math.min(...stats.map((item) => item.min)),
|
|
1411
|
+
max: Math.max(...stats.map((item) => item.max)),
|
|
1412
|
+
last: latest.last,
|
|
1413
|
+
lastRecordedAt: latest.lastRecordedAt,
|
|
1414
|
+
count: totalCount,
|
|
1415
|
+
};
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
function rollupDocToStats(rollup: StoredRollup): AggregatedStats {
|
|
1419
|
+
return {
|
|
1420
|
+
avg: rollup.avg,
|
|
1421
|
+
min: rollup.min,
|
|
1422
|
+
max: rollup.max,
|
|
1423
|
+
last: rollup.last,
|
|
1424
|
+
lastRecordedAt: rollup.lastRecordedAt,
|
|
1425
|
+
count: rollup.count,
|
|
1426
|
+
};
|
|
1427
|
+
}
|
|
1428
|
+
|
|
1429
|
+
function addStatsToRollupGroup(
|
|
1430
|
+
groups: Map<string, { bucketMs: number; bucketStart: number; stats: AggregatedStats }>,
|
|
1431
|
+
bucketMs: number,
|
|
1432
|
+
bucketStart: number,
|
|
1433
|
+
stats: AggregatedStats,
|
|
1434
|
+
) {
|
|
1435
|
+
const key = `${bucketMs}:${bucketStart}`;
|
|
1436
|
+
const existing = groups.get(key);
|
|
1437
|
+
if (existing) {
|
|
1438
|
+
existing.stats = combineAggregatedStats([existing.stats, stats]);
|
|
1439
|
+
return;
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
groups.set(key, {
|
|
1443
|
+
bucketMs,
|
|
1444
|
+
bucketStart,
|
|
1445
|
+
stats,
|
|
1446
|
+
});
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
// ---------------------------------------------------------------------------
|
|
1450
|
+
// Read path
|
|
1451
|
+
// ---------------------------------------------------------------------------
|
|
1452
|
+
|
|
1453
|
+
async function getPolicyAwarePointsForSource(
|
|
1454
|
+
db: TimeSeriesReadDb,
|
|
1455
|
+
args: {
|
|
1456
|
+
dataSourceId: Id<"dataSources">;
|
|
1457
|
+
userId: string;
|
|
1458
|
+
provider: Doc<"dataSources">["provider"];
|
|
1459
|
+
seriesType: string;
|
|
1460
|
+
startDate: number;
|
|
1461
|
+
endDate: number;
|
|
1462
|
+
limit: number;
|
|
1463
|
+
order: "asc" | "desc";
|
|
1464
|
+
},
|
|
1465
|
+
) {
|
|
1466
|
+
const effective = await resolveEffectivePolicy(db, args.userId, args.provider, args.seriesType);
|
|
1467
|
+
const now = Date.now();
|
|
1468
|
+
const points: TimeSeriesPoint[] = [];
|
|
1469
|
+
|
|
1470
|
+
for (const tier of effective.tiers) {
|
|
1471
|
+
const interval = getAbsoluteTimeRangeForTier(tier, now, args.startDate, args.endDate);
|
|
1472
|
+
if (!interval) {
|
|
1473
|
+
continue;
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
const preferred =
|
|
1477
|
+
tier.kind === "raw"
|
|
1478
|
+
? await queryRawPoints(db, {
|
|
1479
|
+
dataSourceId: args.dataSourceId,
|
|
1480
|
+
seriesType: args.seriesType,
|
|
1481
|
+
startDate: interval.start,
|
|
1482
|
+
endDate: interval.end,
|
|
1483
|
+
limit: args.limit + 1,
|
|
1484
|
+
order: args.order,
|
|
1485
|
+
})
|
|
1486
|
+
: await queryRollupPoints(db, {
|
|
1487
|
+
dataSourceId: args.dataSourceId,
|
|
1488
|
+
seriesType: args.seriesType,
|
|
1489
|
+
bucketMs: tier.bucketMs,
|
|
1490
|
+
startDate: interval.start,
|
|
1491
|
+
endDate: interval.end,
|
|
1492
|
+
limit: args.limit + 1,
|
|
1493
|
+
order: args.order,
|
|
1494
|
+
aggregations: tier.aggregations,
|
|
1495
|
+
});
|
|
1496
|
+
|
|
1497
|
+
if (preferred.length > 0) {
|
|
1498
|
+
points.push(...preferred);
|
|
1499
|
+
continue;
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
points.push(
|
|
1503
|
+
...(await queryFallbackPoints(db, {
|
|
1504
|
+
dataSourceId: args.dataSourceId,
|
|
1505
|
+
seriesType: args.seriesType,
|
|
1506
|
+
startDate: interval.start,
|
|
1507
|
+
endDate: interval.end,
|
|
1508
|
+
limit: args.limit + 1,
|
|
1509
|
+
order: args.order,
|
|
1510
|
+
})),
|
|
1511
|
+
);
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
points.sort((a, b) =>
|
|
1515
|
+
args.order === "asc" ? a.timestamp - b.timestamp : b.timestamp - a.timestamp,
|
|
1516
|
+
);
|
|
1517
|
+
return points.slice(0, args.limit);
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
function getAbsoluteTimeRangeForTier(
|
|
1521
|
+
tier: NormalizedTimeSeriesTier,
|
|
1522
|
+
now: number,
|
|
1523
|
+
startDate: number,
|
|
1524
|
+
endDate: number,
|
|
1525
|
+
) {
|
|
1526
|
+
const absoluteStart =
|
|
1527
|
+
tier.toAgeMs === null ? startDate : Math.max(startDate, now - tier.toAgeMs + 1);
|
|
1528
|
+
const absoluteEnd = Math.min(endDate, now - tier.fromAgeMs);
|
|
1529
|
+
|
|
1530
|
+
if (absoluteStart > absoluteEnd) {
|
|
1531
|
+
return null;
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
return {
|
|
1535
|
+
start: absoluteStart,
|
|
1536
|
+
end: absoluteEnd,
|
|
1537
|
+
};
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
async function queryRawPoints(
|
|
1541
|
+
db: Pick<TimeSeriesReadDb, "query">,
|
|
1542
|
+
args: {
|
|
1543
|
+
dataSourceId: Id<"dataSources">;
|
|
1544
|
+
seriesType: string;
|
|
1545
|
+
startDate: number;
|
|
1546
|
+
endDate: number;
|
|
1547
|
+
limit: number;
|
|
1548
|
+
order: "asc" | "desc";
|
|
1549
|
+
},
|
|
1550
|
+
) {
|
|
1551
|
+
const raw = await db
|
|
1552
|
+
.query("dataPoints")
|
|
1553
|
+
.withIndex("by_source_type_time", (idx) =>
|
|
1554
|
+
idx
|
|
1555
|
+
.eq("dataSourceId", args.dataSourceId)
|
|
1556
|
+
.eq("seriesType", args.seriesType)
|
|
1557
|
+
.gte("recordedAt", args.startDate)
|
|
1558
|
+
.lte("recordedAt", args.endDate),
|
|
1559
|
+
)
|
|
1560
|
+
.order(args.order)
|
|
1561
|
+
.take(args.limit);
|
|
1562
|
+
|
|
1563
|
+
return raw.map((point: Doc<"dataPoints">) => ({
|
|
1564
|
+
timestamp: point.recordedAt,
|
|
1565
|
+
value: point.value,
|
|
1566
|
+
resolution: "raw" as const,
|
|
1567
|
+
}));
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
async function queryRollupPoints(
|
|
1571
|
+
db: Pick<TimeSeriesReadDb, "query">,
|
|
1572
|
+
args: {
|
|
1573
|
+
dataSourceId: Id<"dataSources">;
|
|
1574
|
+
seriesType: string;
|
|
1575
|
+
bucketMs: number;
|
|
1576
|
+
startDate: number;
|
|
1577
|
+
endDate: number;
|
|
1578
|
+
limit: number;
|
|
1579
|
+
order: "asc" | "desc";
|
|
1580
|
+
aggregations: readonly TimeSeriesAggregation[];
|
|
1581
|
+
},
|
|
1582
|
+
) {
|
|
1583
|
+
const rows = await db
|
|
1584
|
+
.query("timeSeriesRollups")
|
|
1585
|
+
.withIndex("by_source_type_bucket_size", (idx) =>
|
|
1586
|
+
idx
|
|
1587
|
+
.eq("dataSourceId", args.dataSourceId)
|
|
1588
|
+
.eq("seriesType", args.seriesType)
|
|
1589
|
+
.eq("bucketMs", args.bucketMs)
|
|
1590
|
+
.gte("bucketStart", Math.max(0, args.startDate - args.bucketMs))
|
|
1591
|
+
.lte("bucketStart", args.endDate),
|
|
1592
|
+
)
|
|
1593
|
+
.order(args.order)
|
|
1594
|
+
.take(args.limit);
|
|
1595
|
+
|
|
1596
|
+
return rows
|
|
1597
|
+
.filter(
|
|
1598
|
+
(row: StoredRollup) => row.bucketEnd >= args.startDate && row.bucketStart <= args.endDate,
|
|
1599
|
+
)
|
|
1600
|
+
.map((row: StoredRollup) => toRollupPoint(row, args.aggregations));
|
|
1601
|
+
}
|
|
1602
|
+
|
|
1603
|
+
async function queryFallbackPoints(
|
|
1604
|
+
db: Pick<TimeSeriesReadDb, "query">,
|
|
1605
|
+
args: {
|
|
1606
|
+
dataSourceId: Id<"dataSources">;
|
|
1607
|
+
seriesType: string;
|
|
1608
|
+
startDate: number;
|
|
1609
|
+
endDate: number;
|
|
1610
|
+
limit: number;
|
|
1611
|
+
order: "asc" | "desc";
|
|
1612
|
+
},
|
|
1613
|
+
) {
|
|
1614
|
+
const raw = await queryRawPoints(db, args);
|
|
1615
|
+
if (raw.length > 0) {
|
|
1616
|
+
return raw;
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
const rows = await db
|
|
1620
|
+
.query("timeSeriesRollups")
|
|
1621
|
+
.withIndex("by_source_type_bucket", (idx) =>
|
|
1622
|
+
idx
|
|
1623
|
+
.eq("dataSourceId", args.dataSourceId)
|
|
1624
|
+
.eq("seriesType", args.seriesType)
|
|
1625
|
+
.gte("bucketStart", Math.max(0, args.startDate - DAY_MS))
|
|
1626
|
+
.lte("bucketStart", args.endDate),
|
|
1627
|
+
)
|
|
1628
|
+
.take(args.limit * 4);
|
|
1629
|
+
|
|
1630
|
+
rows.sort((a: StoredRollup, b: StoredRollup) =>
|
|
1631
|
+
a.bucketMs === b.bucketMs ? a.bucketStart - b.bucketStart : a.bucketMs - b.bucketMs,
|
|
1632
|
+
);
|
|
1633
|
+
|
|
1634
|
+
const filtered = rows.filter(
|
|
1635
|
+
(row: StoredRollup) => row.bucketEnd >= args.startDate && row.bucketStart <= args.endDate,
|
|
1636
|
+
);
|
|
1637
|
+
return filtered.map((row: StoredRollup) =>
|
|
1638
|
+
toRollupPoint(row, normalizeAggregations(DEFAULT_TIME_SERIES_AGGREGATIONS)),
|
|
1639
|
+
);
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
function toRollupPoint(
|
|
1643
|
+
rollup: StoredRollup,
|
|
1644
|
+
aggregations: readonly TimeSeriesAggregation[],
|
|
1645
|
+
): TimeSeriesPoint {
|
|
1646
|
+
return {
|
|
1647
|
+
timestamp: rollup.bucketStart,
|
|
1648
|
+
value: selectPrimaryRollupValue(rollup, aggregations),
|
|
1649
|
+
resolution: "rollup",
|
|
1650
|
+
bucketMinutes: Math.floor(rollup.bucketMs / (60 * 1000)),
|
|
1651
|
+
avg: rollup.avg,
|
|
1652
|
+
min: rollup.min,
|
|
1653
|
+
max: rollup.max,
|
|
1654
|
+
last: rollup.last,
|
|
1655
|
+
count: rollup.count,
|
|
1656
|
+
};
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
function selectPrimaryRollupValue(
|
|
1660
|
+
rollup: StoredRollup,
|
|
1661
|
+
aggregations: readonly TimeSeriesAggregation[],
|
|
1662
|
+
) {
|
|
1663
|
+
if (aggregations.includes("avg")) {
|
|
1664
|
+
return rollup.avg;
|
|
1665
|
+
}
|
|
1666
|
+
if (aggregations.includes("last")) {
|
|
1667
|
+
return rollup.last;
|
|
1668
|
+
}
|
|
1669
|
+
if (aggregations.includes("max")) {
|
|
1670
|
+
return rollup.max;
|
|
1671
|
+
}
|
|
1672
|
+
if (aggregations.includes("min")) {
|
|
1673
|
+
return rollup.min;
|
|
1674
|
+
}
|
|
1675
|
+
return rollup.count;
|
|
1676
|
+
}
|