@devwithbobby/loops 0.1.19 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.d.ts +47 -12
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +42 -4
- package/dist/component/_generated/api.d.ts +185 -1
- package/dist/component/_generated/api.d.ts.map +1 -1
- package/dist/component/_generated/component.d.ts +12 -5
- package/dist/component/_generated/component.d.ts.map +1 -1
- package/dist/component/_generated/dataModel.d.ts +1 -1
- package/dist/component/aggregates.d.ts +42 -0
- package/dist/component/aggregates.d.ts.map +1 -0
- package/dist/component/aggregates.js +54 -0
- package/dist/component/convex.config.d.ts.map +1 -1
- package/dist/component/convex.config.js +2 -0
- package/dist/component/helpers.d.ts.map +1 -1
- package/dist/component/http.js +1 -1
- package/dist/component/lib.d.ts +66 -17
- package/dist/component/lib.d.ts.map +1 -1
- package/dist/component/lib.js +194 -73
- package/dist/component/schema.d.ts +2 -2
- package/dist/component/tables/contacts.d.ts.map +1 -1
- package/dist/component/tables/emailOperations.d.ts +4 -4
- package/dist/component/tables/emailOperations.d.ts.map +1 -1
- package/dist/test.d.ts +2 -2
- package/dist/types.d.ts +249 -62
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +4 -2
- package/dist/utils.d.ts +6 -6
- package/package.json +15 -9
- package/src/client/index.ts +52 -6
- package/src/component/_generated/api.ts +190 -1
- package/src/component/_generated/component.ts +10 -5
- package/src/component/_generated/dataModel.ts +1 -1
- package/src/component/aggregates.ts +89 -0
- package/src/component/convex.config.ts +3 -0
- package/src/component/http.ts +1 -1
- package/src/component/lib.ts +226 -89
- package/src/types.ts +20 -122
package/dist/component/lib.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import { paginator } from "convex-helpers/server/pagination";
|
|
1
2
|
import { z } from "zod";
|
|
2
3
|
import { internalLib } from "../types";
|
|
3
4
|
import { za, zm, zq } from "../utils.js";
|
|
5
|
+
import { aggregateClear, aggregateCountByUserGroup, aggregateCountTotal, aggregateDelete, aggregateInsert, aggregateReplace, } from "./aggregates";
|
|
4
6
|
import { loopsFetch, sanitizeLoopsError } from "./helpers";
|
|
7
|
+
import schema from "./schema";
|
|
5
8
|
import { contactValidator } from "./validators.js";
|
|
6
9
|
/**
|
|
7
10
|
* Internal mutation to store/update a contact in the database
|
|
@@ -25,6 +28,7 @@ export const storeContact = zm({
|
|
|
25
28
|
.withIndex("email", (q) => q.eq("email", args.email))
|
|
26
29
|
.unique();
|
|
27
30
|
if (existing) {
|
|
31
|
+
// Update the contact
|
|
28
32
|
await ctx.db.patch(existing._id, {
|
|
29
33
|
firstName: args.firstName,
|
|
30
34
|
lastName: args.lastName,
|
|
@@ -35,9 +39,15 @@ export const storeContact = zm({
|
|
|
35
39
|
loopsContactId: args.loopsContactId,
|
|
36
40
|
updatedAt: now,
|
|
37
41
|
});
|
|
42
|
+
// Get the updated document and update aggregate if userGroup changed
|
|
43
|
+
const updated = await ctx.db.get(existing._id);
|
|
44
|
+
if (updated && existing.userGroup !== updated.userGroup) {
|
|
45
|
+
await aggregateReplace(ctx, existing, updated);
|
|
46
|
+
}
|
|
38
47
|
}
|
|
39
48
|
else {
|
|
40
|
-
|
|
49
|
+
// Insert new contact
|
|
50
|
+
const id = await ctx.db.insert("contacts", {
|
|
41
51
|
email: args.email,
|
|
42
52
|
firstName: args.firstName,
|
|
43
53
|
lastName: args.lastName,
|
|
@@ -49,6 +59,11 @@ export const storeContact = zm({
|
|
|
49
59
|
createdAt: now,
|
|
50
60
|
updatedAt: now,
|
|
51
61
|
});
|
|
62
|
+
// Add to aggregate for counting
|
|
63
|
+
const newDoc = await ctx.db.get(id);
|
|
64
|
+
if (newDoc) {
|
|
65
|
+
await aggregateInsert(ctx, newDoc);
|
|
66
|
+
}
|
|
52
67
|
}
|
|
53
68
|
},
|
|
54
69
|
});
|
|
@@ -66,6 +81,8 @@ export const removeContact = zm({
|
|
|
66
81
|
.withIndex("email", (q) => q.eq("email", args.email))
|
|
67
82
|
.unique();
|
|
68
83
|
if (existing) {
|
|
84
|
+
// Remove from aggregate first (before deleting the document)
|
|
85
|
+
await aggregateDelete(ctx, existing);
|
|
69
86
|
await ctx.db.delete(existing._id);
|
|
70
87
|
}
|
|
71
88
|
},
|
|
@@ -104,13 +121,26 @@ export const logEmailOperation = zm({
|
|
|
104
121
|
await ctx.db.insert("emailOperations", operationData);
|
|
105
122
|
},
|
|
106
123
|
});
|
|
124
|
+
/**
|
|
125
|
+
* Maximum number of documents to read when counting with filters.
|
|
126
|
+
* This limit prevents query read limit errors while still providing accurate
|
|
127
|
+
* counts for most use cases. If you have more contacts than this, consider
|
|
128
|
+
* using the aggregate-based counting with userGroup only.
|
|
129
|
+
*/
|
|
130
|
+
const MAX_COUNT_LIMIT = 8000;
|
|
107
131
|
/**
|
|
108
132
|
* Count contacts in the database
|
|
109
133
|
* Can filter by audience criteria (userGroup, source, subscribed status)
|
|
110
134
|
*
|
|
111
|
-
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
135
|
+
* For userGroup-only filtering, uses efficient O(log n) aggregate counting.
|
|
136
|
+
* For other filters (source, subscribed), uses indexed queries with a read limit.
|
|
137
|
+
*
|
|
138
|
+
* IMPORTANT: Before using this with existing data, run the backfillContactAggregate
|
|
139
|
+
* mutation to populate the aggregate with existing contacts.
|
|
140
|
+
*
|
|
141
|
+
* NOTE: When filtering by source or subscribed, counts are capped at MAX_COUNT_LIMIT
|
|
142
|
+
* to avoid query read limit errors. For exact counts with large datasets, use
|
|
143
|
+
* userGroup-only filtering which uses efficient aggregate counting.
|
|
114
144
|
*/
|
|
115
145
|
export const countContacts = zq({
|
|
116
146
|
args: z.object({
|
|
@@ -120,39 +150,43 @@ export const countContacts = zq({
|
|
|
120
150
|
}),
|
|
121
151
|
returns: z.number(),
|
|
122
152
|
handler: async (ctx, args) => {
|
|
123
|
-
//
|
|
124
|
-
|
|
153
|
+
// If only userGroup is specified (or no filters), use efficient aggregate counting
|
|
154
|
+
const onlyUserGroupFilter = args.source === undefined && args.subscribed === undefined;
|
|
155
|
+
if (onlyUserGroupFilter) {
|
|
156
|
+
// Use O(log n) aggregate counting - much more efficient than .collect()
|
|
157
|
+
if (args.userGroup === undefined) {
|
|
158
|
+
// Count ALL contacts across all namespaces
|
|
159
|
+
return await aggregateCountTotal(ctx);
|
|
160
|
+
}
|
|
161
|
+
// Count contacts in specific userGroup namespace
|
|
162
|
+
return await aggregateCountByUserGroup(ctx, args.userGroup);
|
|
163
|
+
}
|
|
164
|
+
// For other filters, we need to use indexed queries with in-memory filtering
|
|
165
|
+
// We use .take() with a reasonable limit to avoid query read limit errors
|
|
125
166
|
let contacts;
|
|
126
167
|
if (args.userGroup !== undefined) {
|
|
127
168
|
contacts = await ctx.db
|
|
128
169
|
.query("contacts")
|
|
129
170
|
.withIndex("userGroup", (q) => q.eq("userGroup", args.userGroup))
|
|
130
|
-
.
|
|
171
|
+
.take(MAX_COUNT_LIMIT);
|
|
131
172
|
}
|
|
132
173
|
else if (args.source !== undefined) {
|
|
133
174
|
contacts = await ctx.db
|
|
134
175
|
.query("contacts")
|
|
135
176
|
.withIndex("source", (q) => q.eq("source", args.source))
|
|
136
|
-
.
|
|
177
|
+
.take(MAX_COUNT_LIMIT);
|
|
137
178
|
}
|
|
138
179
|
else if (args.subscribed !== undefined) {
|
|
139
180
|
contacts = await ctx.db
|
|
140
181
|
.query("contacts")
|
|
141
182
|
.withIndex("subscribed", (q) => q.eq("subscribed", args.subscribed))
|
|
142
|
-
.
|
|
183
|
+
.take(MAX_COUNT_LIMIT);
|
|
143
184
|
}
|
|
144
185
|
else {
|
|
145
|
-
|
|
186
|
+
// This branch shouldn't be reached due to onlyUserGroupFilter check above
|
|
187
|
+
contacts = await ctx.db.query("contacts").take(MAX_COUNT_LIMIT);
|
|
146
188
|
}
|
|
147
189
|
// Apply additional filters if multiple criteria were provided
|
|
148
|
-
// This avoids redundant filtering when only one filter was used
|
|
149
|
-
const needsFiltering = (args.userGroup !== undefined ? 1 : 0) +
|
|
150
|
-
(args.source !== undefined ? 1 : 0) +
|
|
151
|
-
(args.subscribed !== undefined ? 1 : 0) >
|
|
152
|
-
1;
|
|
153
|
-
if (!needsFiltering) {
|
|
154
|
-
return contacts.length;
|
|
155
|
-
}
|
|
156
190
|
const filtered = contacts.filter((c) => {
|
|
157
191
|
if (args.userGroup !== undefined && c.userGroup !== args.userGroup) {
|
|
158
192
|
return false;
|
|
@@ -169,12 +203,15 @@ export const countContacts = zq({
|
|
|
169
203
|
},
|
|
170
204
|
});
|
|
171
205
|
/**
|
|
172
|
-
* List contacts from the database with pagination
|
|
206
|
+
* List contacts from the database with cursor-based pagination
|
|
173
207
|
* Can filter by audience criteria (userGroup, source, subscribed status)
|
|
174
208
|
* Returns actual contact data, not just a count
|
|
175
209
|
*
|
|
210
|
+
* Uses cursor-based pagination for efficient querying - only reads documents
|
|
211
|
+
* from the cursor position forward, not all preceding documents.
|
|
212
|
+
*
|
|
176
213
|
* Note: When multiple filters are provided, only one index can be used.
|
|
177
|
-
* Additional filters are applied in-memory
|
|
214
|
+
* Additional filters are applied in-memory after fetching.
|
|
178
215
|
*/
|
|
179
216
|
export const listContacts = zq({
|
|
180
217
|
args: z.object({
|
|
@@ -182,7 +219,7 @@ export const listContacts = zq({
|
|
|
182
219
|
source: z.string().optional(),
|
|
183
220
|
subscribed: z.boolean().optional(),
|
|
184
221
|
limit: z.number().min(1).max(1000).default(100),
|
|
185
|
-
|
|
222
|
+
cursor: z.string().nullable().optional(),
|
|
186
223
|
}),
|
|
187
224
|
returns: z.object({
|
|
188
225
|
contacts: z.array(z.object({
|
|
@@ -198,42 +235,51 @@ export const listContacts = zq({
|
|
|
198
235
|
createdAt: z.number(),
|
|
199
236
|
updatedAt: z.number(),
|
|
200
237
|
})),
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
offset: z.number(),
|
|
204
|
-
hasMore: z.boolean(),
|
|
238
|
+
continueCursor: z.string().nullable(),
|
|
239
|
+
isDone: z.boolean(),
|
|
205
240
|
}),
|
|
206
241
|
handler: async (ctx, args) => {
|
|
207
|
-
|
|
208
|
-
|
|
242
|
+
const paginationOpts = {
|
|
243
|
+
cursor: args.cursor ?? null,
|
|
244
|
+
numItems: args.limit,
|
|
245
|
+
};
|
|
246
|
+
// Determine which index to use based on filters
|
|
247
|
+
const needsFiltering = (args.userGroup !== undefined ? 1 : 0) +
|
|
248
|
+
(args.source !== undefined ? 1 : 0) +
|
|
249
|
+
(args.subscribed !== undefined ? 1 : 0) >
|
|
250
|
+
1;
|
|
251
|
+
let result;
|
|
209
252
|
if (args.userGroup !== undefined) {
|
|
210
|
-
|
|
253
|
+
result = await paginator(ctx.db, schema)
|
|
211
254
|
.query("contacts")
|
|
212
255
|
.withIndex("userGroup", (q) => q.eq("userGroup", args.userGroup))
|
|
213
|
-
.
|
|
256
|
+
.order("desc")
|
|
257
|
+
.paginate(paginationOpts);
|
|
214
258
|
}
|
|
215
259
|
else if (args.source !== undefined) {
|
|
216
|
-
|
|
260
|
+
result = await paginator(ctx.db, schema)
|
|
217
261
|
.query("contacts")
|
|
218
262
|
.withIndex("source", (q) => q.eq("source", args.source))
|
|
219
|
-
.
|
|
263
|
+
.order("desc")
|
|
264
|
+
.paginate(paginationOpts);
|
|
220
265
|
}
|
|
221
266
|
else if (args.subscribed !== undefined) {
|
|
222
|
-
|
|
267
|
+
result = await paginator(ctx.db, schema)
|
|
223
268
|
.query("contacts")
|
|
224
269
|
.withIndex("subscribed", (q) => q.eq("subscribed", args.subscribed))
|
|
225
|
-
.
|
|
270
|
+
.order("desc")
|
|
271
|
+
.paginate(paginationOpts);
|
|
226
272
|
}
|
|
227
273
|
else {
|
|
228
|
-
|
|
274
|
+
result = await paginator(ctx.db, schema)
|
|
275
|
+
.query("contacts")
|
|
276
|
+
.order("desc")
|
|
277
|
+
.paginate(paginationOpts);
|
|
229
278
|
}
|
|
279
|
+
let contacts = result.page;
|
|
230
280
|
// Apply additional filters if multiple criteria were provided
|
|
231
|
-
const needsFiltering = (args.userGroup !== undefined ? 1 : 0) +
|
|
232
|
-
(args.source !== undefined ? 1 : 0) +
|
|
233
|
-
(args.subscribed !== undefined ? 1 : 0) >
|
|
234
|
-
1;
|
|
235
281
|
if (needsFiltering) {
|
|
236
|
-
|
|
282
|
+
contacts = contacts.filter((c) => {
|
|
237
283
|
if (args.userGroup !== undefined && c.userGroup !== args.userGroup) {
|
|
238
284
|
return false;
|
|
239
285
|
}
|
|
@@ -246,22 +292,14 @@ export const listContacts = zq({
|
|
|
246
292
|
return true;
|
|
247
293
|
});
|
|
248
294
|
}
|
|
249
|
-
|
|
250
|
-
allContacts.sort((a, b) => b.createdAt - a.createdAt);
|
|
251
|
-
const total = allContacts.length;
|
|
252
|
-
const paginatedContacts = allContacts
|
|
253
|
-
.slice(args.offset, args.offset + args.limit)
|
|
254
|
-
.map((contact) => ({
|
|
295
|
+
const mappedContacts = contacts.map((contact) => ({
|
|
255
296
|
...contact,
|
|
256
|
-
subscribed: contact.subscribed ?? true,
|
|
297
|
+
subscribed: contact.subscribed ?? true,
|
|
257
298
|
}));
|
|
258
|
-
const hasMore = args.offset + args.limit < total;
|
|
259
299
|
return {
|
|
260
|
-
contacts:
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
offset: args.offset,
|
|
264
|
-
hasMore,
|
|
300
|
+
contacts: mappedContacts,
|
|
301
|
+
continueCursor: result.continueCursor,
|
|
302
|
+
isDone: result.isDone,
|
|
265
303
|
};
|
|
266
304
|
},
|
|
267
305
|
});
|
|
@@ -766,9 +804,18 @@ export const resubscribeContact = za({
|
|
|
766
804
|
return { success: true };
|
|
767
805
|
},
|
|
768
806
|
});
|
|
807
|
+
/**
|
|
808
|
+
* Maximum number of email operations to read for spam detection.
|
|
809
|
+
* This limit prevents query read limit errors while covering most spam scenarios.
|
|
810
|
+
* If you need to analyze more operations, consider using scheduled jobs with pagination.
|
|
811
|
+
*/
|
|
812
|
+
const MAX_SPAM_DETECTION_LIMIT = 8000;
|
|
769
813
|
/**
|
|
770
814
|
* Check for spam patterns: too many emails to the same recipient in a time window
|
|
771
|
-
* Returns email addresses that received too many emails
|
|
815
|
+
* Returns email addresses that received too many emails.
|
|
816
|
+
*
|
|
817
|
+
* NOTE: Analysis is limited to the most recent MAX_SPAM_DETECTION_LIMIT operations
|
|
818
|
+
* in the time window to avoid query read limit errors.
|
|
772
819
|
*/
|
|
773
820
|
export const detectRecipientSpam = zq({
|
|
774
821
|
args: z.object({
|
|
@@ -787,7 +834,7 @@ export const detectRecipientSpam = zq({
|
|
|
787
834
|
const operations = await ctx.db
|
|
788
835
|
.query("emailOperations")
|
|
789
836
|
.withIndex("timestamp", (q) => q.gte("timestamp", cutoffTime))
|
|
790
|
-
.
|
|
837
|
+
.take(MAX_SPAM_DETECTION_LIMIT);
|
|
791
838
|
const emailCounts = new Map();
|
|
792
839
|
for (const op of operations) {
|
|
793
840
|
if (op.email && op.email !== "audience") {
|
|
@@ -809,7 +856,10 @@ export const detectRecipientSpam = zq({
|
|
|
809
856
|
});
|
|
810
857
|
/**
|
|
811
858
|
* Check for spam patterns: too many emails from the same actor/user
|
|
812
|
-
* Returns actor IDs that sent too many emails
|
|
859
|
+
* Returns actor IDs that sent too many emails.
|
|
860
|
+
*
|
|
861
|
+
* NOTE: Analysis is limited to the most recent MAX_SPAM_DETECTION_LIMIT operations
|
|
862
|
+
* in the time window to avoid query read limit errors.
|
|
813
863
|
*/
|
|
814
864
|
export const detectActorSpam = zq({
|
|
815
865
|
args: z.object({
|
|
@@ -826,7 +876,7 @@ export const detectActorSpam = zq({
|
|
|
826
876
|
const operations = await ctx.db
|
|
827
877
|
.query("emailOperations")
|
|
828
878
|
.withIndex("timestamp", (q) => q.gte("timestamp", cutoffTime))
|
|
829
|
-
.
|
|
879
|
+
.take(MAX_SPAM_DETECTION_LIMIT);
|
|
830
880
|
const actorCounts = new Map();
|
|
831
881
|
for (const op of operations) {
|
|
832
882
|
if (op.actorId) {
|
|
@@ -847,7 +897,11 @@ export const detectActorSpam = zq({
|
|
|
847
897
|
},
|
|
848
898
|
});
|
|
849
899
|
/**
|
|
850
|
-
* Get recent email operation statistics for monitoring
|
|
900
|
+
* Get recent email operation statistics for monitoring.
|
|
901
|
+
*
|
|
902
|
+
* NOTE: Statistics are calculated from the most recent MAX_SPAM_DETECTION_LIMIT
|
|
903
|
+
* operations in the time window to avoid query read limit errors. For high-volume
|
|
904
|
+
* applications, consider using scheduled jobs with pagination for exact statistics.
|
|
851
905
|
*/
|
|
852
906
|
export const getEmailStats = zq({
|
|
853
907
|
args: z.object({
|
|
@@ -868,7 +922,7 @@ export const getEmailStats = zq({
|
|
|
868
922
|
const operations = await ctx.db
|
|
869
923
|
.query("emailOperations")
|
|
870
924
|
.withIndex("timestamp", (q) => q.gte("timestamp", cutoffTime))
|
|
871
|
-
.
|
|
925
|
+
.take(MAX_SPAM_DETECTION_LIMIT);
|
|
872
926
|
const stats = {
|
|
873
927
|
totalOperations: operations.length,
|
|
874
928
|
successfulOperations: operations.filter((op) => op.success).length,
|
|
@@ -896,7 +950,10 @@ export const getEmailStats = zq({
|
|
|
896
950
|
});
|
|
897
951
|
/**
|
|
898
952
|
* Detect rapid-fire email sending patterns (multiple emails sent in quick succession)
|
|
899
|
-
* Returns suspicious patterns indicating potential spam
|
|
953
|
+
* Returns suspicious patterns indicating potential spam.
|
|
954
|
+
*
|
|
955
|
+
* NOTE: Analysis is limited to the most recent MAX_SPAM_DETECTION_LIMIT operations
|
|
956
|
+
* in the time window to avoid query read limit errors.
|
|
900
957
|
*/
|
|
901
958
|
export const detectRapidFirePatterns = zq({
|
|
902
959
|
args: z.object({
|
|
@@ -916,11 +973,11 @@ export const detectRapidFirePatterns = zq({
|
|
|
916
973
|
const operations = await ctx.db
|
|
917
974
|
.query("emailOperations")
|
|
918
975
|
.withIndex("timestamp", (q) => q.gte("timestamp", cutoffTime))
|
|
919
|
-
.
|
|
920
|
-
operations.sort((a, b) => a.timestamp - b.timestamp);
|
|
976
|
+
.take(MAX_SPAM_DETECTION_LIMIT);
|
|
977
|
+
const sortedOps = [...operations].sort((a, b) => a.timestamp - b.timestamp);
|
|
921
978
|
const patterns = [];
|
|
922
979
|
const emailGroups = new Map();
|
|
923
|
-
for (const op of
|
|
980
|
+
for (const op of sortedOps) {
|
|
924
981
|
if (op.email && op.email !== "audience") {
|
|
925
982
|
if (!emailGroups.has(op.email)) {
|
|
926
983
|
emailGroups.set(op.email, []);
|
|
@@ -948,7 +1005,7 @@ export const detectRapidFirePatterns = zq({
|
|
|
948
1005
|
}
|
|
949
1006
|
}
|
|
950
1007
|
const actorGroups = new Map();
|
|
951
|
-
for (const op of
|
|
1008
|
+
for (const op of sortedOps) {
|
|
952
1009
|
if (op.actorId) {
|
|
953
1010
|
if (!actorGroups.has(op.actorId)) {
|
|
954
1011
|
actorGroups.set(op.actorId, []);
|
|
@@ -980,7 +1037,10 @@ export const detectRapidFirePatterns = zq({
|
|
|
980
1037
|
});
|
|
981
1038
|
/**
|
|
982
1039
|
* Rate limiting: Check if an email can be sent to a recipient
|
|
983
|
-
* Based on recent email operations in the database
|
|
1040
|
+
* Based on recent email operations in the database.
|
|
1041
|
+
*
|
|
1042
|
+
* Uses efficient .take() query - only reads the minimum number of documents
|
|
1043
|
+
* needed to determine if the rate limit is exceeded.
|
|
984
1044
|
*/
|
|
985
1045
|
export const checkRecipientRateLimit = zq({
|
|
986
1046
|
args: z.object({
|
|
@@ -999,11 +1059,14 @@ export const checkRecipientRateLimit = zq({
|
|
|
999
1059
|
.catchall(z.any()),
|
|
1000
1060
|
handler: async (ctx, args) => {
|
|
1001
1061
|
const cutoffTime = Date.now() - args.timeWindowMs;
|
|
1062
|
+
// Use the compound index (email, timestamp) to efficiently query
|
|
1063
|
+
// Only fetch up to maxEmails + 1 to check if limit exceeded
|
|
1002
1064
|
const operations = await ctx.db
|
|
1003
1065
|
.query("emailOperations")
|
|
1004
|
-
.withIndex("email", (q) => q.eq("email", args.email))
|
|
1005
|
-
.
|
|
1006
|
-
|
|
1066
|
+
.withIndex("email", (q) => q.eq("email", args.email).gte("timestamp", cutoffTime))
|
|
1067
|
+
.take(args.maxEmails + 1);
|
|
1068
|
+
// Filter for successful operations only
|
|
1069
|
+
const recentOps = operations.filter((op) => op.success);
|
|
1007
1070
|
const count = recentOps.length;
|
|
1008
1071
|
const allowed = count < args.maxEmails;
|
|
1009
1072
|
let retryAfter;
|
|
@@ -1024,7 +1087,10 @@ export const checkRecipientRateLimit = zq({
|
|
|
1024
1087
|
});
|
|
1025
1088
|
/**
|
|
1026
1089
|
* Rate limiting: Check if an actor/user can send more emails
|
|
1027
|
-
* Based on recent email operations in the database
|
|
1090
|
+
* Based on recent email operations in the database.
|
|
1091
|
+
*
|
|
1092
|
+
* Uses efficient .take() query - only reads the minimum number of documents
|
|
1093
|
+
* needed to determine if the rate limit is exceeded.
|
|
1028
1094
|
*/
|
|
1029
1095
|
export const checkActorRateLimit = zq({
|
|
1030
1096
|
args: z.object({
|
|
@@ -1041,11 +1107,14 @@ export const checkActorRateLimit = zq({
|
|
|
1041
1107
|
}),
|
|
1042
1108
|
handler: async (ctx, args) => {
|
|
1043
1109
|
const cutoffTime = Date.now() - args.timeWindowMs;
|
|
1110
|
+
// Use the compound index (actorId, timestamp) to efficiently query
|
|
1111
|
+
// Only fetch up to maxEmails + 1 to check if limit exceeded
|
|
1044
1112
|
const operations = await ctx.db
|
|
1045
1113
|
.query("emailOperations")
|
|
1046
|
-
.withIndex("actorId", (q) => q.eq("actorId", args.actorId))
|
|
1047
|
-
.
|
|
1048
|
-
|
|
1114
|
+
.withIndex("actorId", (q) => q.eq("actorId", args.actorId).gte("timestamp", cutoffTime))
|
|
1115
|
+
.take(args.maxEmails + 1);
|
|
1116
|
+
// Filter for successful operations only
|
|
1117
|
+
const recentOps = operations.filter((op) => op.success);
|
|
1049
1118
|
const count = recentOps.length;
|
|
1050
1119
|
const allowed = count < args.maxEmails;
|
|
1051
1120
|
let retryAfter;
|
|
@@ -1066,7 +1135,10 @@ export const checkActorRateLimit = zq({
|
|
|
1066
1135
|
});
|
|
1067
1136
|
/**
|
|
1068
1137
|
* Rate limiting: Check global email sending rate
|
|
1069
|
-
* Checks total email operations across all senders
|
|
1138
|
+
* Checks total email operations across all senders.
|
|
1139
|
+
*
|
|
1140
|
+
* Uses efficient .take() query - only reads the minimum number of documents
|
|
1141
|
+
* needed to determine if the rate limit is exceeded.
|
|
1070
1142
|
*/
|
|
1071
1143
|
export const checkGlobalRateLimit = zq({
|
|
1072
1144
|
args: z.object({
|
|
@@ -1081,10 +1153,13 @@ export const checkGlobalRateLimit = zq({
|
|
|
1081
1153
|
}),
|
|
1082
1154
|
handler: async (ctx, args) => {
|
|
1083
1155
|
const cutoffTime = Date.now() - args.timeWindowMs;
|
|
1156
|
+
// Use the timestamp index to efficiently query recent operations
|
|
1157
|
+
// Only fetch up to maxEmails + 1 to check if limit exceeded
|
|
1084
1158
|
const operations = await ctx.db
|
|
1085
1159
|
.query("emailOperations")
|
|
1086
1160
|
.withIndex("timestamp", (q) => q.gte("timestamp", cutoffTime))
|
|
1087
|
-
.
|
|
1161
|
+
.take(args.maxEmails + 1);
|
|
1162
|
+
// Filter for successful operations only
|
|
1088
1163
|
const recentOps = operations.filter((op) => op.success);
|
|
1089
1164
|
const count = recentOps.length;
|
|
1090
1165
|
const allowed = count < args.maxEmails;
|
|
@@ -1096,3 +1171,49 @@ export const checkGlobalRateLimit = zq({
|
|
|
1096
1171
|
};
|
|
1097
1172
|
},
|
|
1098
1173
|
});
|
|
1174
|
+
/**
|
|
1175
|
+
* Backfill the contact aggregate with existing contacts.
|
|
1176
|
+
* Run this mutation after upgrading to a version with aggregate support.
|
|
1177
|
+
*
|
|
1178
|
+
* This processes contacts in batches to avoid timeout issues with large datasets.
|
|
1179
|
+
* Call repeatedly with the returned cursor until isDone is true.
|
|
1180
|
+
*
|
|
1181
|
+
* Usage:
|
|
1182
|
+
* 1. First call with clear: true to reset the aggregate
|
|
1183
|
+
* 2. Subsequent calls with the returned cursor until isDone is true
|
|
1184
|
+
*/
|
|
1185
|
+
export const backfillContactAggregate = zm({
|
|
1186
|
+
args: z.object({
|
|
1187
|
+
cursor: z.string().nullable().optional(),
|
|
1188
|
+
batchSize: z.number().min(1).max(500).default(100),
|
|
1189
|
+
clear: z.boolean().optional(), // Set to true on first call to clear existing aggregate
|
|
1190
|
+
}),
|
|
1191
|
+
returns: z.object({
|
|
1192
|
+
processed: z.number(),
|
|
1193
|
+
cursor: z.string().nullable(),
|
|
1194
|
+
isDone: z.boolean(),
|
|
1195
|
+
}),
|
|
1196
|
+
handler: async (ctx, args) => {
|
|
1197
|
+
// Clear aggregate on first call if requested
|
|
1198
|
+
if (args.clear && !args.cursor) {
|
|
1199
|
+
await aggregateClear(ctx);
|
|
1200
|
+
}
|
|
1201
|
+
const paginationOpts = {
|
|
1202
|
+
cursor: args.cursor ?? null,
|
|
1203
|
+
numItems: args.batchSize,
|
|
1204
|
+
};
|
|
1205
|
+
const result = await paginator(ctx.db, schema)
|
|
1206
|
+
.query("contacts")
|
|
1207
|
+
.order("asc")
|
|
1208
|
+
.paginate(paginationOpts);
|
|
1209
|
+
// Insert each contact into the aggregate
|
|
1210
|
+
for (const contact of result.page) {
|
|
1211
|
+
await aggregateInsert(ctx, contact);
|
|
1212
|
+
}
|
|
1213
|
+
return {
|
|
1214
|
+
processed: result.page.length,
|
|
1215
|
+
cursor: result.continueCursor,
|
|
1216
|
+
isDone: result.isDone,
|
|
1217
|
+
};
|
|
1218
|
+
},
|
|
1219
|
+
});
|
|
@@ -29,13 +29,13 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
29
29
|
subscribed: ["subscribed", "_creationTime"];
|
|
30
30
|
}, {}, {}>;
|
|
31
31
|
emailOperations: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
32
|
-
metadata?: Record<string, any> | undefined;
|
|
33
32
|
actorId?: string | undefined;
|
|
34
33
|
transactionalId?: string | undefined;
|
|
35
34
|
campaignId?: string | undefined;
|
|
36
35
|
loopId?: string | undefined;
|
|
37
36
|
eventName?: string | undefined;
|
|
38
37
|
messageId?: string | undefined;
|
|
38
|
+
metadata?: Record<string, any> | undefined;
|
|
39
39
|
email: string;
|
|
40
40
|
success: boolean;
|
|
41
41
|
operationType: "transactional" | "event" | "campaign" | "loop";
|
|
@@ -57,7 +57,7 @@ declare const _default: import("convex/server").SchemaDefinition<{
|
|
|
57
57
|
success: import("zod").ZodBoolean;
|
|
58
58
|
messageId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
59
59
|
metadata: import("zod").ZodOptional<import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodAny>>;
|
|
60
|
-
}>, "required", "email" | "success" | "
|
|
60
|
+
}>, "required", "email" | "success" | "operationType" | "actorId" | "transactionalId" | "campaignId" | "loopId" | "eventName" | "timestamp" | "messageId" | "metadata" | `metadata.${string}`>, {
|
|
61
61
|
email: ["email", "timestamp", "_creationTime"];
|
|
62
62
|
actorId: ["actorId", "timestamp", "_creationTime"];
|
|
63
63
|
operationType: ["operationType", "timestamp", "_creationTime"];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contacts.d.ts","sourceRoot":"","sources":["../../../src/component/tables/contacts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,eAAO,MAAM,QAAQ
|
|
1
|
+
{"version":3,"file":"contacts.d.ts","sourceRoot":"","sources":["../../../src/component/tables/contacts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uBAYu99B,EAAG,SAAS;;;;;;;;uBAAgW,EAAG,SAAS;;CADl2+B,CAAC"}
|
|
@@ -2,13 +2,13 @@ import { z } from "zod";
|
|
|
2
2
|
export declare const EmailOperations: {
|
|
3
3
|
name: "emailOperations";
|
|
4
4
|
table: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
5
|
-
metadata?: Record<string, any> | undefined;
|
|
6
5
|
actorId?: string | undefined;
|
|
7
6
|
transactionalId?: string | undefined;
|
|
8
7
|
campaignId?: string | undefined;
|
|
9
8
|
loopId?: string | undefined;
|
|
10
9
|
eventName?: string | undefined;
|
|
11
10
|
messageId?: string | undefined;
|
|
11
|
+
metadata?: Record<string, any> | undefined;
|
|
12
12
|
email: string;
|
|
13
13
|
success: boolean;
|
|
14
14
|
operationType: "transactional" | "event" | "campaign" | "loop";
|
|
@@ -30,15 +30,15 @@ export declare const EmailOperations: {
|
|
|
30
30
|
success: z.ZodBoolean;
|
|
31
31
|
messageId: z.ZodOptional<z.ZodString>;
|
|
32
32
|
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
|
|
33
|
-
}>, "required", "email" | "success" | "
|
|
33
|
+
}>, "required", "email" | "success" | "operationType" | "actorId" | "transactionalId" | "campaignId" | "loopId" | "eventName" | "timestamp" | "messageId" | "metadata" | `metadata.${string}`>, {}, {}, {}>;
|
|
34
34
|
doc: import("convex/values").VObject<{
|
|
35
|
-
metadata?: Record<string, any> | undefined;
|
|
36
35
|
actorId?: string | undefined;
|
|
37
36
|
transactionalId?: string | undefined;
|
|
38
37
|
campaignId?: string | undefined;
|
|
39
38
|
loopId?: string | undefined;
|
|
40
39
|
eventName?: string | undefined;
|
|
41
40
|
messageId?: string | undefined;
|
|
41
|
+
metadata?: Record<string, any> | undefined;
|
|
42
42
|
email: string;
|
|
43
43
|
success: boolean;
|
|
44
44
|
operationType: "transactional" | "event" | "campaign" | "loop";
|
|
@@ -59,7 +59,7 @@ export declare const EmailOperations: {
|
|
|
59
59
|
metadata: import("convex/values").VRecord<Record<string, any> | undefined, import("convex/values").VString<string, "required">, import("convex/values").VAny<"required", "required", string>, "optional", string>;
|
|
60
60
|
_id: import("convex/values").VId<import("convex/values").GenericId<"emailOperations">, "required">;
|
|
61
61
|
_creationTime: import("convex/values").VFloat64<number, "required">;
|
|
62
|
-
}, "required", "email" | "success" | "
|
|
62
|
+
}, "required", "email" | "success" | "operationType" | "actorId" | "transactionalId" | "campaignId" | "loopId" | "eventName" | "timestamp" | "messageId" | "metadata" | "_creationTime" | `metadata.${string}` | "_id">;
|
|
63
63
|
withoutSystemFields: import("zodvex").ConvexValidatorFromZodFieldsAuto<{
|
|
64
64
|
operationType: z.ZodEnum<{
|
|
65
65
|
transactional: "transactional";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"emailOperations.d.ts","sourceRoot":"","sources":["../../../src/component/tables/emailOperations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,eAAO,MAAM,eAAe
|
|
1
|
+
{"version":3,"file":"emailOperations.d.ts","sourceRoot":"","sources":["../../../src/component/tables/emailOperations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;uBAay29B,EAAG,SAAS;;;;;;;;uBAAgW,EAAG,SAAS;;CAD3v+B,CAAC"}
|
package/dist/test.d.ts
CHANGED
|
@@ -43,13 +43,13 @@ declare const _default: {
|
|
|
43
43
|
subscribed: ["subscribed", "_creationTime"];
|
|
44
44
|
}, {}, {}>;
|
|
45
45
|
emailOperations: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
46
|
-
metadata?: Record<string, any> | undefined;
|
|
47
46
|
actorId?: string | undefined;
|
|
48
47
|
transactionalId?: string | undefined;
|
|
49
48
|
campaignId?: string | undefined;
|
|
50
49
|
loopId?: string | undefined;
|
|
51
50
|
eventName?: string | undefined;
|
|
52
51
|
messageId?: string | undefined;
|
|
52
|
+
metadata?: Record<string, any> | undefined;
|
|
53
53
|
email: string;
|
|
54
54
|
success: boolean;
|
|
55
55
|
operationType: "transactional" | "event" | "campaign" | "loop";
|
|
@@ -71,7 +71,7 @@ declare const _default: {
|
|
|
71
71
|
success: import("zod").ZodBoolean;
|
|
72
72
|
messageId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
73
73
|
metadata: import("zod").ZodOptional<import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodAny>>;
|
|
74
|
-
}>, "required", "email" | "success" | "
|
|
74
|
+
}>, "required", "email" | "success" | "operationType" | "actorId" | "transactionalId" | "campaignId" | "loopId" | "eventName" | "timestamp" | "messageId" | "metadata" | `metadata.${string}`>, {
|
|
75
75
|
email: ["email", "timestamp", "_creationTime"];
|
|
76
76
|
actorId: ["actorId", "timestamp", "_creationTime"];
|
|
77
77
|
operationType: ["operationType", "timestamp", "_creationTime"];
|