@mastra/upstash 0.10.2 → 0.10.3-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +17 -17
- package/CHANGELOG.md +26 -0
- package/PAGINATION.md +397 -0
- package/dist/_tsup-dts-rollup.d.cts +80 -6
- package/dist/_tsup-dts-rollup.d.ts +80 -6
- package/dist/index.cjs +1794 -1267
- package/dist/index.js +1794 -1267
- package/package.json +11 -10
- package/src/storage/index.ts +576 -121
- package/src/storage/upstash.test.ts +428 -53
package/src/storage/index.ts
CHANGED
|
@@ -93,17 +93,6 @@ export class UpstashStore extends MastraStorage {
|
|
|
93
93
|
return `${tableName}:${keyParts.join(':')}`;
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
private ensureDate(date: Date | string | undefined): Date | undefined {
|
|
97
|
-
if (!date) return undefined;
|
|
98
|
-
return date instanceof Date ? date : new Date(date);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
private serializeDate(date: Date | string | undefined): string | undefined {
|
|
102
|
-
if (!date) return undefined;
|
|
103
|
-
const dateObj = this.ensureDate(date);
|
|
104
|
-
return dateObj?.toISOString();
|
|
105
|
-
}
|
|
106
|
-
|
|
107
96
|
/**
|
|
108
97
|
* Scans for keys matching the given pattern using SCAN and returns them as an array.
|
|
109
98
|
* @param pattern Redis key pattern, e.g. "table:*"
|
|
@@ -204,27 +193,30 @@ export class UpstashStore extends MastraStorage {
|
|
|
204
193
|
return { key, processedRecord };
|
|
205
194
|
}
|
|
206
195
|
|
|
196
|
+
/**
|
|
197
|
+
* @deprecated Use getEvals instead
|
|
198
|
+
*/
|
|
207
199
|
async getEvalsByAgentName(agentName: string, type?: 'test' | 'live'): Promise<EvalRow[]> {
|
|
208
200
|
try {
|
|
209
|
-
// Get all keys that match the evals table pattern
|
|
210
201
|
const pattern = `${TABLE_EVALS}:*`;
|
|
211
202
|
const keys = await this.scanKeys(pattern);
|
|
212
203
|
|
|
213
|
-
//
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
);
|
|
204
|
+
// Check if we have any keys before using pipeline
|
|
205
|
+
if (keys.length === 0) {
|
|
206
|
+
return [];
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Use pipeline for batch fetching to improve performance
|
|
210
|
+
const pipeline = this.redis.pipeline();
|
|
211
|
+
keys.forEach(key => pipeline.get(key));
|
|
212
|
+
const results = await pipeline.exec();
|
|
220
213
|
|
|
221
214
|
// Filter by agent name and remove nulls
|
|
222
|
-
const nonNullRecords =
|
|
215
|
+
const nonNullRecords = results.filter(
|
|
223
216
|
(record): record is Record<string, any> =>
|
|
224
217
|
record !== null && typeof record === 'object' && 'agent_name' in record && record.agent_name === agentName,
|
|
225
218
|
);
|
|
226
219
|
|
|
227
|
-
// Apply additional filtering based on type
|
|
228
220
|
let filteredEvals = nonNullRecords;
|
|
229
221
|
|
|
230
222
|
if (type === 'test') {
|
|
@@ -271,103 +263,126 @@ export class UpstashStore extends MastraStorage {
|
|
|
271
263
|
}
|
|
272
264
|
}
|
|
273
265
|
|
|
274
|
-
async getTraces(
|
|
275
|
-
|
|
266
|
+
public async getTraces(args: {
|
|
267
|
+
name?: string;
|
|
268
|
+
scope?: string;
|
|
269
|
+
attributes?: Record<string, string>;
|
|
270
|
+
filters?: Record<string, any>;
|
|
271
|
+
page: number;
|
|
272
|
+
perPage?: number;
|
|
273
|
+
fromDate?: Date;
|
|
274
|
+
toDate?: Date;
|
|
275
|
+
}): Promise<any[]>;
|
|
276
|
+
public async getTraces(args: {
|
|
277
|
+
name?: string;
|
|
278
|
+
scope?: string;
|
|
279
|
+
page: number;
|
|
280
|
+
perPage?: number;
|
|
281
|
+
attributes?: Record<string, string>;
|
|
282
|
+
filters?: Record<string, any>;
|
|
283
|
+
fromDate?: Date;
|
|
284
|
+
toDate?: Date;
|
|
285
|
+
returnPaginationResults: true;
|
|
286
|
+
}): Promise<{
|
|
287
|
+
traces: any[];
|
|
288
|
+
total: number;
|
|
289
|
+
page: number;
|
|
290
|
+
perPage: number;
|
|
291
|
+
hasMore: boolean;
|
|
292
|
+
}>;
|
|
293
|
+
public async getTraces(args: {
|
|
294
|
+
name?: string;
|
|
295
|
+
scope?: string;
|
|
296
|
+
page: number;
|
|
297
|
+
perPage?: number;
|
|
298
|
+
attributes?: Record<string, string>;
|
|
299
|
+
filters?: Record<string, any>;
|
|
300
|
+
fromDate?: Date;
|
|
301
|
+
toDate?: Date;
|
|
302
|
+
returnPaginationResults?: boolean;
|
|
303
|
+
}): Promise<
|
|
304
|
+
| any[]
|
|
305
|
+
| {
|
|
306
|
+
traces: any[];
|
|
307
|
+
total: number;
|
|
308
|
+
page: number;
|
|
309
|
+
perPage: number;
|
|
310
|
+
hasMore: boolean;
|
|
311
|
+
}
|
|
312
|
+
> {
|
|
313
|
+
const {
|
|
276
314
|
name,
|
|
277
315
|
scope,
|
|
278
|
-
page
|
|
279
|
-
perPage
|
|
316
|
+
page,
|
|
317
|
+
perPage: perPageInput,
|
|
280
318
|
attributes,
|
|
281
319
|
filters,
|
|
282
320
|
fromDate,
|
|
283
321
|
toDate,
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
attributes?: Record<string, string>;
|
|
290
|
-
filters?: Record<string, any>;
|
|
291
|
-
fromDate?: Date;
|
|
292
|
-
toDate?: Date;
|
|
293
|
-
} = {
|
|
294
|
-
page: 0,
|
|
295
|
-
perPage: 100,
|
|
296
|
-
},
|
|
297
|
-
): Promise<any[]> {
|
|
322
|
+
returnPaginationResults,
|
|
323
|
+
} = args;
|
|
324
|
+
|
|
325
|
+
const perPage = perPageInput !== undefined ? perPageInput : 100;
|
|
326
|
+
|
|
298
327
|
try {
|
|
299
|
-
// Get all keys that match the traces table pattern
|
|
300
328
|
const pattern = `${TABLE_TRACES}:*`;
|
|
301
329
|
const keys = await this.scanKeys(pattern);
|
|
302
330
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
331
|
+
if (keys.length === 0) {
|
|
332
|
+
if (returnPaginationResults) {
|
|
333
|
+
return {
|
|
334
|
+
traces: [],
|
|
335
|
+
total: 0,
|
|
336
|
+
page,
|
|
337
|
+
perPage: perPage || 100,
|
|
338
|
+
hasMore: false,
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
return [];
|
|
342
|
+
}
|
|
310
343
|
|
|
311
|
-
|
|
312
|
-
|
|
344
|
+
const pipeline = this.redis.pipeline();
|
|
345
|
+
keys.forEach(key => pipeline.get(key));
|
|
346
|
+
const results = await pipeline.exec();
|
|
347
|
+
|
|
348
|
+
let filteredTraces = results.filter(
|
|
313
349
|
(record): record is Record<string, any> => record !== null && typeof record === 'object',
|
|
314
350
|
);
|
|
315
351
|
|
|
316
|
-
// Apply name filter if provided
|
|
317
352
|
if (name) {
|
|
318
353
|
filteredTraces = filteredTraces.filter(record => record.name?.toLowerCase().startsWith(name.toLowerCase()));
|
|
319
354
|
}
|
|
320
|
-
|
|
321
|
-
// Apply scope filter if provided
|
|
322
355
|
if (scope) {
|
|
323
356
|
filteredTraces = filteredTraces.filter(record => record.scope === scope);
|
|
324
357
|
}
|
|
325
|
-
|
|
326
|
-
// Apply attributes filter if provided
|
|
327
358
|
if (attributes) {
|
|
328
359
|
filteredTraces = filteredTraces.filter(record => {
|
|
329
360
|
const recordAttributes = record.attributes;
|
|
330
361
|
if (!recordAttributes) return false;
|
|
331
|
-
|
|
332
|
-
// Parse attributes if stored as string
|
|
333
362
|
const parsedAttributes =
|
|
334
363
|
typeof recordAttributes === 'string' ? JSON.parse(recordAttributes) : recordAttributes;
|
|
335
|
-
|
|
336
364
|
return Object.entries(attributes).every(([key, value]) => parsedAttributes[key] === value);
|
|
337
365
|
});
|
|
338
366
|
}
|
|
339
|
-
|
|
340
|
-
// Apply custom filters if provided
|
|
341
367
|
if (filters) {
|
|
342
368
|
filteredTraces = filteredTraces.filter(record =>
|
|
343
369
|
Object.entries(filters).every(([key, value]) => record[key] === value),
|
|
344
370
|
);
|
|
345
371
|
}
|
|
346
|
-
|
|
347
|
-
// Apply fromDate filter if provided
|
|
348
372
|
if (fromDate) {
|
|
349
373
|
filteredTraces = filteredTraces.filter(
|
|
350
374
|
record => new Date(record.createdAt).getTime() >= new Date(fromDate).getTime(),
|
|
351
375
|
);
|
|
352
376
|
}
|
|
353
|
-
|
|
354
|
-
// Apply toDate filter if provided
|
|
355
377
|
if (toDate) {
|
|
356
378
|
filteredTraces = filteredTraces.filter(
|
|
357
379
|
record => new Date(record.createdAt).getTime() <= new Date(toDate).getTime(),
|
|
358
380
|
);
|
|
359
381
|
}
|
|
360
382
|
|
|
361
|
-
// Sort traces by creation date (newest first)
|
|
362
383
|
filteredTraces.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
|
|
363
384
|
|
|
364
|
-
|
|
365
|
-
const start = page * perPage;
|
|
366
|
-
const end = start + perPage;
|
|
367
|
-
const paginatedTraces = filteredTraces.slice(start, end);
|
|
368
|
-
|
|
369
|
-
// Transform and return the traces
|
|
370
|
-
return paginatedTraces.map(record => ({
|
|
385
|
+
const transformedTraces = filteredTraces.map(record => ({
|
|
371
386
|
id: record.id,
|
|
372
387
|
parentSpanId: record.parentSpanId,
|
|
373
388
|
traceId: record.traceId,
|
|
@@ -383,8 +398,35 @@ export class UpstashStore extends MastraStorage {
|
|
|
383
398
|
other: this.parseJSON(record.other),
|
|
384
399
|
createdAt: this.ensureDate(record.createdAt),
|
|
385
400
|
}));
|
|
401
|
+
|
|
402
|
+
const total = transformedTraces.length;
|
|
403
|
+
const resolvedPerPage = perPage || 100;
|
|
404
|
+
const start = page * resolvedPerPage;
|
|
405
|
+
const end = start + resolvedPerPage;
|
|
406
|
+
const paginatedTraces = transformedTraces.slice(start, end);
|
|
407
|
+
const hasMore = end < total;
|
|
408
|
+
if (returnPaginationResults) {
|
|
409
|
+
return {
|
|
410
|
+
traces: paginatedTraces,
|
|
411
|
+
total,
|
|
412
|
+
page,
|
|
413
|
+
perPage: resolvedPerPage,
|
|
414
|
+
hasMore,
|
|
415
|
+
};
|
|
416
|
+
} else {
|
|
417
|
+
return paginatedTraces;
|
|
418
|
+
}
|
|
386
419
|
} catch (error) {
|
|
387
420
|
console.error('Failed to get traces:', error);
|
|
421
|
+
if (returnPaginationResults) {
|
|
422
|
+
return {
|
|
423
|
+
traces: [],
|
|
424
|
+
total: 0,
|
|
425
|
+
page,
|
|
426
|
+
perPage: perPage || 100,
|
|
427
|
+
hasMore: false,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
388
430
|
return [];
|
|
389
431
|
}
|
|
390
432
|
}
|
|
@@ -401,6 +443,20 @@ export class UpstashStore extends MastraStorage {
|
|
|
401
443
|
await this.redis.set(`schema:${tableName}`, schema);
|
|
402
444
|
}
|
|
403
445
|
|
|
446
|
+
/**
|
|
447
|
+
* No-op: This backend is schemaless and does not require schema changes.
|
|
448
|
+
* @param tableName Name of the table
|
|
449
|
+
* @param schema Schema of the table
|
|
450
|
+
* @param ifNotExists Array of column names to add if they don't exist
|
|
451
|
+
*/
|
|
452
|
+
async alterTable(_args: {
|
|
453
|
+
tableName: TABLE_NAMES;
|
|
454
|
+
schema: Record<string, StorageColumn>;
|
|
455
|
+
ifNotExists: string[];
|
|
456
|
+
}): Promise<void> {
|
|
457
|
+
// Nothing to do here, Redis is schemaless
|
|
458
|
+
}
|
|
459
|
+
|
|
404
460
|
async clearTable({ tableName }: { tableName: TABLE_NAMES }): Promise<void> {
|
|
405
461
|
const pattern = `${tableName}:*`;
|
|
406
462
|
await this.scanAndDelete(pattern);
|
|
@@ -450,24 +506,97 @@ export class UpstashStore extends MastraStorage {
|
|
|
450
506
|
};
|
|
451
507
|
}
|
|
452
508
|
|
|
453
|
-
async getThreadsByResourceId(
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
509
|
+
async getThreadsByResourceId(args: { resourceId: string }): Promise<StorageThreadType[]>;
|
|
510
|
+
async getThreadsByResourceId(args: { resourceId: string; page: number; perPage?: number }): Promise<{
|
|
511
|
+
threads: StorageThreadType[];
|
|
512
|
+
total: number;
|
|
513
|
+
page: number;
|
|
514
|
+
perPage: number;
|
|
515
|
+
hasMore: boolean;
|
|
516
|
+
}>;
|
|
517
|
+
async getThreadsByResourceId(args: { resourceId: string; page?: number; perPage?: number }): Promise<
|
|
518
|
+
| StorageThreadType[]
|
|
519
|
+
| {
|
|
520
|
+
threads: StorageThreadType[];
|
|
521
|
+
total: number;
|
|
522
|
+
page: number;
|
|
523
|
+
perPage: number;
|
|
524
|
+
hasMore: boolean;
|
|
525
|
+
}
|
|
526
|
+
> {
|
|
527
|
+
const resourceId: string = args.resourceId;
|
|
528
|
+
const page: number | undefined = args.page;
|
|
529
|
+
// Determine perPage only if page is actually provided. Otherwise, its value is not critical for the non-paginated path.
|
|
530
|
+
// If page is provided, perPage defaults to 100 if not specified.
|
|
531
|
+
const perPage: number = page !== undefined ? (args.perPage !== undefined ? args.perPage : 100) : 100;
|
|
532
|
+
|
|
533
|
+
try {
|
|
534
|
+
const pattern = `${TABLE_THREADS}:*`;
|
|
535
|
+
const keys = await this.scanKeys(pattern);
|
|
536
|
+
|
|
537
|
+
if (keys.length === 0) {
|
|
538
|
+
if (page !== undefined) {
|
|
539
|
+
return {
|
|
540
|
+
threads: [],
|
|
541
|
+
total: 0,
|
|
542
|
+
page,
|
|
543
|
+
perPage, // perPage is number here
|
|
544
|
+
hasMore: false,
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
return [];
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
const allThreads: StorageThreadType[] = [];
|
|
551
|
+
const pipeline = this.redis.pipeline();
|
|
552
|
+
keys.forEach(key => pipeline.get(key));
|
|
553
|
+
const results = await pipeline.exec();
|
|
554
|
+
|
|
555
|
+
for (let i = 0; i < results.length; i++) {
|
|
556
|
+
const thread = results[i] as StorageThreadType | null;
|
|
557
|
+
if (thread && thread.resourceId === resourceId) {
|
|
558
|
+
allThreads.push({
|
|
559
|
+
...thread,
|
|
560
|
+
createdAt: this.ensureDate(thread.createdAt)!,
|
|
561
|
+
updatedAt: this.ensureDate(thread.updatedAt)!,
|
|
562
|
+
metadata: typeof thread.metadata === 'string' ? JSON.parse(thread.metadata) : thread.metadata,
|
|
563
|
+
});
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
allThreads.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
|
|
568
|
+
|
|
569
|
+
if (page !== undefined) {
|
|
570
|
+
// If page is defined, perPage is also a number (due to the defaulting logic above)
|
|
571
|
+
const total = allThreads.length;
|
|
572
|
+
const start = page * perPage;
|
|
573
|
+
const end = start + perPage;
|
|
574
|
+
const paginatedThreads = allThreads.slice(start, end);
|
|
575
|
+
const hasMore = end < total;
|
|
576
|
+
return {
|
|
577
|
+
threads: paginatedThreads,
|
|
578
|
+
total,
|
|
579
|
+
page,
|
|
580
|
+
perPage,
|
|
581
|
+
hasMore,
|
|
582
|
+
};
|
|
583
|
+
} else {
|
|
584
|
+
// page is undefined, return all threads
|
|
585
|
+
return allThreads;
|
|
586
|
+
}
|
|
587
|
+
} catch (error) {
|
|
588
|
+
console.error('Error in getThreadsByResourceId:', error);
|
|
589
|
+
if (page !== undefined) {
|
|
590
|
+
return {
|
|
591
|
+
threads: [],
|
|
592
|
+
total: 0,
|
|
593
|
+
page,
|
|
594
|
+
perPage, // perPage is number here
|
|
595
|
+
hasMore: false,
|
|
596
|
+
};
|
|
597
|
+
}
|
|
598
|
+
return [];
|
|
599
|
+
}
|
|
471
600
|
}
|
|
472
601
|
|
|
473
602
|
async saveThread({ thread }: { thread: StorageThreadType }): Promise<StorageThreadType> {
|
|
@@ -506,8 +635,25 @@ export class UpstashStore extends MastraStorage {
|
|
|
506
635
|
}
|
|
507
636
|
|
|
508
637
|
async deleteThread({ threadId }: { threadId: string }): Promise<void> {
|
|
509
|
-
|
|
510
|
-
|
|
638
|
+
// Delete thread metadata and sorted set
|
|
639
|
+
const threadKey = this.getKey(TABLE_THREADS, { id: threadId });
|
|
640
|
+
const threadMessagesKey = this.getThreadMessagesKey(threadId);
|
|
641
|
+
const messageIds: string[] = await this.redis.zrange(threadMessagesKey, 0, -1);
|
|
642
|
+
|
|
643
|
+
const pipeline = this.redis.pipeline();
|
|
644
|
+
pipeline.del(threadKey);
|
|
645
|
+
pipeline.del(threadMessagesKey);
|
|
646
|
+
|
|
647
|
+
for (let i = 0; i < messageIds.length; i++) {
|
|
648
|
+
const messageId = messageIds[i];
|
|
649
|
+
const messageKey = this.getMessageKey(threadId, messageId as string);
|
|
650
|
+
pipeline.del(messageKey);
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
await pipeline.exec();
|
|
654
|
+
|
|
655
|
+
// Bulk delete all message keys for this thread if any remain
|
|
656
|
+
await this.scanAndDelete(this.getMessageKey(threadId, '*'));
|
|
511
657
|
}
|
|
512
658
|
|
|
513
659
|
async saveMessages(args: { messages: MastraMessageV1[]; format?: undefined | 'v1' }): Promise<MastraMessageV1[]>;
|
|
@@ -550,17 +696,155 @@ export class UpstashStore extends MastraStorage {
|
|
|
550
696
|
return list.get.all.v1();
|
|
551
697
|
}
|
|
552
698
|
|
|
699
|
+
// Function overloads for different return types
|
|
553
700
|
public async getMessages(args: StorageGetMessagesArg & { format?: 'v1' }): Promise<MastraMessageV1[]>;
|
|
554
701
|
public async getMessages(args: StorageGetMessagesArg & { format: 'v2' }): Promise<MastraMessageV2[]>;
|
|
702
|
+
public async getMessages(
|
|
703
|
+
args: StorageGetMessagesArg & {
|
|
704
|
+
format?: 'v1' | 'v2';
|
|
705
|
+
page: number;
|
|
706
|
+
perPage?: number;
|
|
707
|
+
fromDate?: Date;
|
|
708
|
+
toDate?: Date;
|
|
709
|
+
},
|
|
710
|
+
): Promise<{
|
|
711
|
+
messages: MastraMessageV1[] | MastraMessageV2[];
|
|
712
|
+
total: number;
|
|
713
|
+
page: number;
|
|
714
|
+
perPage: number;
|
|
715
|
+
hasMore: boolean;
|
|
716
|
+
}>;
|
|
555
717
|
public async getMessages({
|
|
556
718
|
threadId,
|
|
557
719
|
selectBy,
|
|
558
720
|
format,
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
721
|
+
page,
|
|
722
|
+
perPage = 40,
|
|
723
|
+
fromDate,
|
|
724
|
+
toDate,
|
|
725
|
+
}: StorageGetMessagesArg & {
|
|
726
|
+
format?: 'v1' | 'v2';
|
|
727
|
+
page?: number;
|
|
728
|
+
perPage?: number;
|
|
729
|
+
fromDate?: Date;
|
|
730
|
+
toDate?: Date;
|
|
731
|
+
}): Promise<
|
|
732
|
+
| MastraMessageV1[]
|
|
733
|
+
| MastraMessageV2[]
|
|
734
|
+
| {
|
|
735
|
+
messages: MastraMessageV1[] | MastraMessageV2[];
|
|
736
|
+
total: number;
|
|
737
|
+
page: number;
|
|
738
|
+
perPage: number;
|
|
739
|
+
hasMore: boolean;
|
|
740
|
+
}
|
|
741
|
+
> {
|
|
562
742
|
const threadMessagesKey = this.getThreadMessagesKey(threadId);
|
|
563
743
|
|
|
744
|
+
const allMessageIds = await this.redis.zrange(threadMessagesKey, 0, -1);
|
|
745
|
+
// If pagination is requested, use the new pagination logic
|
|
746
|
+
if (page !== undefined) {
|
|
747
|
+
try {
|
|
748
|
+
// Get all message IDs from the sorted set
|
|
749
|
+
|
|
750
|
+
if (allMessageIds.length === 0) {
|
|
751
|
+
return {
|
|
752
|
+
messages: [],
|
|
753
|
+
total: 0,
|
|
754
|
+
page,
|
|
755
|
+
perPage,
|
|
756
|
+
hasMore: false,
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
// Use pipeline to fetch all messages efficiently
|
|
761
|
+
const pipeline = this.redis.pipeline();
|
|
762
|
+
allMessageIds.forEach(id => pipeline.get(this.getMessageKey(threadId, id as string)));
|
|
763
|
+
const results = await pipeline.exec();
|
|
764
|
+
|
|
765
|
+
// Process messages and apply filters - handle undefined results from pipeline
|
|
766
|
+
let messages = results
|
|
767
|
+
.map((result: any) => result as MastraMessageV2 | null)
|
|
768
|
+
.filter((msg): msg is MastraMessageV2 => msg !== null) as (MastraMessageV2 & { _index?: number })[];
|
|
769
|
+
|
|
770
|
+
// Apply date filters if provided
|
|
771
|
+
if (fromDate) {
|
|
772
|
+
messages = messages.filter(msg => msg && new Date(msg.createdAt).getTime() >= fromDate.getTime());
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
if (toDate) {
|
|
776
|
+
messages = messages.filter(msg => msg && new Date(msg.createdAt).getTime() <= toDate.getTime());
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
// Sort messages by their position in the sorted set
|
|
780
|
+
messages.sort((a, b) => allMessageIds.indexOf(a!.id) - allMessageIds.indexOf(b!.id));
|
|
781
|
+
|
|
782
|
+
const total = messages.length;
|
|
783
|
+
|
|
784
|
+
// Apply pagination
|
|
785
|
+
const start = page * perPage;
|
|
786
|
+
const end = start + perPage;
|
|
787
|
+
const hasMore = end < total;
|
|
788
|
+
const paginatedMessages = messages.slice(start, end);
|
|
789
|
+
|
|
790
|
+
// Remove _index before returning and handle format conversion properly
|
|
791
|
+
const prepared = paginatedMessages
|
|
792
|
+
.filter(message => message !== null && message !== undefined)
|
|
793
|
+
.map(message => {
|
|
794
|
+
const { _index, ...messageWithoutIndex } = message as MastraMessageV2 & { _index?: number };
|
|
795
|
+
return messageWithoutIndex as unknown as MastraMessageV1;
|
|
796
|
+
});
|
|
797
|
+
|
|
798
|
+
// Return pagination object with correct format
|
|
799
|
+
if (format === 'v2') {
|
|
800
|
+
// Convert V1 format back to V2 format
|
|
801
|
+
const v2Messages = prepared.map(msg => ({
|
|
802
|
+
...msg,
|
|
803
|
+
content: msg.content || { format: 2, parts: [{ type: 'text', text: '' }] },
|
|
804
|
+
})) as MastraMessageV2[];
|
|
805
|
+
|
|
806
|
+
return {
|
|
807
|
+
messages: v2Messages,
|
|
808
|
+
total,
|
|
809
|
+
page,
|
|
810
|
+
perPage,
|
|
811
|
+
hasMore,
|
|
812
|
+
};
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
return {
|
|
816
|
+
messages: prepared,
|
|
817
|
+
total,
|
|
818
|
+
page,
|
|
819
|
+
perPage,
|
|
820
|
+
hasMore,
|
|
821
|
+
};
|
|
822
|
+
} catch (error) {
|
|
823
|
+
console.error('Failed to get paginated messages:', error);
|
|
824
|
+
return {
|
|
825
|
+
messages: [],
|
|
826
|
+
total: 0,
|
|
827
|
+
page,
|
|
828
|
+
perPage,
|
|
829
|
+
hasMore: false,
|
|
830
|
+
};
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
// Original logic for backward compatibility
|
|
835
|
+
// When selectBy is undefined or selectBy.last is undefined, get ALL messages (not just 40)
|
|
836
|
+
let limit: number;
|
|
837
|
+
if (typeof selectBy?.last === 'number') {
|
|
838
|
+
limit = Math.max(0, selectBy.last);
|
|
839
|
+
} else if (selectBy?.last === false) {
|
|
840
|
+
limit = 0;
|
|
841
|
+
} else {
|
|
842
|
+
// No limit specified - get all messages
|
|
843
|
+
limit = Number.MAX_SAFE_INTEGER;
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
const messageIds = new Set<string>();
|
|
847
|
+
|
|
564
848
|
if (limit === 0 && !selectBy?.include) {
|
|
565
849
|
return [];
|
|
566
850
|
}
|
|
@@ -591,9 +875,16 @@ export class UpstashStore extends MastraStorage {
|
|
|
591
875
|
}
|
|
592
876
|
}
|
|
593
877
|
|
|
594
|
-
// Then get the most recent messages
|
|
595
|
-
|
|
596
|
-
|
|
878
|
+
// Then get the most recent messages (or all if no limit)
|
|
879
|
+
if (limit === Number.MAX_SAFE_INTEGER) {
|
|
880
|
+
// Get all messages
|
|
881
|
+
const allIds = await this.redis.zrange(threadMessagesKey, 0, -1);
|
|
882
|
+
allIds.forEach(id => messageIds.add(id as string));
|
|
883
|
+
} else if (limit > 0) {
|
|
884
|
+
// Get limited number of recent messages
|
|
885
|
+
const latestIds = await this.redis.zrange(threadMessagesKey, -limit, -1);
|
|
886
|
+
latestIds.forEach(id => messageIds.add(id as string));
|
|
887
|
+
}
|
|
597
888
|
|
|
598
889
|
// Fetch all needed messages in parallel
|
|
599
890
|
const messages = (
|
|
@@ -605,15 +896,27 @@ export class UpstashStore extends MastraStorage {
|
|
|
605
896
|
).filter(msg => msg !== null) as (MastraMessageV2 & { _index?: number })[];
|
|
606
897
|
|
|
607
898
|
// Sort messages by their position in the sorted set
|
|
608
|
-
|
|
609
|
-
|
|
899
|
+
messages.sort((a, b) => allMessageIds.indexOf(a!.id) - allMessageIds.indexOf(b!.id));
|
|
900
|
+
|
|
901
|
+
// Remove _index before returning and handle format conversion properly
|
|
902
|
+
const prepared = messages
|
|
903
|
+
.filter(message => message !== null && message !== undefined)
|
|
904
|
+
.map(message => {
|
|
905
|
+
const { _index, ...messageWithoutIndex } = message as MastraMessageV2 & { _index?: number };
|
|
906
|
+
return messageWithoutIndex as unknown as MastraMessageV1;
|
|
907
|
+
});
|
|
610
908
|
|
|
611
|
-
//
|
|
612
|
-
|
|
909
|
+
// For backward compatibility, return messages directly without using MessageList
|
|
910
|
+
// since MessageList has deduplication logic that can cause issues
|
|
911
|
+
if (format === 'v2') {
|
|
912
|
+
// Convert V1 format back to V2 format
|
|
913
|
+
return prepared.map(msg => ({
|
|
914
|
+
...msg,
|
|
915
|
+
content: msg.content || { format: 2, parts: [{ type: 'text', text: '' }] },
|
|
916
|
+
})) as MastraMessageV2[];
|
|
917
|
+
}
|
|
613
918
|
|
|
614
|
-
|
|
615
|
-
if (format === `v2`) return list.get.all.v2();
|
|
616
|
-
return list.get.all.v1();
|
|
919
|
+
return prepared;
|
|
617
920
|
}
|
|
618
921
|
|
|
619
922
|
async persistWorkflowSnapshot(params: {
|
|
@@ -657,6 +960,157 @@ export class UpstashStore extends MastraStorage {
|
|
|
657
960
|
return data.snapshot;
|
|
658
961
|
}
|
|
659
962
|
|
|
963
|
+
/**
|
|
964
|
+
* Get all evaluations with pagination and total count
|
|
965
|
+
* @param options Pagination and filtering options
|
|
966
|
+
* @returns Object with evals array and total count
|
|
967
|
+
*/
|
|
968
|
+
async getEvals(options?: {
|
|
969
|
+
agentName?: string;
|
|
970
|
+
type?: 'test' | 'live';
|
|
971
|
+
page?: number;
|
|
972
|
+
perPage?: number;
|
|
973
|
+
limit?: number;
|
|
974
|
+
offset?: number;
|
|
975
|
+
fromDate?: Date;
|
|
976
|
+
toDate?: Date;
|
|
977
|
+
}): Promise<{
|
|
978
|
+
evals: EvalRow[];
|
|
979
|
+
total: number;
|
|
980
|
+
page?: number;
|
|
981
|
+
perPage?: number;
|
|
982
|
+
hasMore?: boolean;
|
|
983
|
+
}> {
|
|
984
|
+
try {
|
|
985
|
+
// Default pagination parameters
|
|
986
|
+
const page = options?.page ?? 0;
|
|
987
|
+
const perPage = options?.perPage ?? 100;
|
|
988
|
+
const limit = options?.limit;
|
|
989
|
+
const offset = options?.offset;
|
|
990
|
+
|
|
991
|
+
// Get all keys that match the evals table pattern using cursor-based scanning
|
|
992
|
+
const pattern = `${TABLE_EVALS}:*`;
|
|
993
|
+
const keys = await this.scanKeys(pattern);
|
|
994
|
+
|
|
995
|
+
// Check if we have any keys before using pipeline
|
|
996
|
+
if (keys.length === 0) {
|
|
997
|
+
return {
|
|
998
|
+
evals: [],
|
|
999
|
+
total: 0,
|
|
1000
|
+
page: options?.page ?? 0,
|
|
1001
|
+
perPage: options?.perPage ?? 100,
|
|
1002
|
+
hasMore: false,
|
|
1003
|
+
};
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
// Use pipeline for batch fetching to improve performance
|
|
1007
|
+
const pipeline = this.redis.pipeline();
|
|
1008
|
+
keys.forEach(key => pipeline.get(key));
|
|
1009
|
+
const results = await pipeline.exec();
|
|
1010
|
+
|
|
1011
|
+
// Process results and apply filters
|
|
1012
|
+
let filteredEvals = results
|
|
1013
|
+
.map((result: any) => result as Record<string, any> | null)
|
|
1014
|
+
.filter((record): record is Record<string, any> => record !== null && typeof record === 'object');
|
|
1015
|
+
|
|
1016
|
+
// Apply agent name filter if provided
|
|
1017
|
+
if (options?.agentName) {
|
|
1018
|
+
filteredEvals = filteredEvals.filter(record => record.agent_name === options.agentName);
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
// Apply type filter if provided
|
|
1022
|
+
if (options?.type === 'test') {
|
|
1023
|
+
filteredEvals = filteredEvals.filter(record => {
|
|
1024
|
+
if (!record.test_info) return false;
|
|
1025
|
+
|
|
1026
|
+
try {
|
|
1027
|
+
if (typeof record.test_info === 'string') {
|
|
1028
|
+
const parsedTestInfo = JSON.parse(record.test_info);
|
|
1029
|
+
return parsedTestInfo && typeof parsedTestInfo === 'object' && 'testPath' in parsedTestInfo;
|
|
1030
|
+
}
|
|
1031
|
+
return typeof record.test_info === 'object' && 'testPath' in record.test_info;
|
|
1032
|
+
} catch {
|
|
1033
|
+
return false;
|
|
1034
|
+
}
|
|
1035
|
+
});
|
|
1036
|
+
} else if (options?.type === 'live') {
|
|
1037
|
+
filteredEvals = filteredEvals.filter(record => {
|
|
1038
|
+
if (!record.test_info) return true;
|
|
1039
|
+
|
|
1040
|
+
try {
|
|
1041
|
+
if (typeof record.test_info === 'string') {
|
|
1042
|
+
const parsedTestInfo = JSON.parse(record.test_info);
|
|
1043
|
+
return !(parsedTestInfo && typeof parsedTestInfo === 'object' && 'testPath' in parsedTestInfo);
|
|
1044
|
+
}
|
|
1045
|
+
return !(typeof record.test_info === 'object' && 'testPath' in record.test_info);
|
|
1046
|
+
} catch {
|
|
1047
|
+
return true;
|
|
1048
|
+
}
|
|
1049
|
+
});
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
// Apply date filters if provided
|
|
1053
|
+
if (options?.fromDate) {
|
|
1054
|
+
filteredEvals = filteredEvals.filter(record => {
|
|
1055
|
+
const createdAt = new Date(record.created_at || record.createdAt || 0);
|
|
1056
|
+
return createdAt.getTime() >= options.fromDate!.getTime();
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
if (options?.toDate) {
|
|
1061
|
+
filteredEvals = filteredEvals.filter(record => {
|
|
1062
|
+
const createdAt = new Date(record.created_at || record.createdAt || 0);
|
|
1063
|
+
return createdAt.getTime() <= options.toDate!.getTime();
|
|
1064
|
+
});
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
// Sort by creation date (newest first)
|
|
1068
|
+
filteredEvals.sort((a, b) => {
|
|
1069
|
+
const dateA = new Date(a.created_at || a.createdAt || 0).getTime();
|
|
1070
|
+
const dateB = new Date(b.created_at || b.createdAt || 0).getTime();
|
|
1071
|
+
return dateB - dateA;
|
|
1072
|
+
});
|
|
1073
|
+
|
|
1074
|
+
const total = filteredEvals.length;
|
|
1075
|
+
|
|
1076
|
+
// Apply pagination - support both page/perPage and limit/offset patterns
|
|
1077
|
+
let paginatedEvals: Record<string, any>[];
|
|
1078
|
+
let hasMore = false;
|
|
1079
|
+
|
|
1080
|
+
if (limit !== undefined && offset !== undefined) {
|
|
1081
|
+
// Offset-based pagination
|
|
1082
|
+
paginatedEvals = filteredEvals.slice(offset, offset + limit);
|
|
1083
|
+
hasMore = offset + limit < total;
|
|
1084
|
+
} else {
|
|
1085
|
+
// Page-based pagination
|
|
1086
|
+
const start = page * perPage;
|
|
1087
|
+
const end = start + perPage;
|
|
1088
|
+
paginatedEvals = filteredEvals.slice(start, end);
|
|
1089
|
+
hasMore = end < total;
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
// Transform to EvalRow format
|
|
1093
|
+
const evals = paginatedEvals.map(record => this.transformEvalRecord(record));
|
|
1094
|
+
|
|
1095
|
+
return {
|
|
1096
|
+
evals,
|
|
1097
|
+
total,
|
|
1098
|
+
page: limit !== undefined ? undefined : page,
|
|
1099
|
+
perPage: limit !== undefined ? undefined : perPage,
|
|
1100
|
+
hasMore,
|
|
1101
|
+
};
|
|
1102
|
+
} catch (error) {
|
|
1103
|
+
console.error('Failed to get evals:', error);
|
|
1104
|
+
return {
|
|
1105
|
+
evals: [],
|
|
1106
|
+
total: 0,
|
|
1107
|
+
page: options?.page ?? 0,
|
|
1108
|
+
perPage: options?.perPage ?? 100,
|
|
1109
|
+
hasMore: false,
|
|
1110
|
+
};
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
|
|
660
1114
|
async getWorkflowRuns(
|
|
661
1115
|
{
|
|
662
1116
|
namespace,
|
|
@@ -693,24 +1147,25 @@ export class UpstashStore extends MastraStorage {
|
|
|
693
1147
|
}
|
|
694
1148
|
const keys = await this.scanKeys(pattern);
|
|
695
1149
|
|
|
696
|
-
//
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
workflow_name: string;
|
|
701
|
-
run_id: string;
|
|
702
|
-
snapshot: WorkflowRunState | string;
|
|
703
|
-
createdAt: string | Date;
|
|
704
|
-
updatedAt: string | Date;
|
|
705
|
-
resourceId: string;
|
|
706
|
-
}>(key);
|
|
707
|
-
return data;
|
|
708
|
-
}),
|
|
709
|
-
);
|
|
1150
|
+
// Check if we have any keys before using pipeline
|
|
1151
|
+
if (keys.length === 0) {
|
|
1152
|
+
return { runs: [], total: 0 };
|
|
1153
|
+
}
|
|
710
1154
|
|
|
711
|
-
//
|
|
712
|
-
|
|
713
|
-
|
|
1155
|
+
// Use pipeline for batch fetching to improve performance
|
|
1156
|
+
const pipeline = this.redis.pipeline();
|
|
1157
|
+
keys.forEach(key => pipeline.get(key));
|
|
1158
|
+
const results = await pipeline.exec();
|
|
1159
|
+
|
|
1160
|
+
// Filter and transform results - handle undefined results
|
|
1161
|
+
let runs = results
|
|
1162
|
+
.map((result: any) => result as Record<string, any> | null)
|
|
1163
|
+
.filter(
|
|
1164
|
+
(record): record is Record<string, any> =>
|
|
1165
|
+
record !== null && record !== undefined && typeof record === 'object' && 'workflow_name' in record,
|
|
1166
|
+
)
|
|
1167
|
+
// Only filter by workflowName if it was specifically requested
|
|
1168
|
+
.filter(record => !workflowName || record.workflow_name === workflowName)
|
|
714
1169
|
.map(w => this.parseWorkflowRun(w!))
|
|
715
1170
|
.filter(w => {
|
|
716
1171
|
if (fromDate && w.createdAt < fromDate) return false;
|