@output.ai/core 0.3.0-dev.pr263-5d2eaa9 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.d.ts +158 -41
- package/src/index.js +2 -0
- package/src/interface/evaluator.js +147 -65
- package/src/interface/evaluator.spec.js +463 -1
- package/src/interface/validations/static.js +16 -0
- package/src/interface/validations/static.spec.js +79 -1
- package/src/interface/workflow_utils.js +49 -0
- package/src/interface/workflow_utils.spec.js +190 -0
package/package.json
CHANGED
package/src/index.d.ts
CHANGED
|
@@ -79,6 +79,15 @@ export type HttpMethod = 'HEAD' | 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH';
|
|
|
79
79
|
*/
|
|
80
80
|
export type TemporalActivityOptions = Omit<ActivityOptions, 'versioningIntent' | 'taskQueue' | 'allowEagerDispatch'>;
|
|
81
81
|
|
|
82
|
+
/**
|
|
83
|
+
* Result of a single job executed by `executeInParallel`.
|
|
84
|
+
*
|
|
85
|
+
* @typeParam T - The return type of the job function
|
|
86
|
+
*/
|
|
87
|
+
export type ParallelJobResult<T> =
|
|
88
|
+
| { ok: true; result: T; index: number } |
|
|
89
|
+
{ ok: false; error: unknown; index: number };
|
|
90
|
+
|
|
82
91
|
/*
|
|
83
92
|
╭─────────╮
|
|
84
93
|
│ S T E P │╮
|
|
@@ -480,6 +489,81 @@ export declare function workflow<
|
|
|
480
489
|
╰───────────────────╯
|
|
481
490
|
*/
|
|
482
491
|
|
|
492
|
+
/**
|
|
493
|
+
* A single feedback for an EvaluationResult
|
|
494
|
+
*/
|
|
495
|
+
export class EvaluationFeedback {
|
|
496
|
+
/**
|
|
497
|
+
* Issue found
|
|
498
|
+
*/
|
|
499
|
+
issue: string;
|
|
500
|
+
|
|
501
|
+
/**
|
|
502
|
+
* Improvement suggestion
|
|
503
|
+
*/
|
|
504
|
+
suggestion?: string;
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* Reference for the issue
|
|
508
|
+
*/
|
|
509
|
+
reference?: string;
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* Issue priority
|
|
513
|
+
*/
|
|
514
|
+
priority?: 'low' | 'medium' | 'high' | 'critical';
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* @constructor
|
|
518
|
+
* @param args
|
|
519
|
+
* @param args.issue
|
|
520
|
+
* @param args.suggestion
|
|
521
|
+
* @param args.reference
|
|
522
|
+
* @param args.priority
|
|
523
|
+
*/
|
|
524
|
+
constructor( args: {
|
|
525
|
+
issue: string;
|
|
526
|
+
suggestion?: string;
|
|
527
|
+
reference?: string;
|
|
528
|
+
priority?: 'low' | 'medium' | 'high' | 'critical';
|
|
529
|
+
} );
|
|
530
|
+
|
|
531
|
+
/**
|
|
532
|
+
* @returns The zod schema for this class
|
|
533
|
+
*/
|
|
534
|
+
static get schema(): z.ZodType;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
/**
|
|
538
|
+
* Base constructor arguments for EvaluationResult classes
|
|
539
|
+
*/
|
|
540
|
+
export type EvaluationResultArgs<TValue = any> = { // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
541
|
+
/**
|
|
542
|
+
* The value of the evaluation
|
|
543
|
+
*/
|
|
544
|
+
value: TValue;
|
|
545
|
+
/**
|
|
546
|
+
* The confidence in the evaluation
|
|
547
|
+
*/
|
|
548
|
+
confidence: number;
|
|
549
|
+
/**
|
|
550
|
+
* The name of the evaluation
|
|
551
|
+
*/
|
|
552
|
+
name?: string;
|
|
553
|
+
/**
|
|
554
|
+
* The reasoning behind the result
|
|
555
|
+
*/
|
|
556
|
+
reasoning?: string;
|
|
557
|
+
/**
|
|
558
|
+
* Feedback for this evaluation
|
|
559
|
+
*/
|
|
560
|
+
feedback?: EvaluationFeedback[];
|
|
561
|
+
/**
|
|
562
|
+
* Dimensions of this evaluation
|
|
563
|
+
*/
|
|
564
|
+
dimensions?: Array<EvaluationStringResult | EvaluationNumberResult | EvaluationBooleanResult>;
|
|
565
|
+
};
|
|
566
|
+
|
|
483
567
|
/**
|
|
484
568
|
* Represents the result of an evaluation.
|
|
485
569
|
*
|
|
@@ -487,33 +571,45 @@ export declare function workflow<
|
|
|
487
571
|
*/
|
|
488
572
|
export class EvaluationResult {
|
|
489
573
|
/**
|
|
490
|
-
*
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
574
|
+
* The name of the evaluation result
|
|
575
|
+
*/
|
|
576
|
+
name?: string;
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* The evaluation result value
|
|
580
|
+
*/
|
|
581
|
+
value: any; // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* The evaluation result confidence
|
|
495
585
|
*/
|
|
496
|
-
|
|
586
|
+
confidence: number;
|
|
497
587
|
|
|
498
588
|
/**
|
|
499
|
-
*
|
|
589
|
+
* The evaluation result reasoning
|
|
500
590
|
*/
|
|
501
|
-
|
|
591
|
+
reasoning?: string;
|
|
502
592
|
|
|
503
593
|
/**
|
|
504
|
-
*
|
|
594
|
+
* Feedback for this evaluation
|
|
505
595
|
*/
|
|
506
|
-
|
|
596
|
+
feedback: EvaluationFeedback[];
|
|
507
597
|
|
|
508
598
|
/**
|
|
509
|
-
*
|
|
599
|
+
* Dimensions of this evaluation
|
|
510
600
|
*/
|
|
511
|
-
|
|
601
|
+
dimensions: Array<EvaluationStringResult | EvaluationNumberResult | EvaluationBooleanResult>;
|
|
512
602
|
|
|
513
603
|
/**
|
|
514
|
-
* @
|
|
604
|
+
* @constructor
|
|
605
|
+
* @param args
|
|
606
|
+
*/
|
|
607
|
+
constructor( args: EvaluationResultArgs );
|
|
608
|
+
|
|
609
|
+
/**
|
|
610
|
+
* @returns The zod schema for this class
|
|
515
611
|
*/
|
|
516
|
-
get schema():
|
|
612
|
+
static get schema(): z.ZodType;
|
|
517
613
|
}
|
|
518
614
|
|
|
519
615
|
/**
|
|
@@ -524,16 +620,8 @@ export class EvaluationStringResult extends EvaluationResult {
|
|
|
524
620
|
/**
|
|
525
621
|
* @constructor
|
|
526
622
|
* @param args
|
|
527
|
-
* @param args.value - The value of the evaluation
|
|
528
|
-
* @param args.confidence - The confidence on the evaluation
|
|
529
|
-
* @param args.reasoning - The reasoning behind the result
|
|
530
623
|
*/
|
|
531
|
-
constructor( args:
|
|
532
|
-
|
|
533
|
-
/**
|
|
534
|
-
* @returns The evaluation result value
|
|
535
|
-
*/
|
|
536
|
-
get value(): string;
|
|
624
|
+
constructor( args: EvaluationResultArgs<string> );
|
|
537
625
|
}
|
|
538
626
|
|
|
539
627
|
/**
|
|
@@ -544,16 +632,8 @@ export class EvaluationNumberResult extends EvaluationResult {
|
|
|
544
632
|
/**
|
|
545
633
|
* @constructor
|
|
546
634
|
* @param args
|
|
547
|
-
* @param args.value - The value of the evaluation
|
|
548
|
-
* @param args.confidence - The confidence on the evaluation
|
|
549
|
-
* @param args.reasoning - The reasoning behind the result
|
|
550
|
-
*/
|
|
551
|
-
constructor( args: { value: number; confidence: number; reasoning?: string } );
|
|
552
|
-
|
|
553
|
-
/**
|
|
554
|
-
* @returns The evaluation result value
|
|
555
635
|
*/
|
|
556
|
-
|
|
636
|
+
constructor( args: EvaluationResultArgs<number> );
|
|
557
637
|
}
|
|
558
638
|
|
|
559
639
|
/**
|
|
@@ -564,16 +644,8 @@ export class EvaluationBooleanResult extends EvaluationResult {
|
|
|
564
644
|
/**
|
|
565
645
|
* @constructor
|
|
566
646
|
* @param args
|
|
567
|
-
* @param args.value - The value of the evaluation
|
|
568
|
-
* @param args.confidence - The confidence on the evaluation
|
|
569
|
-
* @param args.reasoning - The reasoning behind the result
|
|
570
647
|
*/
|
|
571
|
-
constructor( args:
|
|
572
|
-
|
|
573
|
-
/**
|
|
574
|
-
* @returns The evaluation result value
|
|
575
|
-
*/
|
|
576
|
-
get value(): boolean;
|
|
648
|
+
constructor( args: EvaluationResultArgs<boolean> );
|
|
577
649
|
}
|
|
578
650
|
|
|
579
651
|
/**
|
|
@@ -717,6 +789,51 @@ export declare function sendHttpRequest( params: {
|
|
|
717
789
|
headers?: Record<string, string>;
|
|
718
790
|
} ): Promise<SerializedFetchResponse>;
|
|
719
791
|
|
|
792
|
+
/**
|
|
793
|
+
* Execute jobs in parallel with optional concurrency limit.
|
|
794
|
+
*
|
|
795
|
+
* Returns all job results (successes and failures) sorted by original job index.
|
|
796
|
+
* Each result contains `ok` (boolean), `index` (original position), and either
|
|
797
|
+
* `result` (on success) or `error` (on failure).
|
|
798
|
+
*
|
|
799
|
+
* Jobs must be wrapped in arrow functions—do not pass promises directly.
|
|
800
|
+
*
|
|
801
|
+
* @example
|
|
802
|
+
* ```ts
|
|
803
|
+
* const results = await executeInParallel( {
|
|
804
|
+
* jobs: [
|
|
805
|
+
* () => myStep( data1 ),
|
|
806
|
+
* () => myStep( data2 ),
|
|
807
|
+
* () => myStep( data3 )
|
|
808
|
+
* ],
|
|
809
|
+
* concurrency: 2
|
|
810
|
+
* } );
|
|
811
|
+
*
|
|
812
|
+
* // Handle the discriminated union (result only exists when ok is true)
|
|
813
|
+
* const successfulResults = results.filter( r => r.ok ).map( r => r.result );
|
|
814
|
+
*
|
|
815
|
+
* // Or handle each result individually
|
|
816
|
+
* for ( const r of results ) {
|
|
817
|
+
* if ( r.ok ) {
|
|
818
|
+
* console.log( `Job ${r.index} succeeded:`, r.result );
|
|
819
|
+
* } else {
|
|
820
|
+
* console.log( `Job ${r.index} failed:`, r.error );
|
|
821
|
+
* }
|
|
822
|
+
* }
|
|
823
|
+
* ```
|
|
824
|
+
*
|
|
825
|
+
* @param params - Parameters object
|
|
826
|
+
* @param params.jobs - Array of arrow functions returning step/activity calls (not promises directly)
|
|
827
|
+
* @param params.concurrency - Max concurrent jobs (default: Infinity)
|
|
828
|
+
* @param params.onJobCompleted - Optional callback invoked as each job completes (in completion order)
|
|
829
|
+
* @returns Array of results sorted by original job index
|
|
830
|
+
*/
|
|
831
|
+
export declare function executeInParallel<T>( params: {
|
|
832
|
+
jobs: Array<() => Promise<T> | T>;
|
|
833
|
+
concurrency?: number;
|
|
834
|
+
onJobCompleted?: ( result: ParallelJobResult<T> ) => void;
|
|
835
|
+
} ): Promise<Array<ParallelJobResult<T>>>;
|
|
836
|
+
|
|
720
837
|
/*
|
|
721
838
|
╭─────────────╮
|
|
722
839
|
│ E R R O R S │╮
|
package/src/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { evaluator, EvaluationStringResult, EvaluationNumberResult, EvaluationBooleanResult } from './interface/evaluator.js';
|
|
2
2
|
import { step } from './interface/step.js';
|
|
3
3
|
import { workflow } from './interface/workflow.js';
|
|
4
|
+
import { executeInParallel } from './interface/workflow_utils.js';
|
|
4
5
|
import { sendHttpRequest, sendPostRequestAndAwaitWebhook } from './interface/webhook.js';
|
|
5
6
|
import { FatalError, ValidationError } from './errors.js';
|
|
6
7
|
export { continueAsNew, sleep } from '@temporalio/workflow';
|
|
@@ -16,6 +17,7 @@ export {
|
|
|
16
17
|
EvaluationStringResult,
|
|
17
18
|
EvaluationBooleanResult,
|
|
18
19
|
// webhook tools
|
|
20
|
+
executeInParallel,
|
|
19
21
|
sendHttpRequest,
|
|
20
22
|
sendPostRequestAndAwaitWebhook,
|
|
21
23
|
// errors
|
|
@@ -11,56 +11,158 @@ import * as z from 'zod';
|
|
|
11
11
|
class EvaluationResultValidationError extends ValidationError {};
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
15
|
-
|
|
14
|
+
* A single feedback for an EvaluationResult
|
|
15
|
+
*/
|
|
16
|
+
export class EvaluationFeedback {
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Issue found
|
|
20
|
+
* @type {string}
|
|
21
|
+
*/
|
|
22
|
+
issue;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Improvement suggestion
|
|
26
|
+
* @type {string}
|
|
27
|
+
*/
|
|
28
|
+
suggestion;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Reference for the issue
|
|
32
|
+
* @type {string}
|
|
33
|
+
*/
|
|
34
|
+
reference;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Issue priority
|
|
38
|
+
* @type {'low' | 'medium' | 'high' | 'critical' | undefined}
|
|
39
|
+
*/
|
|
40
|
+
priority;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The zod schema for this class
|
|
44
|
+
* @type {z.ZodType}
|
|
45
|
+
*/
|
|
46
|
+
static get schema() {
|
|
47
|
+
return z.object( {
|
|
48
|
+
issue: z.string(),
|
|
49
|
+
suggestion: z.string().optional(),
|
|
50
|
+
reference: z.string().optional(),
|
|
51
|
+
priority: z.enum( [ 'low', 'medium', 'high', 'critical' ] ).optional()
|
|
52
|
+
} );
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* @constructor
|
|
57
|
+
* @param {object} args
|
|
58
|
+
* @param {string} args.issue
|
|
59
|
+
* @param {string} [args.suggestion]
|
|
60
|
+
* @param {string} [args.reference]
|
|
61
|
+
* @param {'low' | 'medium' | 'high' | 'critical'} [args.priority]
|
|
62
|
+
*/
|
|
63
|
+
constructor( { issue, suggestion = undefined, reference = undefined, priority = undefined } ) {
|
|
64
|
+
const result = this.constructor.schema.safeParse( { issue, suggestion, reference, priority } );
|
|
65
|
+
if ( result.error ) {
|
|
66
|
+
throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
|
|
67
|
+
}
|
|
68
|
+
this.issue = issue;
|
|
69
|
+
this.suggestion = suggestion;
|
|
70
|
+
this.reference = reference;
|
|
71
|
+
this.priority = priority;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Generic EvaluationResult class, represents the result of an evaluation.
|
|
16
77
|
*/
|
|
17
78
|
export class EvaluationResult {
|
|
18
79
|
|
|
19
80
|
/**
|
|
20
|
-
*
|
|
81
|
+
* The name of the evaluation result
|
|
82
|
+
* @type {string}
|
|
83
|
+
*/
|
|
84
|
+
name;
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* The evaluation result value
|
|
21
88
|
* @type {any}
|
|
22
89
|
*/
|
|
23
90
|
value = null;
|
|
24
91
|
|
|
25
92
|
/**
|
|
26
|
-
*
|
|
93
|
+
* The confidence value, between 0 and 1
|
|
27
94
|
* @type {number}
|
|
28
95
|
*/
|
|
29
|
-
confidence
|
|
96
|
+
confidence;
|
|
30
97
|
|
|
31
98
|
/**
|
|
32
|
-
*
|
|
99
|
+
* The reasoning value
|
|
33
100
|
* @type {string}
|
|
34
101
|
*/
|
|
35
|
-
reasoning
|
|
102
|
+
reasoning;
|
|
36
103
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
104
|
+
/**
|
|
105
|
+
* Feedback for this evaluation
|
|
106
|
+
* @type {EvaluationFeedback[]}
|
|
107
|
+
*/
|
|
108
|
+
feedback = [];
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Dimensions of this evaluation
|
|
112
|
+
* @type {EvaluationResult[]}
|
|
113
|
+
*/
|
|
114
|
+
dimensions = [];
|
|
40
115
|
|
|
116
|
+
/**
|
|
117
|
+
* The schema main field
|
|
118
|
+
* @type {z.ZodAny}
|
|
119
|
+
*/
|
|
120
|
+
static valueSchema = z.any();
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* The zod schema for this class
|
|
124
|
+
* @type {z.ZodType}
|
|
125
|
+
*/
|
|
41
126
|
static get schema() {
|
|
42
|
-
|
|
127
|
+
const baseSchema = z.object( {
|
|
43
128
|
value: this.valueSchema,
|
|
44
129
|
confidence: z.number(),
|
|
45
|
-
reasoning: z.string().optional()
|
|
130
|
+
reasoning: z.string().optional(),
|
|
131
|
+
name: z.string().optional(),
|
|
132
|
+
feedback: z.array( EvaluationFeedback.schema ).optional()
|
|
133
|
+
} );
|
|
134
|
+
|
|
135
|
+
// Adds dimension but keep it only one level deep
|
|
136
|
+
return baseSchema.extend( {
|
|
137
|
+
dimensions: z.array(
|
|
138
|
+
baseSchema.extend( {
|
|
139
|
+
value: z.union( [ z.string(), z.number(), z.boolean() ] )
|
|
140
|
+
} )
|
|
141
|
+
).optional()
|
|
46
142
|
} );
|
|
47
143
|
};
|
|
48
144
|
|
|
49
145
|
/**
|
|
50
146
|
* @constructor
|
|
51
147
|
* @param {object} args
|
|
52
|
-
* @param {any} args.value
|
|
53
|
-
* @param {number} args.confidence
|
|
54
|
-
* @param {string} [args.
|
|
148
|
+
* @param {any} args.value
|
|
149
|
+
* @param {number} args.confidence
|
|
150
|
+
* @param {string} [args.name]
|
|
151
|
+
* @param {EvaluationResult[]} [args.dimensions]
|
|
152
|
+
* @param {EvaluationFeedback[]} [args.feedback]
|
|
153
|
+
* @param {string} [args.reasoning]
|
|
55
154
|
*/
|
|
56
|
-
constructor(
|
|
57
|
-
const result = this.constructor.schema.safeParse(
|
|
155
|
+
constructor( { value, confidence, dimensions = [], feedback = [], name = undefined, reasoning = undefined } ) {
|
|
156
|
+
const result = this.constructor.schema.safeParse( { value, confidence, dimensions, feedback, name, reasoning } );
|
|
58
157
|
if ( result.error ) {
|
|
59
158
|
throw new EvaluationResultValidationError( z.prettifyError( result.error ) );
|
|
60
159
|
}
|
|
61
|
-
this.
|
|
62
|
-
this.
|
|
63
|
-
this.
|
|
160
|
+
this.confidence = confidence;
|
|
161
|
+
this.value = value;
|
|
162
|
+
this.dimensions = dimensions;
|
|
163
|
+
this.feedback = feedback;
|
|
164
|
+
this.name = name;
|
|
165
|
+
this.reasoning = reasoning;
|
|
64
166
|
}
|
|
65
167
|
};
|
|
66
168
|
|
|
@@ -68,71 +170,51 @@ export class EvaluationResult {
|
|
|
68
170
|
* An evaluation result that uses a string value
|
|
69
171
|
* @extends EvaluationResult
|
|
70
172
|
* @property {string} value - The evaluation result value
|
|
173
|
+
* @constructor
|
|
174
|
+
* @param {object} args
|
|
175
|
+
* @param {string} args.value - The value of the evaluation (must be a string)
|
|
176
|
+
* @see EvaluationResult#constructor for other parameters (confidence, reasoning)
|
|
71
177
|
*/
|
|
72
178
|
export class EvaluationStringResult extends EvaluationResult {
|
|
73
|
-
static
|
|
74
|
-
return z.string();
|
|
75
|
-
};
|
|
76
|
-
|
|
77
|
-
/**
|
|
78
|
-
* @constructor
|
|
79
|
-
* @param {object} args
|
|
80
|
-
* @param {string} args.value - The value of the evaluation
|
|
81
|
-
* @param {number} args.confidence - The confidence on the evaluation
|
|
82
|
-
* @param {string} [args.reasoning] - The reasoning behind the result
|
|
83
|
-
*/
|
|
84
|
-
// eslint-disable-next-line no-useless-constructor
|
|
85
|
-
constructor( args ) {
|
|
86
|
-
super( args );
|
|
87
|
-
}
|
|
179
|
+
static valueSchema = z.string();
|
|
88
180
|
};
|
|
89
181
|
|
|
90
182
|
/**
|
|
91
183
|
* An evaluation result that uses a boolean value
|
|
92
184
|
* @extends EvaluationResult
|
|
93
185
|
* @property {boolean} value - The evaluation result value
|
|
186
|
+
* @constructor
|
|
187
|
+
* @param {object} args
|
|
188
|
+
* @param {boolean} args.value - The value of the evaluation (must be a boolean)
|
|
189
|
+
* @see EvaluationResult#constructor for other parameters (confidence, reasoning)
|
|
94
190
|
*/
|
|
95
191
|
export class EvaluationBooleanResult extends EvaluationResult {
|
|
96
|
-
static
|
|
97
|
-
return z.boolean();
|
|
98
|
-
};
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* @constructor
|
|
102
|
-
* @param {object} args
|
|
103
|
-
* @param {boolean} args.value - The value of the evaluation
|
|
104
|
-
* @param {number} args.confidence - The confidence on the evaluation
|
|
105
|
-
* @param {string} [args.reasoning] - The reasoning behind the result
|
|
106
|
-
*/
|
|
107
|
-
// eslint-disable-next-line no-useless-constructor
|
|
108
|
-
constructor( args ) {
|
|
109
|
-
super( args );
|
|
110
|
-
}
|
|
192
|
+
static valueSchema = z.boolean();
|
|
111
193
|
};
|
|
112
194
|
|
|
113
195
|
/**
|
|
114
196
|
* An evaluation result that uses a number value
|
|
115
197
|
* @extends EvaluationResult
|
|
116
198
|
* @property {number} value - The evaluation result value
|
|
199
|
+
* @constructor
|
|
200
|
+
* @param {object} args
|
|
201
|
+
* @param {number} args.value - The value of the evaluation (must be a number)
|
|
202
|
+
* @see EvaluationResult#constructor for other parameters (confidence, reasoning)
|
|
117
203
|
*/
|
|
118
204
|
export class EvaluationNumberResult extends EvaluationResult {
|
|
119
|
-
static
|
|
120
|
-
return z.number();
|
|
121
|
-
};
|
|
122
|
-
|
|
123
|
-
/**
|
|
124
|
-
* @constructor
|
|
125
|
-
* @param {object} args
|
|
126
|
-
* @param {number} args.value - The value of the evaluation
|
|
127
|
-
* @param {number} args.confidence - The confidence on the evaluation
|
|
128
|
-
* @param {string} [args.reasoning] - The reasoning behind the result
|
|
129
|
-
*/
|
|
130
|
-
// eslint-disable-next-line no-useless-constructor
|
|
131
|
-
constructor( args ) {
|
|
132
|
-
super( args );
|
|
133
|
-
}
|
|
205
|
+
static valueSchema = z.number();
|
|
134
206
|
};
|
|
135
207
|
|
|
208
|
+
/**
|
|
209
|
+
* Expose the function to create a new evaluator
|
|
210
|
+
* @param {object} opts
|
|
211
|
+
* @param {string} opts.name
|
|
212
|
+
* @param {string} opts.description
|
|
213
|
+
* @param {z.ZodType} opts.inputSchema
|
|
214
|
+
* @param {Function} opts.fn
|
|
215
|
+
* @param {object} opts.options
|
|
216
|
+
* @returns {Function}
|
|
217
|
+
*/
|
|
136
218
|
export function evaluator( { name, description, inputSchema, fn, options } ) {
|
|
137
219
|
validateEvaluator( { name, description, inputSchema, fn, options } );
|
|
138
220
|
|
|
@@ -3,7 +3,8 @@ import {
|
|
|
3
3
|
EvaluationResult,
|
|
4
4
|
EvaluationStringResult,
|
|
5
5
|
EvaluationNumberResult,
|
|
6
|
-
EvaluationBooleanResult
|
|
6
|
+
EvaluationBooleanResult,
|
|
7
|
+
EvaluationFeedback
|
|
7
8
|
} from './evaluator.js';
|
|
8
9
|
import { ValidationError } from '#errors';
|
|
9
10
|
|
|
@@ -98,6 +99,467 @@ describe( 'interface/evaluator - EvaluationResult classes', () => {
|
|
|
98
99
|
const bad = EvaluationBooleanResult.schema.safeParse( { value: 'false', confidence: 1 } );
|
|
99
100
|
expect( bad.success ).toBe( false );
|
|
100
101
|
} );
|
|
102
|
+
|
|
103
|
+
it( 'schema getter does not cause infinite recursion', () => {
|
|
104
|
+
// Access schema multiple times to ensure no stack overflow
|
|
105
|
+
const schema1 = EvaluationResult.schema;
|
|
106
|
+
const schema2 = EvaluationResult.schema;
|
|
107
|
+
const schema3 = EvaluationStringResult.schema;
|
|
108
|
+
const schema4 = EvaluationNumberResult.schema;
|
|
109
|
+
const schema5 = EvaluationBooleanResult.schema;
|
|
110
|
+
|
|
111
|
+
expect( schema1 ).toBeDefined();
|
|
112
|
+
expect( schema2 ).toBeDefined();
|
|
113
|
+
expect( schema3 ).toBeDefined();
|
|
114
|
+
expect( schema4 ).toBeDefined();
|
|
115
|
+
expect( schema5 ).toBeDefined();
|
|
116
|
+
|
|
117
|
+
// Verify schemas can be used for validation
|
|
118
|
+
const result = schema1.safeParse( {
|
|
119
|
+
value: 'test',
|
|
120
|
+
confidence: 0.8,
|
|
121
|
+
dimensions: [
|
|
122
|
+
{ value: 'dim1', confidence: 0.9 },
|
|
123
|
+
{ value: 42, confidence: 0.8 }
|
|
124
|
+
]
|
|
125
|
+
} );
|
|
126
|
+
expect( result.success ).toBe( true );
|
|
127
|
+
} );
|
|
128
|
+
|
|
129
|
+
it( 'schema includes dimensions field', () => {
|
|
130
|
+
const schema = EvaluationResult.schema;
|
|
131
|
+
const result = schema.safeParse( {
|
|
132
|
+
value: 'test',
|
|
133
|
+
confidence: 0.8,
|
|
134
|
+
dimensions: [
|
|
135
|
+
{ value: 'string-dim', confidence: 0.9 },
|
|
136
|
+
{ value: 42, confidence: 0.8 },
|
|
137
|
+
{ value: true, confidence: 0.7 }
|
|
138
|
+
]
|
|
139
|
+
} );
|
|
140
|
+
expect( result.success ).toBe( true );
|
|
141
|
+
if ( result.success ) {
|
|
142
|
+
expect( result.data.dimensions ).toHaveLength( 3 );
|
|
143
|
+
}
|
|
144
|
+
} );
|
|
145
|
+
|
|
146
|
+
it( 'schema validates dimensions value types', () => {
|
|
147
|
+
const schema = EvaluationResult.schema;
|
|
148
|
+
|
|
149
|
+
// Valid: primitive value types
|
|
150
|
+
const valid = schema.safeParse( {
|
|
151
|
+
value: 'test',
|
|
152
|
+
confidence: 0.8,
|
|
153
|
+
dimensions: [
|
|
154
|
+
{ value: 'string', confidence: 0.9 },
|
|
155
|
+
{ value: 42, confidence: 0.8 },
|
|
156
|
+
{ value: true, confidence: 0.7 }
|
|
157
|
+
]
|
|
158
|
+
} );
|
|
159
|
+
expect( valid.success ).toBe( true );
|
|
160
|
+
|
|
161
|
+
// Invalid: non-primitive value type in dimensions
|
|
162
|
+
const invalid = schema.safeParse( {
|
|
163
|
+
value: 'test',
|
|
164
|
+
confidence: 0.8,
|
|
165
|
+
dimensions: [
|
|
166
|
+
{ value: { object: 'not allowed' }, confidence: 0.9 }
|
|
167
|
+
]
|
|
168
|
+
} );
|
|
169
|
+
expect( invalid.success ).toBe( false );
|
|
170
|
+
} );
|
|
171
|
+
} );
|
|
172
|
+
|
|
173
|
+
describe( 'new fields: name, dimensions, feedback', () => {
|
|
174
|
+
it( 'accepts optional name field', () => {
|
|
175
|
+
const result = new EvaluationResult( {
|
|
176
|
+
value: 'test',
|
|
177
|
+
confidence: 0.8,
|
|
178
|
+
name: 'test-evaluation'
|
|
179
|
+
} );
|
|
180
|
+
expect( result.name ).toBe( 'test-evaluation' );
|
|
181
|
+
} );
|
|
182
|
+
|
|
183
|
+
it( 'name defaults to undefined when not provided', () => {
|
|
184
|
+
const result = new EvaluationResult( {
|
|
185
|
+
value: 'test',
|
|
186
|
+
confidence: 0.8
|
|
187
|
+
} );
|
|
188
|
+
expect( result.name ).toBeUndefined();
|
|
189
|
+
} );
|
|
190
|
+
|
|
191
|
+
it( 'validates name must be string if provided', () => {
|
|
192
|
+
expect( () => new EvaluationResult( {
|
|
193
|
+
value: 'test',
|
|
194
|
+
confidence: 0.8,
|
|
195
|
+
name: 123
|
|
196
|
+
} ) ).toThrow( ValidationError );
|
|
197
|
+
} );
|
|
198
|
+
|
|
199
|
+
it( 'accepts dimensions array with EvaluationResult instances', () => {
|
|
200
|
+
const dim1 = new EvaluationStringResult( { value: 'dim1', confidence: 0.9 } );
|
|
201
|
+
const dim2 = new EvaluationNumberResult( { value: 42, confidence: 0.8 } );
|
|
202
|
+
const result = new EvaluationResult( {
|
|
203
|
+
value: 'main',
|
|
204
|
+
confidence: 0.7,
|
|
205
|
+
dimensions: [ dim1, dim2 ]
|
|
206
|
+
} );
|
|
207
|
+
expect( result.dimensions ).toHaveLength( 2 );
|
|
208
|
+
expect( result.dimensions[0] ).toBe( dim1 );
|
|
209
|
+
expect( result.dimensions[1] ).toBe( dim2 );
|
|
210
|
+
} );
|
|
211
|
+
|
|
212
|
+
it( 'dimensions defaults to empty array when not provided', () => {
|
|
213
|
+
const result = new EvaluationResult( {
|
|
214
|
+
value: 'test',
|
|
215
|
+
confidence: 0.8
|
|
216
|
+
} );
|
|
217
|
+
expect( result.dimensions ).toEqual( [] );
|
|
218
|
+
} );
|
|
219
|
+
|
|
220
|
+
it( 'validates dimensions must match subclass schema', () => {
|
|
221
|
+
expect( () => new EvaluationResult( {
|
|
222
|
+
value: 'test',
|
|
223
|
+
confidence: 0.8,
|
|
224
|
+
dimensions: [ { invalid: 'object' } ]
|
|
225
|
+
} ) ).toThrow( ValidationError );
|
|
226
|
+
} );
|
|
227
|
+
|
|
228
|
+
it( 'validates dimensions array structure', () => {
|
|
229
|
+
expect( () => new EvaluationResult( {
|
|
230
|
+
value: 'test',
|
|
231
|
+
confidence: 0.8,
|
|
232
|
+
dimensions: [
|
|
233
|
+
{ value: 'valid', confidence: 0.9 },
|
|
234
|
+
{ invalid: 'missing confidence' }
|
|
235
|
+
]
|
|
236
|
+
} ) ).toThrow( ValidationError );
|
|
237
|
+
} );
|
|
238
|
+
|
|
239
|
+
it( 'accepts dimensions with all three subclass types', () => {
|
|
240
|
+
const result = new EvaluationResult( {
|
|
241
|
+
value: 'main',
|
|
242
|
+
confidence: 0.7,
|
|
243
|
+
dimensions: [
|
|
244
|
+
new EvaluationStringResult( { value: 'string-dim', confidence: 0.9 } ),
|
|
245
|
+
new EvaluationNumberResult( { value: 42, confidence: 0.8 } ),
|
|
246
|
+
new EvaluationBooleanResult( { value: true, confidence: 0.7 } )
|
|
247
|
+
]
|
|
248
|
+
} );
|
|
249
|
+
expect( result.dimensions ).toHaveLength( 3 );
|
|
250
|
+
expect( result.dimensions[0] ).toBeInstanceOf( EvaluationStringResult );
|
|
251
|
+
expect( result.dimensions[1] ).toBeInstanceOf( EvaluationNumberResult );
|
|
252
|
+
expect( result.dimensions[2] ).toBeInstanceOf( EvaluationBooleanResult );
|
|
253
|
+
} );
|
|
254
|
+
|
|
255
|
+
it( 'accepts plain objects matching subclass schemas in dimensions', () => {
|
|
256
|
+
const result = new EvaluationResult( {
|
|
257
|
+
value: 'main',
|
|
258
|
+
confidence: 0.7,
|
|
259
|
+
dimensions: [
|
|
260
|
+
{ value: 'string-dim', confidence: 0.9 },
|
|
261
|
+
{ value: 42, confidence: 0.8 },
|
|
262
|
+
{ value: true, confidence: 0.7 }
|
|
263
|
+
]
|
|
264
|
+
} );
|
|
265
|
+
expect( result.dimensions ).toHaveLength( 3 );
|
|
266
|
+
expect( result.dimensions[0].value ).toBe( 'string-dim' );
|
|
267
|
+
expect( result.dimensions[1].value ).toBe( 42 );
|
|
268
|
+
expect( result.dimensions[2].value ).toBe( true );
|
|
269
|
+
} );
|
|
270
|
+
|
|
271
|
+
it( 'accepts string dimension value type', () => {
|
|
272
|
+
const result = new EvaluationResult( {
|
|
273
|
+
value: 'test',
|
|
274
|
+
confidence: 0.8,
|
|
275
|
+
dimensions: [
|
|
276
|
+
{ value: 'string-value', confidence: 0.9 }
|
|
277
|
+
]
|
|
278
|
+
} );
|
|
279
|
+
expect( result.dimensions[0].value ).toBe( 'string-value' );
|
|
280
|
+
} );
|
|
281
|
+
|
|
282
|
+
it( 'accepts number dimension value type', () => {
|
|
283
|
+
const result = new EvaluationResult( {
|
|
284
|
+
value: 'test',
|
|
285
|
+
confidence: 0.8,
|
|
286
|
+
dimensions: [
|
|
287
|
+
{ value: 123, confidence: 0.9 }
|
|
288
|
+
]
|
|
289
|
+
} );
|
|
290
|
+
expect( result.dimensions[0].value ).toBe( 123 );
|
|
291
|
+
} );
|
|
292
|
+
|
|
293
|
+
it( 'accepts boolean dimension value type', () => {
|
|
294
|
+
const result = new EvaluationResult( {
|
|
295
|
+
value: 'test',
|
|
296
|
+
confidence: 0.8,
|
|
297
|
+
dimensions: [
|
|
298
|
+
{ value: true, confidence: 0.9 }
|
|
299
|
+
]
|
|
300
|
+
} );
|
|
301
|
+
expect( result.dimensions[0].value ).toBe( true );
|
|
302
|
+
} );
|
|
303
|
+
|
|
304
|
+
it( 'rejects dimensions with non-primitive value types', () => {
|
|
305
|
+
expect( () => new EvaluationResult( {
|
|
306
|
+
value: 'test',
|
|
307
|
+
confidence: 0.8,
|
|
308
|
+
dimensions: [
|
|
309
|
+
{ value: { object: 'not allowed' }, confidence: 0.9 }
|
|
310
|
+
]
|
|
311
|
+
} ) ).toThrow( ValidationError );
|
|
312
|
+
} );
|
|
313
|
+
|
|
314
|
+
it( 'rejects dimensions with array value types', () => {
|
|
315
|
+
expect( () => new EvaluationResult( {
|
|
316
|
+
value: 'test',
|
|
317
|
+
confidence: 0.8,
|
|
318
|
+
dimensions: [
|
|
319
|
+
{ value: [ 1, 2, 3 ], confidence: 0.9 }
|
|
320
|
+
]
|
|
321
|
+
} ) ).toThrow( ValidationError );
|
|
322
|
+
} );
|
|
323
|
+
|
|
324
|
+
it( 'accepts nested dimensions (recursive)', () => {
|
|
325
|
+
const nestedDim = new EvaluationStringResult( {
|
|
326
|
+
value: 'nested',
|
|
327
|
+
confidence: 0.9,
|
|
328
|
+
dimensions: [
|
|
329
|
+
new EvaluationNumberResult( { value: 10, confidence: 0.8 } )
|
|
330
|
+
]
|
|
331
|
+
} );
|
|
332
|
+
const result = new EvaluationResult( {
|
|
333
|
+
value: 'main',
|
|
334
|
+
confidence: 0.7,
|
|
335
|
+
dimensions: [ nestedDim ]
|
|
336
|
+
} );
|
|
337
|
+
expect( result.dimensions ).toHaveLength( 1 );
|
|
338
|
+
expect( result.dimensions[0].dimensions ).toHaveLength( 1 );
|
|
339
|
+
expect( result.dimensions[0].dimensions[0].value ).toBe( 10 );
|
|
340
|
+
} );
|
|
341
|
+
|
|
342
|
+
it( 'accepts nested dimensions with plain objects', () => {
|
|
343
|
+
const result = new EvaluationResult( {
|
|
344
|
+
value: 'main',
|
|
345
|
+
confidence: 0.7,
|
|
346
|
+
dimensions: [
|
|
347
|
+
{
|
|
348
|
+
value: 'nested',
|
|
349
|
+
confidence: 0.9,
|
|
350
|
+
dimensions: [
|
|
351
|
+
{ value: 10, confidence: 0.8 }
|
|
352
|
+
]
|
|
353
|
+
}
|
|
354
|
+
]
|
|
355
|
+
} );
|
|
356
|
+
expect( result.dimensions ).toHaveLength( 1 );
|
|
357
|
+
expect( result.dimensions[0].dimensions ).toHaveLength( 1 );
|
|
358
|
+
expect( result.dimensions[0].dimensions[0].value ).toBe( 10 );
|
|
359
|
+
} );
|
|
360
|
+
|
|
361
|
+
it( 'accepts feedback array with EvaluationFeedback instances', () => {
|
|
362
|
+
const feedback1 = new EvaluationFeedback( {
|
|
363
|
+
issue: 'Issue 1',
|
|
364
|
+
suggestion: 'Fix this',
|
|
365
|
+
priority: 'high'
|
|
366
|
+
} );
|
|
367
|
+
const feedback2 = new EvaluationFeedback( {
|
|
368
|
+
issue: 'Issue 2',
|
|
369
|
+
reference: 'ref-123'
|
|
370
|
+
} );
|
|
371
|
+
const result = new EvaluationResult( {
|
|
372
|
+
value: 'test',
|
|
373
|
+
confidence: 0.8,
|
|
374
|
+
feedback: [ feedback1, feedback2 ]
|
|
375
|
+
} );
|
|
376
|
+
expect( result.feedback ).toHaveLength( 2 );
|
|
377
|
+
expect( result.feedback[0] ).toBe( feedback1 );
|
|
378
|
+
expect( result.feedback[1] ).toBe( feedback2 );
|
|
379
|
+
} );
|
|
380
|
+
|
|
381
|
+
it( 'accepts feedback array with plain objects matching schema', () => {
|
|
382
|
+
const result = new EvaluationResult( {
|
|
383
|
+
value: 'test',
|
|
384
|
+
confidence: 0.8,
|
|
385
|
+
feedback: [
|
|
386
|
+
{ issue: 'Issue 1', suggestion: 'Fix', priority: 'high' },
|
|
387
|
+
{ issue: 'Issue 2', reference: 'ref-123' }
|
|
388
|
+
]
|
|
389
|
+
} );
|
|
390
|
+
expect( result.feedback ).toHaveLength( 2 );
|
|
391
|
+
expect( result.feedback[0].issue ).toBe( 'Issue 1' );
|
|
392
|
+
expect( result.feedback[1].issue ).toBe( 'Issue 2' );
|
|
393
|
+
} );
|
|
394
|
+
|
|
395
|
+
it( 'feedback defaults to empty array when not provided', () => {
|
|
396
|
+
const result = new EvaluationResult( {
|
|
397
|
+
value: 'test',
|
|
398
|
+
confidence: 0.8
|
|
399
|
+
} );
|
|
400
|
+
expect( result.feedback ).toEqual( [] );
|
|
401
|
+
} );
|
|
402
|
+
|
|
403
|
+
it( 'validates feedback must match EvaluationFeedback schema', () => {
|
|
404
|
+
expect( () => new EvaluationResult( {
|
|
405
|
+
value: 'test',
|
|
406
|
+
confidence: 0.8,
|
|
407
|
+
feedback: [ { invalid: 'object' } ]
|
|
408
|
+
} ) ).toThrow( ValidationError );
|
|
409
|
+
} );
|
|
410
|
+
|
|
411
|
+
it( 'validates feedback issue is required', () => {
|
|
412
|
+
expect( () => new EvaluationResult( {
|
|
413
|
+
value: 'test',
|
|
414
|
+
confidence: 0.8,
|
|
415
|
+
feedback: [ { suggestion: 'missing issue' } ]
|
|
416
|
+
} ) ).toThrow( ValidationError );
|
|
417
|
+
} );
|
|
418
|
+
|
|
419
|
+
it( 'validates feedback priority enum values', () => {
|
|
420
|
+
expect( () => new EvaluationResult( {
|
|
421
|
+
value: 'test',
|
|
422
|
+
confidence: 0.8,
|
|
423
|
+
feedback: [ { issue: 'test', priority: 'invalid' } ]
|
|
424
|
+
} ) ).toThrow( ValidationError );
|
|
425
|
+
} );
|
|
426
|
+
|
|
427
|
+
it( 'accepts all new fields together', () => {
|
|
428
|
+
const dim = new EvaluationStringResult( { value: 'dim', confidence: 0.9 } );
|
|
429
|
+
const feedback = new EvaluationFeedback( { issue: 'test issue', priority: 'medium' } );
|
|
430
|
+
const result = new EvaluationResult( {
|
|
431
|
+
value: 'main',
|
|
432
|
+
confidence: 0.8,
|
|
433
|
+
name: 'comprehensive-test',
|
|
434
|
+
dimensions: [ dim ],
|
|
435
|
+
feedback: [ feedback ],
|
|
436
|
+
reasoning: 'test reasoning'
|
|
437
|
+
} );
|
|
438
|
+
expect( result.name ).toBe( 'comprehensive-test' );
|
|
439
|
+
expect( result.dimensions ).toHaveLength( 1 );
|
|
440
|
+
expect( result.feedback ).toHaveLength( 1 );
|
|
441
|
+
expect( result.reasoning ).toBe( 'test reasoning' );
|
|
442
|
+
} );
|
|
443
|
+
} );
|
|
444
|
+
} );
|
|
445
|
+
|
|
446
|
+
describe( 'interface/evaluator - EvaluationFeedback class', () => {
|
|
447
|
+
describe( 'constructor', () => {
|
|
448
|
+
it( 'creates feedback with required issue', () => {
|
|
449
|
+
const feedback = new EvaluationFeedback( { issue: 'Test issue' } );
|
|
450
|
+
expect( feedback.issue ).toBe( 'Test issue' );
|
|
451
|
+
expect( feedback.suggestion ).toBeUndefined();
|
|
452
|
+
expect( feedback.reference ).toBeUndefined();
|
|
453
|
+
expect( feedback.priority ).toBeUndefined();
|
|
454
|
+
} );
|
|
455
|
+
|
|
456
|
+
it( 'creates feedback with all fields', () => {
|
|
457
|
+
const feedback = new EvaluationFeedback( {
|
|
458
|
+
issue: 'Critical bug',
|
|
459
|
+
suggestion: 'Fix immediately',
|
|
460
|
+
reference: 'BUG-123',
|
|
461
|
+
priority: 'critical'
|
|
462
|
+
} );
|
|
463
|
+
expect( feedback.issue ).toBe( 'Critical bug' );
|
|
464
|
+
expect( feedback.suggestion ).toBe( 'Fix immediately' );
|
|
465
|
+
expect( feedback.reference ).toBe( 'BUG-123' );
|
|
466
|
+
expect( feedback.priority ).toBe( 'critical' );
|
|
467
|
+
} );
|
|
468
|
+
|
|
469
|
+
it( 'accepts optional fields', () => {
|
|
470
|
+
const feedback = new EvaluationFeedback( {
|
|
471
|
+
issue: 'Minor issue',
|
|
472
|
+
suggestion: 'Consider fixing'
|
|
473
|
+
} );
|
|
474
|
+
expect( feedback.issue ).toBe( 'Minor issue' );
|
|
475
|
+
expect( feedback.suggestion ).toBe( 'Consider fixing' );
|
|
476
|
+
expect( feedback.reference ).toBeUndefined();
|
|
477
|
+
expect( feedback.priority ).toBeUndefined();
|
|
478
|
+
} );
|
|
479
|
+
} );
|
|
480
|
+
|
|
481
|
+
describe( 'static schema getter', () => {
|
|
482
|
+
it( 'validates required issue field', () => {
|
|
483
|
+
const ok = EvaluationFeedback.schema.safeParse( { issue: 'Test issue' } );
|
|
484
|
+
expect( ok.success ).toBe( true );
|
|
485
|
+
|
|
486
|
+
const bad = EvaluationFeedback.schema.safeParse( {} );
|
|
487
|
+
expect( bad.success ).toBe( false );
|
|
488
|
+
} );
|
|
489
|
+
|
|
490
|
+
it( 'validates issue must be string', () => {
|
|
491
|
+
const bad = EvaluationFeedback.schema.safeParse( { issue: 123 } );
|
|
492
|
+
expect( bad.success ).toBe( false );
|
|
493
|
+
} );
|
|
494
|
+
|
|
495
|
+
it( 'accepts optional suggestion field', () => {
|
|
496
|
+
const ok = EvaluationFeedback.schema.safeParse( {
|
|
497
|
+
issue: 'Test',
|
|
498
|
+
suggestion: 'Fix it'
|
|
499
|
+
} );
|
|
500
|
+
expect( ok.success ).toBe( true );
|
|
501
|
+
} );
|
|
502
|
+
|
|
503
|
+
it( 'validates suggestion must be string if provided', () => {
|
|
504
|
+
const bad = EvaluationFeedback.schema.safeParse( {
|
|
505
|
+
issue: 'Test',
|
|
506
|
+
suggestion: 123
|
|
507
|
+
} );
|
|
508
|
+
expect( bad.success ).toBe( false );
|
|
509
|
+
} );
|
|
510
|
+
|
|
511
|
+
it( 'accepts optional reference field', () => {
|
|
512
|
+
const ok = EvaluationFeedback.schema.safeParse( {
|
|
513
|
+
issue: 'Test',
|
|
514
|
+
reference: 'REF-123'
|
|
515
|
+
} );
|
|
516
|
+
expect( ok.success ).toBe( true );
|
|
517
|
+
} );
|
|
518
|
+
|
|
519
|
+
it( 'validates reference must be string if provided', () => {
|
|
520
|
+
const bad = EvaluationFeedback.schema.safeParse( {
|
|
521
|
+
issue: 'Test',
|
|
522
|
+
reference: 123
|
|
523
|
+
} );
|
|
524
|
+
expect( bad.success ).toBe( false );
|
|
525
|
+
} );
|
|
526
|
+
|
|
527
|
+
it( 'accepts valid priority enum values', () => {
|
|
528
|
+
const priorities = [ 'low', 'medium', 'high', 'critical' ];
|
|
529
|
+
for ( const priority of priorities ) {
|
|
530
|
+
const ok = EvaluationFeedback.schema.safeParse( {
|
|
531
|
+
issue: 'Test',
|
|
532
|
+
priority
|
|
533
|
+
} );
|
|
534
|
+
expect( ok.success ).toBe( true );
|
|
535
|
+
}
|
|
536
|
+
} );
|
|
537
|
+
|
|
538
|
+
it( 'rejects invalid priority values', () => {
|
|
539
|
+
const bad = EvaluationFeedback.schema.safeParse( {
|
|
540
|
+
issue: 'Test',
|
|
541
|
+
priority: 'invalid'
|
|
542
|
+
} );
|
|
543
|
+
expect( bad.success ).toBe( false );
|
|
544
|
+
} );
|
|
545
|
+
|
|
546
|
+
it( 'validates priority must be string if provided', () => {
|
|
547
|
+
const bad = EvaluationFeedback.schema.safeParse( {
|
|
548
|
+
issue: 'Test',
|
|
549
|
+
priority: 123
|
|
550
|
+
} );
|
|
551
|
+
expect( bad.success ).toBe( false );
|
|
552
|
+
} );
|
|
553
|
+
|
|
554
|
+
it( 'accepts all fields together', () => {
|
|
555
|
+
const ok = EvaluationFeedback.schema.safeParse( {
|
|
556
|
+
issue: 'Critical bug',
|
|
557
|
+
suggestion: 'Fix immediately',
|
|
558
|
+
reference: 'BUG-123',
|
|
559
|
+
priority: 'critical'
|
|
560
|
+
} );
|
|
561
|
+
expect( ok.success ).toBe( true );
|
|
562
|
+
} );
|
|
101
563
|
} );
|
|
102
564
|
} );
|
|
103
565
|
|
|
@@ -17,6 +17,12 @@ const refineSchema = ( value, ctx ) => {
|
|
|
17
17
|
} );
|
|
18
18
|
};
|
|
19
19
|
|
|
20
|
+
export const executeInParallelSchema = z.object( {
|
|
21
|
+
jobs: z.array( z.function() ),
|
|
22
|
+
concurrency: z.number().min( 1 ).or( z.literal( Infinity ) ),
|
|
23
|
+
onJobCompleted: z.function().optional()
|
|
24
|
+
} );
|
|
25
|
+
|
|
20
26
|
export const durationSchema = z.union( [ z.string().regex(
|
|
21
27
|
/^(\d+)(ms|s|m|h|d)$/,
|
|
22
28
|
'Expected duration like "500ms", "10s", "5m", "2h", or "1d"'
|
|
@@ -108,3 +114,13 @@ export function validateWorkflow( args ) {
|
|
|
108
114
|
export function validateRequestPayload( args ) {
|
|
109
115
|
validateAgainstSchema( httpRequestSchema, args );
|
|
110
116
|
};
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Validate executeInParallel
|
|
120
|
+
*
|
|
121
|
+
* @param {object} args - The request arguments
|
|
122
|
+
* @throws {StaticValidationError} Throws if args are invalid
|
|
123
|
+
*/
|
|
124
|
+
export function validateExecuteInParallel( args ) {
|
|
125
|
+
validateAgainstSchema( executeInParallelSchema, args );
|
|
126
|
+
};
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
validateStep,
|
|
5
|
+
validateWorkflow,
|
|
6
|
+
validateRequestPayload,
|
|
7
|
+
validateEvaluator,
|
|
8
|
+
validateExecuteInParallel,
|
|
9
|
+
StaticValidationError
|
|
10
|
+
} from './static.js';
|
|
4
11
|
|
|
5
12
|
const validArgs = Object.freeze( {
|
|
6
13
|
name: 'valid_name',
|
|
@@ -259,4 +266,75 @@ describe( 'interface/validator', () => {
|
|
|
259
266
|
expect( () => validateRequestPayload( request ) ).not.toThrow();
|
|
260
267
|
} );
|
|
261
268
|
} );
|
|
269
|
+
|
|
270
|
+
describe( 'validateExecuteInParallel', () => {
|
|
271
|
+
const validArgs = Object.freeze( {
|
|
272
|
+
jobs: [ () => {}, () => {} ],
|
|
273
|
+
concurrency: 5
|
|
274
|
+
} );
|
|
275
|
+
|
|
276
|
+
it( 'passes for valid args', () => {
|
|
277
|
+
expect( () => validateExecuteInParallel( { ...validArgs } ) ).not.toThrow();
|
|
278
|
+
} );
|
|
279
|
+
|
|
280
|
+
it( 'rejects missing concurrency', () => {
|
|
281
|
+
const error = new StaticValidationError( '✖ Invalid input\n → at concurrency' );
|
|
282
|
+
expect( () => validateExecuteInParallel( { jobs: validArgs.jobs } ) ).toThrow( error );
|
|
283
|
+
} );
|
|
284
|
+
|
|
285
|
+
it( 'passes with onJobCompleted callback', () => {
|
|
286
|
+
const args = {
|
|
287
|
+
...validArgs,
|
|
288
|
+
onJobCompleted: () => {}
|
|
289
|
+
};
|
|
290
|
+
expect( () => validateExecuteInParallel( args ) ).not.toThrow();
|
|
291
|
+
} );
|
|
292
|
+
|
|
293
|
+
it( 'passes with concurrency 1', () => {
|
|
294
|
+
expect( () => validateExecuteInParallel( { ...validArgs, concurrency: 1 } ) ).not.toThrow();
|
|
295
|
+
} );
|
|
296
|
+
|
|
297
|
+
it( 'passes with concurrency Infinity', () => {
|
|
298
|
+
expect( () => validateExecuteInParallel( { ...validArgs, concurrency: Infinity } ) ).not.toThrow();
|
|
299
|
+
} );
|
|
300
|
+
|
|
301
|
+
it( 'rejects missing jobs', () => {
|
|
302
|
+
const error = new StaticValidationError( '✖ Invalid input: expected array, received undefined\n → at jobs' );
|
|
303
|
+
expect( () => validateExecuteInParallel( { concurrency: 5 } ) ).toThrow( error );
|
|
304
|
+
} );
|
|
305
|
+
|
|
306
|
+
it( 'rejects non-array jobs', () => {
|
|
307
|
+
const error = new StaticValidationError( '✖ Invalid input: expected array, received string\n → at jobs' );
|
|
308
|
+
expect( () => validateExecuteInParallel( { jobs: 'not-array', concurrency: 5 } ) ).toThrow( error );
|
|
309
|
+
} );
|
|
310
|
+
|
|
311
|
+
it( 'passes with empty jobs array', () => {
|
|
312
|
+
expect( () => validateExecuteInParallel( { jobs: [], concurrency: 5 } ) ).not.toThrow();
|
|
313
|
+
} );
|
|
314
|
+
|
|
315
|
+
it( 'rejects jobs array with non-function', () => {
|
|
316
|
+
const error = new StaticValidationError( '✖ Invalid input: expected function, received string\n → at jobs[1]' );
|
|
317
|
+
expect( () => validateExecuteInParallel( { jobs: [ () => {}, 'not-function' ], concurrency: 5 } ) ).toThrow( error );
|
|
318
|
+
} );
|
|
319
|
+
|
|
320
|
+
it( 'rejects non-number concurrency', () => {
|
|
321
|
+
const error = new StaticValidationError( '✖ Invalid input\n → at concurrency' );
|
|
322
|
+
expect( () => validateExecuteInParallel( { jobs: validArgs.jobs, concurrency: '5' } ) ).toThrow( error );
|
|
323
|
+
} );
|
|
324
|
+
|
|
325
|
+
it( 'rejects zero concurrency', () => {
|
|
326
|
+
const error = new StaticValidationError( '✖ Too small: expected number to be >=1\n → at concurrency' );
|
|
327
|
+
expect( () => validateExecuteInParallel( { jobs: validArgs.jobs, concurrency: 0 } ) ).toThrow( error );
|
|
328
|
+
} );
|
|
329
|
+
|
|
330
|
+
it( 'rejects negative concurrency', () => {
|
|
331
|
+
const error = new StaticValidationError( '✖ Too small: expected number to be >=1\n → at concurrency' );
|
|
332
|
+
expect( () => validateExecuteInParallel( { ...validArgs, concurrency: -1 } ) ).toThrow( error );
|
|
333
|
+
} );
|
|
334
|
+
|
|
335
|
+
it( 'rejects non-function onJobCompleted', () => {
|
|
336
|
+
const error = new StaticValidationError( '✖ Invalid input: expected function, received string\n → at onJobCompleted' );
|
|
337
|
+
expect( () => validateExecuteInParallel( { ...validArgs, onJobCompleted: 'not-function' } ) ).toThrow( error );
|
|
338
|
+
} );
|
|
339
|
+
} );
|
|
262
340
|
} );
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { validateExecuteInParallel } from './validations/static.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Execute jobs in parallel with optional concurrency limit.
|
|
5
|
+
*
|
|
6
|
+
* Returns all job results (successes and failures) sorted by original job index.
|
|
7
|
+
*
|
|
8
|
+
* @param {Array<Function>} jobs Array of functions to execute
|
|
9
|
+
* @param {Number} [concurrency] Max concurrent jobs, default is Infinity (no concurrency limit)
|
|
10
|
+
* @param {Function} [onJobCompleted] Optional callback invoked as each job completes
|
|
11
|
+
*/
|
|
12
|
+
export const executeInParallel = async ( { jobs, concurrency = Infinity, onJobCompleted } ) => {
|
|
13
|
+
validateExecuteInParallel( { jobs, concurrency, onJobCompleted } );
|
|
14
|
+
// allows this function to be called without testing over and over to check if it is not null;
|
|
15
|
+
const onJobCompletedSafeCb = onJobCompleted ?? ( _ => 0 );
|
|
16
|
+
const results = [];
|
|
17
|
+
const jobsCount = jobs.length;
|
|
18
|
+
const jobsPool = jobs.slice().map( ( job, index ) => ( {
|
|
19
|
+
index,
|
|
20
|
+
fn: async () => {
|
|
21
|
+
try {
|
|
22
|
+
const result = await job();
|
|
23
|
+
return { ok: true, result, index };
|
|
24
|
+
} catch ( error ) {
|
|
25
|
+
return { ok: false, error, index };
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
promise: null
|
|
29
|
+
} ) );
|
|
30
|
+
|
|
31
|
+
const activeJobs = jobsPool.splice( 0, concurrency );
|
|
32
|
+
activeJobs.forEach( job => job.promise = job.fn() ); // start jobs
|
|
33
|
+
|
|
34
|
+
while ( results.length < jobsCount ) {
|
|
35
|
+
const result = await Promise.race( activeJobs.map( job => job.promise ) );
|
|
36
|
+
results.push( result );
|
|
37
|
+
onJobCompletedSafeCb( result );
|
|
38
|
+
|
|
39
|
+
activeJobs.splice( activeJobs.findIndex( job => job.index === result.index ), 1 ); // remove completed job
|
|
40
|
+
|
|
41
|
+
if ( jobsPool.length > 0 ) {
|
|
42
|
+
const nextJob = jobsPool.shift();
|
|
43
|
+
nextJob.promise = nextJob.fn();
|
|
44
|
+
activeJobs.push( nextJob );
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return results.sort( ( a, b ) => a.index - b.index );
|
|
49
|
+
};
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { executeInParallel } from './workflow_utils.js';
|
|
3
|
+
|
|
4
|
+
describe( 'executeInParallel', () => {
|
|
5
|
+
it( 'returns empty array for empty jobs', async () => {
|
|
6
|
+
const results = await executeInParallel( { jobs: [] } );
|
|
7
|
+
expect( results ).toEqual( [] );
|
|
8
|
+
} );
|
|
9
|
+
|
|
10
|
+
it( 'executes all jobs and returns results', async () => {
|
|
11
|
+
const jobs = [
|
|
12
|
+
() => Promise.resolve( 'a' ),
|
|
13
|
+
() => Promise.resolve( 'b' ),
|
|
14
|
+
() => Promise.resolve( 'c' )
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
const results = await executeInParallel( { jobs } );
|
|
18
|
+
|
|
19
|
+
expect( results ).toHaveLength( 3 );
|
|
20
|
+
expect( results ).toContainEqual( { ok: true, result: 'a', index: 0 } );
|
|
21
|
+
expect( results ).toContainEqual( { ok: true, result: 'b', index: 1 } );
|
|
22
|
+
expect( results ).toContainEqual( { ok: true, result: 'c', index: 2 } );
|
|
23
|
+
} );
|
|
24
|
+
|
|
25
|
+
it( 'handles job failures without throwing', async () => {
|
|
26
|
+
const error = new Error( 'job failed' );
|
|
27
|
+
const jobs = [
|
|
28
|
+
() => Promise.resolve( 'ok' ),
|
|
29
|
+
() => Promise.reject( error )
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
const results = await executeInParallel( { jobs } );
|
|
33
|
+
|
|
34
|
+
expect( results ).toHaveLength( 2 );
|
|
35
|
+
expect( results ).toContainEqual( { ok: true, result: 'ok', index: 0 } );
|
|
36
|
+
expect( results ).toContainEqual( { ok: false, error, index: 1 } );
|
|
37
|
+
} );
|
|
38
|
+
|
|
39
|
+
it( 'handles all jobs failing', async () => {
|
|
40
|
+
const errors = [ new Error( 'e1' ), new Error( 'e2' ) ];
|
|
41
|
+
const jobs = [
|
|
42
|
+
() => Promise.reject( errors[0] ),
|
|
43
|
+
() => Promise.reject( errors[1] )
|
|
44
|
+
];
|
|
45
|
+
|
|
46
|
+
const results = await executeInParallel( { jobs } );
|
|
47
|
+
|
|
48
|
+
expect( results ).toHaveLength( 2 );
|
|
49
|
+
expect( results ).toContainEqual( { ok: false, error: errors[0], index: 0 } );
|
|
50
|
+
expect( results ).toContainEqual( { ok: false, error: errors[1], index: 1 } );
|
|
51
|
+
} );
|
|
52
|
+
|
|
53
|
+
it( 'respects concurrency limit', async () => {
|
|
54
|
+
const tracker = { active: 0, max: 0 };
|
|
55
|
+
|
|
56
|
+
const createJob = ( id, delay ) => async () => {
|
|
57
|
+
tracker.active++;
|
|
58
|
+
tracker.max = Math.max( tracker.max, tracker.active );
|
|
59
|
+
await new Promise( r => setTimeout( r, delay ) );
|
|
60
|
+
tracker.active--;
|
|
61
|
+
return id;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const jobs = [
|
|
65
|
+
createJob( 'a', 50 ),
|
|
66
|
+
createJob( 'b', 50 ),
|
|
67
|
+
createJob( 'c', 50 ),
|
|
68
|
+
createJob( 'd', 50 )
|
|
69
|
+
];
|
|
70
|
+
|
|
71
|
+
await executeInParallel( { jobs, concurrency: 2 } );
|
|
72
|
+
|
|
73
|
+
expect( tracker.max ).toBe( 2 );
|
|
74
|
+
} );
|
|
75
|
+
|
|
76
|
+
it( 'runs all jobs concurrently when concurrency is Infinity', async () => {
|
|
77
|
+
const tracker = { active: 0, max: 0 };
|
|
78
|
+
|
|
79
|
+
const createJob = delay => async () => {
|
|
80
|
+
tracker.active++;
|
|
81
|
+
tracker.max = Math.max( tracker.max, tracker.active );
|
|
82
|
+
await new Promise( r => setTimeout( r, delay ) );
|
|
83
|
+
tracker.active--;
|
|
84
|
+
return 'done';
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const jobs = [ createJob( 30 ), createJob( 30 ), createJob( 30 ), createJob( 30 ) ];
|
|
88
|
+
|
|
89
|
+
await executeInParallel( { jobs } );
|
|
90
|
+
|
|
91
|
+
expect( tracker.max ).toBe( 4 );
|
|
92
|
+
} );
|
|
93
|
+
|
|
94
|
+
it( 'calls onJobCompleted for each job', async () => {
|
|
95
|
+
const onJobCompleted = vi.fn();
|
|
96
|
+
const jobs = [
|
|
97
|
+
() => Promise.resolve( 'a' ),
|
|
98
|
+
() => Promise.resolve( 'b' )
|
|
99
|
+
];
|
|
100
|
+
|
|
101
|
+
await executeInParallel( { jobs, onJobCompleted } );
|
|
102
|
+
|
|
103
|
+
expect( onJobCompleted ).toHaveBeenCalledTimes( 2 );
|
|
104
|
+
expect( onJobCompleted ).toHaveBeenCalledWith( expect.objectContaining( { ok: true, result: 'a', index: 0 } ) );
|
|
105
|
+
expect( onJobCompleted ).toHaveBeenCalledWith( expect.objectContaining( { ok: true, result: 'b', index: 1 } ) );
|
|
106
|
+
} );
|
|
107
|
+
|
|
108
|
+
it( 'works without onJobCompleted callback', async () => {
|
|
109
|
+
const jobs = [ () => Promise.resolve( 'x' ) ];
|
|
110
|
+
|
|
111
|
+
const results = await executeInParallel( { jobs } );
|
|
112
|
+
|
|
113
|
+
expect( results ).toEqual( [ { ok: true, result: 'x', index: 0 } ] );
|
|
114
|
+
} );
|
|
115
|
+
|
|
116
|
+
it( 'handles synchronous job functions', async () => {
|
|
117
|
+
const jobs = [
|
|
118
|
+
() => 'sync-a',
|
|
119
|
+
() => 'sync-b'
|
|
120
|
+
];
|
|
121
|
+
|
|
122
|
+
const results = await executeInParallel( { jobs } );
|
|
123
|
+
|
|
124
|
+
expect( results ).toContainEqual( { ok: true, result: 'sync-a', index: 0 } );
|
|
125
|
+
expect( results ).toContainEqual( { ok: true, result: 'sync-b', index: 1 } );
|
|
126
|
+
} );
|
|
127
|
+
|
|
128
|
+
it( 'handles synchronous throwing job functions', async () => {
|
|
129
|
+
const error = new Error( 'sync throw' );
|
|
130
|
+
const jobs = [
|
|
131
|
+
() => {
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
];
|
|
135
|
+
|
|
136
|
+
const results = await executeInParallel( { jobs } );
|
|
137
|
+
|
|
138
|
+
expect( results ).toEqual( [ { ok: false, error, index: 0 } ] );
|
|
139
|
+
} );
|
|
140
|
+
|
|
141
|
+
it( 'handles concurrency greater than job count', async () => {
|
|
142
|
+
const jobs = [ () => 'only-one' ];
|
|
143
|
+
|
|
144
|
+
const results = await executeInParallel( { jobs, concurrency: 10 } );
|
|
145
|
+
|
|
146
|
+
expect( results ).toEqual( [ { ok: true, result: 'only-one', index: 0 } ] );
|
|
147
|
+
} );
|
|
148
|
+
|
|
149
|
+
it( 'calls onJobCompleted in completion order, not submission order', async () => {
|
|
150
|
+
const completionOrder = [];
|
|
151
|
+
const onJobCompleted = result => completionOrder.push( result.index );
|
|
152
|
+
|
|
153
|
+
const jobs = [
|
|
154
|
+
async () => {
|
|
155
|
+
await new Promise( r => setTimeout( r, 60 ) );
|
|
156
|
+
return 'slow';
|
|
157
|
+
},
|
|
158
|
+
async () => {
|
|
159
|
+
await new Promise( r => setTimeout( r, 10 ) );
|
|
160
|
+
return 'fast';
|
|
161
|
+
}
|
|
162
|
+
];
|
|
163
|
+
|
|
164
|
+
await executeInParallel( { jobs, onJobCompleted } );
|
|
165
|
+
|
|
166
|
+
expect( completionOrder ).toEqual( [ 1, 0 ] ); // fast (index 1) completes before slow (index 0)
|
|
167
|
+
} );
|
|
168
|
+
|
|
169
|
+
it( 'returns results sorted by job index for determinism', async () => {
|
|
170
|
+
const jobs = [
|
|
171
|
+
async () => {
|
|
172
|
+
await new Promise( r => setTimeout( r, 60 ) );
|
|
173
|
+
return 'slow';
|
|
174
|
+
},
|
|
175
|
+
async () => {
|
|
176
|
+
await new Promise( r => setTimeout( r, 10 ) );
|
|
177
|
+
return 'fast';
|
|
178
|
+
}
|
|
179
|
+
];
|
|
180
|
+
|
|
181
|
+
const results = await executeInParallel( { jobs } );
|
|
182
|
+
|
|
183
|
+
expect( results[0].index ).toBe( 0 );
|
|
184
|
+
expect( results[1].index ).toBe( 1 );
|
|
185
|
+
expect( results ).toEqual( [
|
|
186
|
+
{ ok: true, result: 'slow', index: 0 },
|
|
187
|
+
{ ok: true, result: 'fast', index: 1 }
|
|
188
|
+
] );
|
|
189
|
+
} );
|
|
190
|
+
} );
|