@juspay/neurolink 9.23.0 → 9.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +10 -13
- package/dist/adapters/tts/googleTTSHandler.js +26 -1
- package/dist/adapters/video/vertexVideoHandler.js +23 -17
- package/dist/cli/commands/config.d.ts +3 -3
- package/dist/cli/commands/observability.d.ts +53 -0
- package/dist/cli/commands/observability.js +453 -0
- package/dist/cli/commands/telemetry.d.ts +63 -0
- package/dist/cli/commands/telemetry.js +689 -0
- package/dist/cli/factories/commandFactory.d.ts +34 -0
- package/dist/cli/factories/commandFactory.js +321 -116
- package/dist/cli/parser.js +6 -9
- package/dist/cli/utils/formatters.d.ts +13 -0
- package/dist/cli/utils/formatters.js +23 -0
- package/dist/constants/contextWindows.js +6 -0
- package/dist/constants/enums.d.ts +6 -0
- package/dist/constants/enums.js +8 -2
- package/dist/context/budgetChecker.js +75 -48
- package/dist/context/contextCompactor.js +135 -127
- package/dist/core/baseProvider.d.ts +5 -0
- package/dist/core/baseProvider.js +158 -102
- package/dist/core/conversationMemoryInitializer.js +7 -4
- package/dist/core/conversationMemoryManager.d.ts +2 -0
- package/dist/core/conversationMemoryManager.js +6 -2
- package/dist/core/modules/GenerationHandler.d.ts +2 -2
- package/dist/core/modules/GenerationHandler.js +12 -12
- package/dist/evaluation/ragasEvaluator.js +39 -19
- package/dist/evaluation/scoring.js +46 -20
- package/dist/features/ppt/index.d.ts +1 -1
- package/dist/features/ppt/index.js +1 -1
- package/dist/features/ppt/presentationOrchestrator.js +23 -0
- package/dist/features/ppt/slideGenerator.js +13 -0
- package/dist/features/ppt/slideRenderers.d.ts +1 -1
- package/dist/features/ppt/slideRenderers.js +6 -4
- package/dist/features/ppt/slideTypeInference.d.ts +1 -1
- package/dist/features/ppt/slideTypeInference.js +75 -73
- package/dist/files/fileTools.d.ts +6 -6
- package/dist/index.d.ts +46 -12
- package/dist/index.js +79 -17
- package/dist/lib/adapters/tts/googleTTSHandler.js +26 -1
- package/dist/lib/adapters/video/vertexVideoHandler.js +23 -17
- package/dist/lib/constants/contextWindows.js +6 -0
- package/dist/lib/constants/enums.d.ts +6 -0
- package/dist/lib/constants/enums.js +8 -2
- package/dist/lib/context/budgetChecker.js +75 -48
- package/dist/lib/context/contextCompactor.js +135 -127
- package/dist/lib/core/baseProvider.d.ts +5 -0
- package/dist/lib/core/baseProvider.js +158 -102
- package/dist/lib/core/conversationMemoryInitializer.js +7 -4
- package/dist/lib/core/conversationMemoryManager.d.ts +2 -0
- package/dist/lib/core/conversationMemoryManager.js +6 -2
- package/dist/lib/core/modules/GenerationHandler.d.ts +2 -2
- package/dist/lib/core/modules/GenerationHandler.js +12 -12
- package/dist/lib/evaluation/ragasEvaluator.js +39 -19
- package/dist/lib/evaluation/scoring.js +46 -20
- package/dist/lib/features/ppt/index.d.ts +1 -1
- package/dist/lib/features/ppt/index.js +1 -1
- package/dist/lib/features/ppt/presentationOrchestrator.js +23 -0
- package/dist/lib/features/ppt/slideGenerator.js +13 -0
- package/dist/lib/features/ppt/slideRenderers.d.ts +1 -1
- package/dist/lib/features/ppt/slideRenderers.js +6 -4
- package/dist/lib/features/ppt/slideTypeInference.d.ts +1 -1
- package/dist/lib/features/ppt/slideTypeInference.js +75 -73
- package/dist/lib/files/fileTools.d.ts +6 -6
- package/dist/lib/index.d.ts +46 -12
- package/dist/lib/index.js +79 -17
- package/dist/lib/mcp/httpRateLimiter.js +39 -12
- package/dist/lib/mcp/httpRetryHandler.js +22 -1
- package/dist/lib/mcp/mcpClientFactory.js +13 -15
- package/dist/lib/memory/memoryRetrievalTools.js +22 -0
- package/dist/lib/neurolink.d.ts +64 -72
- package/dist/lib/neurolink.js +984 -566
- package/dist/lib/observability/exporterRegistry.d.ts +152 -0
- package/dist/lib/observability/exporterRegistry.js +414 -0
- package/dist/lib/observability/exporters/arizeExporter.d.ts +32 -0
- package/dist/lib/observability/exporters/arizeExporter.js +139 -0
- package/dist/lib/observability/exporters/baseExporter.d.ts +117 -0
- package/dist/lib/observability/exporters/baseExporter.js +191 -0
- package/dist/lib/observability/exporters/braintrustExporter.d.ts +30 -0
- package/dist/lib/observability/exporters/braintrustExporter.js +155 -0
- package/dist/lib/observability/exporters/datadogExporter.d.ts +37 -0
- package/dist/lib/observability/exporters/datadogExporter.js +197 -0
- package/dist/lib/observability/exporters/index.d.ts +13 -0
- package/dist/lib/observability/exporters/index.js +14 -0
- package/dist/lib/observability/exporters/laminarExporter.d.ts +48 -0
- package/dist/lib/observability/exporters/laminarExporter.js +303 -0
- package/dist/lib/observability/exporters/langfuseExporter.d.ts +47 -0
- package/dist/lib/observability/exporters/langfuseExporter.js +200 -0
- package/dist/lib/observability/exporters/langsmithExporter.d.ts +26 -0
- package/dist/lib/observability/exporters/langsmithExporter.js +124 -0
- package/dist/lib/observability/exporters/otelExporter.d.ts +39 -0
- package/dist/lib/observability/exporters/otelExporter.js +165 -0
- package/dist/lib/observability/exporters/posthogExporter.d.ts +48 -0
- package/dist/lib/observability/exporters/posthogExporter.js +288 -0
- package/dist/lib/observability/exporters/sentryExporter.d.ts +32 -0
- package/dist/lib/observability/exporters/sentryExporter.js +166 -0
- package/dist/lib/observability/index.d.ts +25 -0
- package/dist/lib/observability/index.js +32 -0
- package/dist/lib/observability/metricsAggregator.d.ts +260 -0
- package/dist/lib/observability/metricsAggregator.js +553 -0
- package/dist/lib/observability/otelBridge.d.ts +49 -0
- package/dist/lib/observability/otelBridge.js +132 -0
- package/dist/lib/observability/retryPolicy.d.ts +192 -0
- package/dist/lib/observability/retryPolicy.js +384 -0
- package/dist/lib/observability/sampling/index.d.ts +4 -0
- package/dist/lib/observability/sampling/index.js +5 -0
- package/dist/lib/observability/sampling/samplers.d.ts +116 -0
- package/dist/lib/observability/sampling/samplers.js +217 -0
- package/dist/lib/observability/spanProcessor.d.ts +129 -0
- package/dist/lib/observability/spanProcessor.js +288 -0
- package/dist/lib/observability/tokenTracker.d.ts +156 -0
- package/dist/lib/observability/tokenTracker.js +414 -0
- package/dist/lib/observability/types/exporterTypes.d.ts +250 -0
- package/dist/lib/observability/types/exporterTypes.js +6 -0
- package/dist/lib/observability/types/index.d.ts +6 -0
- package/dist/lib/observability/types/index.js +5 -0
- package/dist/lib/observability/types/spanTypes.d.ts +244 -0
- package/dist/lib/observability/types/spanTypes.js +93 -0
- package/dist/lib/observability/utils/index.d.ts +4 -0
- package/dist/lib/observability/utils/index.js +5 -0
- package/dist/lib/observability/utils/spanSerializer.d.ts +115 -0
- package/dist/lib/observability/utils/spanSerializer.js +287 -0
- package/dist/lib/providers/amazonSagemaker.d.ts +5 -4
- package/dist/lib/providers/amazonSagemaker.js +3 -4
- package/dist/lib/providers/googleVertex.d.ts +7 -0
- package/dist/lib/providers/googleVertex.js +80 -2
- package/dist/lib/rag/pipeline/RAGPipeline.d.ts +0 -5
- package/dist/lib/rag/pipeline/RAGPipeline.js +122 -87
- package/dist/lib/rag/ragIntegration.js +30 -0
- package/dist/lib/rag/retrieval/hybridSearch.js +22 -0
- package/dist/lib/server/abstract/baseServerAdapter.js +51 -19
- package/dist/lib/server/middleware/common.js +44 -12
- package/dist/lib/services/server/ai/observability/instrumentation.d.ts +2 -2
- package/dist/lib/services/server/ai/observability/instrumentation.js +10 -5
- package/dist/lib/types/cli.d.ts +18 -2
- package/dist/lib/types/conversationMemoryInterface.d.ts +2 -0
- package/dist/lib/types/generateTypes.d.ts +2 -2
- package/dist/lib/types/modelTypes.d.ts +18 -18
- package/dist/lib/types/providers.d.ts +5 -0
- package/dist/lib/utils/pricing.js +25 -1
- package/dist/lib/utils/ttsProcessor.js +74 -59
- package/dist/lib/workflow/config.d.ts +36 -36
- package/dist/lib/workflow/core/ensembleExecutor.js +10 -0
- package/dist/lib/workflow/core/judgeScorer.js +20 -2
- package/dist/lib/workflow/core/workflowRunner.js +34 -1
- package/dist/mcp/httpRateLimiter.js +39 -12
- package/dist/mcp/httpRetryHandler.js +22 -1
- package/dist/mcp/mcpClientFactory.js +13 -15
- package/dist/memory/memoryRetrievalTools.js +22 -0
- package/dist/neurolink.d.ts +64 -72
- package/dist/neurolink.js +984 -566
- package/dist/observability/FEATURE-STATUS.md +269 -0
- package/dist/observability/exporterRegistry.d.ts +152 -0
- package/dist/observability/exporterRegistry.js +413 -0
- package/dist/observability/exporters/arizeExporter.d.ts +32 -0
- package/dist/observability/exporters/arizeExporter.js +138 -0
- package/dist/observability/exporters/baseExporter.d.ts +117 -0
- package/dist/observability/exporters/baseExporter.js +190 -0
- package/dist/observability/exporters/braintrustExporter.d.ts +30 -0
- package/dist/observability/exporters/braintrustExporter.js +154 -0
- package/dist/observability/exporters/datadogExporter.d.ts +37 -0
- package/dist/observability/exporters/datadogExporter.js +196 -0
- package/dist/observability/exporters/index.d.ts +13 -0
- package/dist/observability/exporters/index.js +13 -0
- package/dist/observability/exporters/laminarExporter.d.ts +48 -0
- package/dist/observability/exporters/laminarExporter.js +302 -0
- package/dist/observability/exporters/langfuseExporter.d.ts +47 -0
- package/dist/observability/exporters/langfuseExporter.js +199 -0
- package/dist/observability/exporters/langsmithExporter.d.ts +26 -0
- package/dist/observability/exporters/langsmithExporter.js +123 -0
- package/dist/observability/exporters/otelExporter.d.ts +39 -0
- package/dist/observability/exporters/otelExporter.js +164 -0
- package/dist/observability/exporters/posthogExporter.d.ts +48 -0
- package/dist/observability/exporters/posthogExporter.js +287 -0
- package/dist/observability/exporters/sentryExporter.d.ts +32 -0
- package/dist/observability/exporters/sentryExporter.js +165 -0
- package/dist/observability/index.d.ts +25 -0
- package/dist/observability/index.js +31 -0
- package/dist/observability/metricsAggregator.d.ts +260 -0
- package/dist/observability/metricsAggregator.js +552 -0
- package/dist/observability/otelBridge.d.ts +49 -0
- package/dist/observability/otelBridge.js +131 -0
- package/dist/observability/retryPolicy.d.ts +192 -0
- package/dist/observability/retryPolicy.js +383 -0
- package/dist/observability/sampling/index.d.ts +4 -0
- package/dist/observability/sampling/index.js +4 -0
- package/dist/observability/sampling/samplers.d.ts +116 -0
- package/dist/observability/sampling/samplers.js +216 -0
- package/dist/observability/spanProcessor.d.ts +129 -0
- package/dist/observability/spanProcessor.js +287 -0
- package/dist/observability/tokenTracker.d.ts +156 -0
- package/dist/observability/tokenTracker.js +413 -0
- package/dist/observability/types/exporterTypes.d.ts +250 -0
- package/dist/observability/types/exporterTypes.js +5 -0
- package/dist/observability/types/index.d.ts +6 -0
- package/dist/observability/types/index.js +4 -0
- package/dist/observability/types/spanTypes.d.ts +244 -0
- package/dist/observability/types/spanTypes.js +92 -0
- package/dist/observability/utils/index.d.ts +4 -0
- package/dist/observability/utils/index.js +4 -0
- package/dist/observability/utils/spanSerializer.d.ts +115 -0
- package/dist/observability/utils/spanSerializer.js +286 -0
- package/dist/providers/amazonSagemaker.d.ts +5 -4
- package/dist/providers/amazonSagemaker.js +3 -4
- package/dist/providers/googleVertex.d.ts +7 -0
- package/dist/providers/googleVertex.js +80 -2
- package/dist/rag/pipeline/RAGPipeline.d.ts +0 -5
- package/dist/rag/pipeline/RAGPipeline.js +122 -87
- package/dist/rag/ragIntegration.js +30 -0
- package/dist/rag/retrieval/hybridSearch.js +22 -0
- package/dist/server/abstract/baseServerAdapter.js +51 -19
- package/dist/server/middleware/common.js +44 -12
- package/dist/services/server/ai/observability/instrumentation.d.ts +2 -2
- package/dist/services/server/ai/observability/instrumentation.js +10 -5
- package/dist/types/cli.d.ts +18 -2
- package/dist/types/conversationMemoryInterface.d.ts +2 -0
- package/dist/types/generateTypes.d.ts +2 -2
- package/dist/types/providers.d.ts +5 -0
- package/dist/utils/pricing.js +25 -1
- package/dist/utils/ttsProcessor.js +74 -59
- package/dist/workflow/config.d.ts +52 -52
- package/dist/workflow/core/ensembleExecutor.js +10 -0
- package/dist/workflow/core/judgeScorer.js +20 -2
- package/dist/workflow/core/workflowRunner.js +34 -1
- package/package.json +1 -1
|
@@ -39,11 +39,11 @@ export declare const ModelConfigSchema: z.ZodObject<{
|
|
|
39
39
|
maxTokens?: number | undefined;
|
|
40
40
|
metadata?: Record<string, JsonValue> | undefined;
|
|
41
41
|
systemPrompt?: string | undefined;
|
|
42
|
+
weight?: number | undefined;
|
|
42
43
|
topK?: number | undefined;
|
|
43
44
|
topP?: number | undefined;
|
|
44
45
|
presencePenalty?: number | undefined;
|
|
45
46
|
frequencyPenalty?: number | undefined;
|
|
46
|
-
weight?: number | undefined;
|
|
47
47
|
label?: string | undefined;
|
|
48
48
|
}, {
|
|
49
49
|
provider: string;
|
|
@@ -53,11 +53,11 @@ export declare const ModelConfigSchema: z.ZodObject<{
|
|
|
53
53
|
maxTokens?: number | undefined;
|
|
54
54
|
metadata?: Record<string, JsonValue> | undefined;
|
|
55
55
|
systemPrompt?: string | undefined;
|
|
56
|
+
weight?: number | undefined;
|
|
56
57
|
topK?: number | undefined;
|
|
57
58
|
topP?: number | undefined;
|
|
58
59
|
presencePenalty?: number | undefined;
|
|
59
60
|
frequencyPenalty?: number | undefined;
|
|
60
|
-
weight?: number | undefined;
|
|
61
61
|
label?: string | undefined;
|
|
62
62
|
}>;
|
|
63
63
|
/**
|
|
@@ -208,11 +208,11 @@ export declare const ModelGroupSchema: z.ZodObject<{
|
|
|
208
208
|
maxTokens?: number | undefined;
|
|
209
209
|
metadata?: Record<string, JsonValue> | undefined;
|
|
210
210
|
systemPrompt?: string | undefined;
|
|
211
|
+
weight?: number | undefined;
|
|
211
212
|
topK?: number | undefined;
|
|
212
213
|
topP?: number | undefined;
|
|
213
214
|
presencePenalty?: number | undefined;
|
|
214
215
|
frequencyPenalty?: number | undefined;
|
|
215
|
-
weight?: number | undefined;
|
|
216
216
|
label?: string | undefined;
|
|
217
217
|
}, {
|
|
218
218
|
provider: string;
|
|
@@ -222,11 +222,11 @@ export declare const ModelGroupSchema: z.ZodObject<{
|
|
|
222
222
|
maxTokens?: number | undefined;
|
|
223
223
|
metadata?: Record<string, JsonValue> | undefined;
|
|
224
224
|
systemPrompt?: string | undefined;
|
|
225
|
+
weight?: number | undefined;
|
|
225
226
|
topK?: number | undefined;
|
|
226
227
|
topP?: number | undefined;
|
|
227
228
|
presencePenalty?: number | undefined;
|
|
228
229
|
frequencyPenalty?: number | undefined;
|
|
229
|
-
weight?: number | undefined;
|
|
230
230
|
label?: string | undefined;
|
|
231
231
|
}>, "many">;
|
|
232
232
|
executionStrategy: z.ZodEnum<["parallel", "sequential"]>;
|
|
@@ -244,11 +244,11 @@ export declare const ModelGroupSchema: z.ZodObject<{
|
|
|
244
244
|
maxTokens?: number | undefined;
|
|
245
245
|
metadata?: Record<string, JsonValue> | undefined;
|
|
246
246
|
systemPrompt?: string | undefined;
|
|
247
|
+
weight?: number | undefined;
|
|
247
248
|
topK?: number | undefined;
|
|
248
249
|
topP?: number | undefined;
|
|
249
250
|
presencePenalty?: number | undefined;
|
|
250
251
|
frequencyPenalty?: number | undefined;
|
|
251
|
-
weight?: number | undefined;
|
|
252
252
|
label?: string | undefined;
|
|
253
253
|
}[];
|
|
254
254
|
id: string;
|
|
@@ -269,11 +269,11 @@ export declare const ModelGroupSchema: z.ZodObject<{
|
|
|
269
269
|
maxTokens?: number | undefined;
|
|
270
270
|
metadata?: Record<string, JsonValue> | undefined;
|
|
271
271
|
systemPrompt?: string | undefined;
|
|
272
|
+
weight?: number | undefined;
|
|
272
273
|
topK?: number | undefined;
|
|
273
274
|
topP?: number | undefined;
|
|
274
275
|
presencePenalty?: number | undefined;
|
|
275
276
|
frequencyPenalty?: number | undefined;
|
|
276
|
-
weight?: number | undefined;
|
|
277
277
|
label?: string | undefined;
|
|
278
278
|
}[];
|
|
279
279
|
id: string;
|
|
@@ -363,11 +363,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
363
363
|
maxTokens?: number | undefined;
|
|
364
364
|
metadata?: Record<string, JsonValue> | undefined;
|
|
365
365
|
systemPrompt?: string | undefined;
|
|
366
|
+
weight?: number | undefined;
|
|
366
367
|
topK?: number | undefined;
|
|
367
368
|
topP?: number | undefined;
|
|
368
369
|
presencePenalty?: number | undefined;
|
|
369
370
|
frequencyPenalty?: number | undefined;
|
|
370
|
-
weight?: number | undefined;
|
|
371
371
|
label?: string | undefined;
|
|
372
372
|
}, {
|
|
373
373
|
provider: string;
|
|
@@ -377,11 +377,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
377
377
|
maxTokens?: number | undefined;
|
|
378
378
|
metadata?: Record<string, JsonValue> | undefined;
|
|
379
379
|
systemPrompt?: string | undefined;
|
|
380
|
+
weight?: number | undefined;
|
|
380
381
|
topK?: number | undefined;
|
|
381
382
|
topP?: number | undefined;
|
|
382
383
|
presencePenalty?: number | undefined;
|
|
383
384
|
frequencyPenalty?: number | undefined;
|
|
384
|
-
weight?: number | undefined;
|
|
385
385
|
label?: string | undefined;
|
|
386
386
|
}>, "many">;
|
|
387
387
|
modelGroups: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
@@ -410,11 +410,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
410
410
|
maxTokens?: number | undefined;
|
|
411
411
|
metadata?: Record<string, JsonValue> | undefined;
|
|
412
412
|
systemPrompt?: string | undefined;
|
|
413
|
+
weight?: number | undefined;
|
|
413
414
|
topK?: number | undefined;
|
|
414
415
|
topP?: number | undefined;
|
|
415
416
|
presencePenalty?: number | undefined;
|
|
416
417
|
frequencyPenalty?: number | undefined;
|
|
417
|
-
weight?: number | undefined;
|
|
418
418
|
label?: string | undefined;
|
|
419
419
|
}, {
|
|
420
420
|
provider: string;
|
|
@@ -424,11 +424,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
424
424
|
maxTokens?: number | undefined;
|
|
425
425
|
metadata?: Record<string, JsonValue> | undefined;
|
|
426
426
|
systemPrompt?: string | undefined;
|
|
427
|
+
weight?: number | undefined;
|
|
427
428
|
topK?: number | undefined;
|
|
428
429
|
topP?: number | undefined;
|
|
429
430
|
presencePenalty?: number | undefined;
|
|
430
431
|
frequencyPenalty?: number | undefined;
|
|
431
|
-
weight?: number | undefined;
|
|
432
432
|
label?: string | undefined;
|
|
433
433
|
}>, "many">;
|
|
434
434
|
executionStrategy: z.ZodEnum<["parallel", "sequential"]>;
|
|
@@ -446,11 +446,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
446
446
|
maxTokens?: number | undefined;
|
|
447
447
|
metadata?: Record<string, JsonValue> | undefined;
|
|
448
448
|
systemPrompt?: string | undefined;
|
|
449
|
+
weight?: number | undefined;
|
|
449
450
|
topK?: number | undefined;
|
|
450
451
|
topP?: number | undefined;
|
|
451
452
|
presencePenalty?: number | undefined;
|
|
452
453
|
frequencyPenalty?: number | undefined;
|
|
453
|
-
weight?: number | undefined;
|
|
454
454
|
label?: string | undefined;
|
|
455
455
|
}[];
|
|
456
456
|
id: string;
|
|
@@ -471,11 +471,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
471
471
|
maxTokens?: number | undefined;
|
|
472
472
|
metadata?: Record<string, JsonValue> | undefined;
|
|
473
473
|
systemPrompt?: string | undefined;
|
|
474
|
+
weight?: number | undefined;
|
|
474
475
|
topK?: number | undefined;
|
|
475
476
|
topP?: number | undefined;
|
|
476
477
|
presencePenalty?: number | undefined;
|
|
477
478
|
frequencyPenalty?: number | undefined;
|
|
478
|
-
weight?: number | undefined;
|
|
479
479
|
label?: string | undefined;
|
|
480
480
|
}[];
|
|
481
481
|
id: string;
|
|
@@ -714,7 +714,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
714
714
|
createdAt: z.ZodOptional<z.ZodString>;
|
|
715
715
|
updatedAt: z.ZodOptional<z.ZodString>;
|
|
716
716
|
}, "strip", z.ZodTypeAny, {
|
|
717
|
-
type: "custom" | "
|
|
717
|
+
type: "custom" | "chain" | "ensemble" | "adaptive";
|
|
718
718
|
name: string;
|
|
719
719
|
models: {
|
|
720
720
|
provider: AIProviderName;
|
|
@@ -724,11 +724,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
724
724
|
maxTokens?: number | undefined;
|
|
725
725
|
metadata?: Record<string, JsonValue> | undefined;
|
|
726
726
|
systemPrompt?: string | undefined;
|
|
727
|
+
weight?: number | undefined;
|
|
727
728
|
topK?: number | undefined;
|
|
728
729
|
topP?: number | undefined;
|
|
729
730
|
presencePenalty?: number | undefined;
|
|
730
731
|
frequencyPenalty?: number | undefined;
|
|
731
|
-
weight?: number | undefined;
|
|
732
732
|
label?: string | undefined;
|
|
733
733
|
}[];
|
|
734
734
|
id: string;
|
|
@@ -751,6 +751,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
751
751
|
version?: string | undefined;
|
|
752
752
|
tags?: string[] | undefined;
|
|
753
753
|
metadata?: Record<string, JsonValue> | undefined;
|
|
754
|
+
description?: string | undefined;
|
|
754
755
|
judge?: {
|
|
755
756
|
provider: AIProviderName;
|
|
756
757
|
model: string;
|
|
@@ -785,7 +786,6 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
785
786
|
addModelAttribution?: boolean | undefined;
|
|
786
787
|
addExecutionTime?: boolean | undefined;
|
|
787
788
|
} | undefined;
|
|
788
|
-
description?: string | undefined;
|
|
789
789
|
createdAt?: string | undefined;
|
|
790
790
|
updatedAt?: string | undefined;
|
|
791
791
|
modelGroups?: {
|
|
@@ -797,11 +797,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
797
797
|
maxTokens?: number | undefined;
|
|
798
798
|
metadata?: Record<string, JsonValue> | undefined;
|
|
799
799
|
systemPrompt?: string | undefined;
|
|
800
|
+
weight?: number | undefined;
|
|
800
801
|
topK?: number | undefined;
|
|
801
802
|
topP?: number | undefined;
|
|
802
803
|
presencePenalty?: number | undefined;
|
|
803
804
|
frequencyPenalty?: number | undefined;
|
|
804
|
-
weight?: number | undefined;
|
|
805
805
|
label?: string | undefined;
|
|
806
806
|
}[];
|
|
807
807
|
id: string;
|
|
@@ -836,7 +836,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
836
836
|
blindEvaluation?: boolean | undefined;
|
|
837
837
|
}[] | undefined;
|
|
838
838
|
}, {
|
|
839
|
-
type: "custom" | "
|
|
839
|
+
type: "custom" | "chain" | "ensemble" | "adaptive";
|
|
840
840
|
name: string;
|
|
841
841
|
models: {
|
|
842
842
|
provider: string;
|
|
@@ -846,11 +846,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
846
846
|
maxTokens?: number | undefined;
|
|
847
847
|
metadata?: Record<string, JsonValue> | undefined;
|
|
848
848
|
systemPrompt?: string | undefined;
|
|
849
|
+
weight?: number | undefined;
|
|
849
850
|
topK?: number | undefined;
|
|
850
851
|
topP?: number | undefined;
|
|
851
852
|
presencePenalty?: number | undefined;
|
|
852
853
|
frequencyPenalty?: number | undefined;
|
|
853
|
-
weight?: number | undefined;
|
|
854
854
|
label?: string | undefined;
|
|
855
855
|
}[];
|
|
856
856
|
id: string;
|
|
@@ -873,6 +873,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
873
873
|
version?: string | undefined;
|
|
874
874
|
tags?: string[] | undefined;
|
|
875
875
|
metadata?: Record<string, JsonValue> | undefined;
|
|
876
|
+
description?: string | undefined;
|
|
876
877
|
judge?: {
|
|
877
878
|
provider: string;
|
|
878
879
|
model: string;
|
|
@@ -907,7 +908,6 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
907
908
|
addModelAttribution?: boolean | undefined;
|
|
908
909
|
addExecutionTime?: boolean | undefined;
|
|
909
910
|
} | undefined;
|
|
910
|
-
description?: string | undefined;
|
|
911
911
|
createdAt?: string | undefined;
|
|
912
912
|
updatedAt?: string | undefined;
|
|
913
913
|
modelGroups?: {
|
|
@@ -919,11 +919,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
919
919
|
maxTokens?: number | undefined;
|
|
920
920
|
metadata?: Record<string, JsonValue> | undefined;
|
|
921
921
|
systemPrompt?: string | undefined;
|
|
922
|
+
weight?: number | undefined;
|
|
922
923
|
topK?: number | undefined;
|
|
923
924
|
topP?: number | undefined;
|
|
924
925
|
presencePenalty?: number | undefined;
|
|
925
926
|
frequencyPenalty?: number | undefined;
|
|
926
|
-
weight?: number | undefined;
|
|
927
927
|
label?: string | undefined;
|
|
928
928
|
}[];
|
|
929
929
|
id: string;
|
|
@@ -958,7 +958,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
958
958
|
blindEvaluation?: boolean | undefined;
|
|
959
959
|
}[] | undefined;
|
|
960
960
|
}>, {
|
|
961
|
-
type: "custom" | "
|
|
961
|
+
type: "custom" | "chain" | "ensemble" | "adaptive";
|
|
962
962
|
name: string;
|
|
963
963
|
models: {
|
|
964
964
|
provider: AIProviderName;
|
|
@@ -968,11 +968,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
968
968
|
maxTokens?: number | undefined;
|
|
969
969
|
metadata?: Record<string, JsonValue> | undefined;
|
|
970
970
|
systemPrompt?: string | undefined;
|
|
971
|
+
weight?: number | undefined;
|
|
971
972
|
topK?: number | undefined;
|
|
972
973
|
topP?: number | undefined;
|
|
973
974
|
presencePenalty?: number | undefined;
|
|
974
975
|
frequencyPenalty?: number | undefined;
|
|
975
|
-
weight?: number | undefined;
|
|
976
976
|
label?: string | undefined;
|
|
977
977
|
}[];
|
|
978
978
|
id: string;
|
|
@@ -995,6 +995,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
995
995
|
version?: string | undefined;
|
|
996
996
|
tags?: string[] | undefined;
|
|
997
997
|
metadata?: Record<string, JsonValue> | undefined;
|
|
998
|
+
description?: string | undefined;
|
|
998
999
|
judge?: {
|
|
999
1000
|
provider: AIProviderName;
|
|
1000
1001
|
model: string;
|
|
@@ -1029,7 +1030,6 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1029
1030
|
addModelAttribution?: boolean | undefined;
|
|
1030
1031
|
addExecutionTime?: boolean | undefined;
|
|
1031
1032
|
} | undefined;
|
|
1032
|
-
description?: string | undefined;
|
|
1033
1033
|
createdAt?: string | undefined;
|
|
1034
1034
|
updatedAt?: string | undefined;
|
|
1035
1035
|
modelGroups?: {
|
|
@@ -1041,11 +1041,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1041
1041
|
maxTokens?: number | undefined;
|
|
1042
1042
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1043
1043
|
systemPrompt?: string | undefined;
|
|
1044
|
+
weight?: number | undefined;
|
|
1044
1045
|
topK?: number | undefined;
|
|
1045
1046
|
topP?: number | undefined;
|
|
1046
1047
|
presencePenalty?: number | undefined;
|
|
1047
1048
|
frequencyPenalty?: number | undefined;
|
|
1048
|
-
weight?: number | undefined;
|
|
1049
1049
|
label?: string | undefined;
|
|
1050
1050
|
}[];
|
|
1051
1051
|
id: string;
|
|
@@ -1080,7 +1080,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1080
1080
|
blindEvaluation?: boolean | undefined;
|
|
1081
1081
|
}[] | undefined;
|
|
1082
1082
|
}, {
|
|
1083
|
-
type: "custom" | "
|
|
1083
|
+
type: "custom" | "chain" | "ensemble" | "adaptive";
|
|
1084
1084
|
name: string;
|
|
1085
1085
|
models: {
|
|
1086
1086
|
provider: string;
|
|
@@ -1090,11 +1090,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1090
1090
|
maxTokens?: number | undefined;
|
|
1091
1091
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1092
1092
|
systemPrompt?: string | undefined;
|
|
1093
|
+
weight?: number | undefined;
|
|
1093
1094
|
topK?: number | undefined;
|
|
1094
1095
|
topP?: number | undefined;
|
|
1095
1096
|
presencePenalty?: number | undefined;
|
|
1096
1097
|
frequencyPenalty?: number | undefined;
|
|
1097
|
-
weight?: number | undefined;
|
|
1098
1098
|
label?: string | undefined;
|
|
1099
1099
|
}[];
|
|
1100
1100
|
id: string;
|
|
@@ -1117,6 +1117,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1117
1117
|
version?: string | undefined;
|
|
1118
1118
|
tags?: string[] | undefined;
|
|
1119
1119
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1120
|
+
description?: string | undefined;
|
|
1120
1121
|
judge?: {
|
|
1121
1122
|
provider: string;
|
|
1122
1123
|
model: string;
|
|
@@ -1151,7 +1152,6 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1151
1152
|
addModelAttribution?: boolean | undefined;
|
|
1152
1153
|
addExecutionTime?: boolean | undefined;
|
|
1153
1154
|
} | undefined;
|
|
1154
|
-
description?: string | undefined;
|
|
1155
1155
|
createdAt?: string | undefined;
|
|
1156
1156
|
updatedAt?: string | undefined;
|
|
1157
1157
|
modelGroups?: {
|
|
@@ -1163,11 +1163,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1163
1163
|
maxTokens?: number | undefined;
|
|
1164
1164
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1165
1165
|
systemPrompt?: string | undefined;
|
|
1166
|
+
weight?: number | undefined;
|
|
1166
1167
|
topK?: number | undefined;
|
|
1167
1168
|
topP?: number | undefined;
|
|
1168
1169
|
presencePenalty?: number | undefined;
|
|
1169
1170
|
frequencyPenalty?: number | undefined;
|
|
1170
|
-
weight?: number | undefined;
|
|
1171
1171
|
label?: string | undefined;
|
|
1172
1172
|
}[];
|
|
1173
1173
|
id: string;
|
|
@@ -1202,7 +1202,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1202
1202
|
blindEvaluation?: boolean | undefined;
|
|
1203
1203
|
}[] | undefined;
|
|
1204
1204
|
}>, {
|
|
1205
|
-
type: "custom" | "
|
|
1205
|
+
type: "custom" | "chain" | "ensemble" | "adaptive";
|
|
1206
1206
|
name: string;
|
|
1207
1207
|
models: {
|
|
1208
1208
|
provider: AIProviderName;
|
|
@@ -1212,11 +1212,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1212
1212
|
maxTokens?: number | undefined;
|
|
1213
1213
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1214
1214
|
systemPrompt?: string | undefined;
|
|
1215
|
+
weight?: number | undefined;
|
|
1215
1216
|
topK?: number | undefined;
|
|
1216
1217
|
topP?: number | undefined;
|
|
1217
1218
|
presencePenalty?: number | undefined;
|
|
1218
1219
|
frequencyPenalty?: number | undefined;
|
|
1219
|
-
weight?: number | undefined;
|
|
1220
1220
|
label?: string | undefined;
|
|
1221
1221
|
}[];
|
|
1222
1222
|
id: string;
|
|
@@ -1239,6 +1239,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1239
1239
|
version?: string | undefined;
|
|
1240
1240
|
tags?: string[] | undefined;
|
|
1241
1241
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1242
|
+
description?: string | undefined;
|
|
1242
1243
|
judge?: {
|
|
1243
1244
|
provider: AIProviderName;
|
|
1244
1245
|
model: string;
|
|
@@ -1273,7 +1274,6 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1273
1274
|
addModelAttribution?: boolean | undefined;
|
|
1274
1275
|
addExecutionTime?: boolean | undefined;
|
|
1275
1276
|
} | undefined;
|
|
1276
|
-
description?: string | undefined;
|
|
1277
1277
|
createdAt?: string | undefined;
|
|
1278
1278
|
updatedAt?: string | undefined;
|
|
1279
1279
|
modelGroups?: {
|
|
@@ -1285,11 +1285,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1285
1285
|
maxTokens?: number | undefined;
|
|
1286
1286
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1287
1287
|
systemPrompt?: string | undefined;
|
|
1288
|
+
weight?: number | undefined;
|
|
1288
1289
|
topK?: number | undefined;
|
|
1289
1290
|
topP?: number | undefined;
|
|
1290
1291
|
presencePenalty?: number | undefined;
|
|
1291
1292
|
frequencyPenalty?: number | undefined;
|
|
1292
|
-
weight?: number | undefined;
|
|
1293
1293
|
label?: string | undefined;
|
|
1294
1294
|
}[];
|
|
1295
1295
|
id: string;
|
|
@@ -1324,7 +1324,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1324
1324
|
blindEvaluation?: boolean | undefined;
|
|
1325
1325
|
}[] | undefined;
|
|
1326
1326
|
}, {
|
|
1327
|
-
type: "custom" | "
|
|
1327
|
+
type: "custom" | "chain" | "ensemble" | "adaptive";
|
|
1328
1328
|
name: string;
|
|
1329
1329
|
models: {
|
|
1330
1330
|
provider: string;
|
|
@@ -1334,11 +1334,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1334
1334
|
maxTokens?: number | undefined;
|
|
1335
1335
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1336
1336
|
systemPrompt?: string | undefined;
|
|
1337
|
+
weight?: number | undefined;
|
|
1337
1338
|
topK?: number | undefined;
|
|
1338
1339
|
topP?: number | undefined;
|
|
1339
1340
|
presencePenalty?: number | undefined;
|
|
1340
1341
|
frequencyPenalty?: number | undefined;
|
|
1341
|
-
weight?: number | undefined;
|
|
1342
1342
|
label?: string | undefined;
|
|
1343
1343
|
}[];
|
|
1344
1344
|
id: string;
|
|
@@ -1361,6 +1361,7 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1361
1361
|
version?: string | undefined;
|
|
1362
1362
|
tags?: string[] | undefined;
|
|
1363
1363
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1364
|
+
description?: string | undefined;
|
|
1364
1365
|
judge?: {
|
|
1365
1366
|
provider: string;
|
|
1366
1367
|
model: string;
|
|
@@ -1395,7 +1396,6 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1395
1396
|
addModelAttribution?: boolean | undefined;
|
|
1396
1397
|
addExecutionTime?: boolean | undefined;
|
|
1397
1398
|
} | undefined;
|
|
1398
|
-
description?: string | undefined;
|
|
1399
1399
|
createdAt?: string | undefined;
|
|
1400
1400
|
updatedAt?: string | undefined;
|
|
1401
1401
|
modelGroups?: {
|
|
@@ -1407,11 +1407,11 @@ export declare const WorkflowConfigSchema: z.ZodEffects<z.ZodEffects<z.ZodObject
|
|
|
1407
1407
|
maxTokens?: number | undefined;
|
|
1408
1408
|
metadata?: Record<string, JsonValue> | undefined;
|
|
1409
1409
|
systemPrompt?: string | undefined;
|
|
1410
|
+
weight?: number | undefined;
|
|
1410
1411
|
topK?: number | undefined;
|
|
1411
1412
|
topP?: number | undefined;
|
|
1412
1413
|
presencePenalty?: number | undefined;
|
|
1413
1414
|
frequencyPenalty?: number | undefined;
|
|
1414
|
-
weight?: number | undefined;
|
|
1415
1415
|
label?: string | undefined;
|
|
1416
1416
|
}[];
|
|
1417
1417
|
id: string;
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import pLimit from "p-limit";
|
|
6
6
|
import { AIProviderFactory } from "../../core/factory.js";
|
|
7
7
|
import { logger } from "../../utils/logger.js";
|
|
8
|
+
import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../../observability/index.js";
|
|
8
9
|
import { WorkflowError } from "../types.js";
|
|
9
10
|
const functionTag = "EnsembleExecutor";
|
|
10
11
|
// ============================================================================
|
|
@@ -18,6 +19,11 @@ const functionTag = "EnsembleExecutor";
|
|
|
18
19
|
export async function executeEnsemble(options) {
|
|
19
20
|
const startTime = Date.now();
|
|
20
21
|
const { prompt, models, executionConfig, systemPrompt, workflowDefaults } = options;
|
|
22
|
+
const span = SpanSerializer.createSpan(SpanType.WORKFLOW, "workflow.ensemble", {
|
|
23
|
+
"workflow.operation": "ensemble",
|
|
24
|
+
"workflow.model_count": models.length,
|
|
25
|
+
"workflow.parallelism": executionConfig?.parallelism || 10,
|
|
26
|
+
});
|
|
21
27
|
logger.info(`[${functionTag}] Starting ensemble execution`, {
|
|
22
28
|
modelCount: models.length,
|
|
23
29
|
parallelism: executionConfig?.parallelism || 10,
|
|
@@ -65,6 +71,10 @@ export async function executeEnsemble(options) {
|
|
|
65
71
|
failureCount,
|
|
66
72
|
totalResponses: responses.length,
|
|
67
73
|
});
|
|
74
|
+
span.durationMs = totalTime;
|
|
75
|
+
const spanStatus = successCount > 0 ? SpanStatus.OK : SpanStatus.ERROR;
|
|
76
|
+
const endedSpan = SpanSerializer.endSpan(span, spanStatus, successCount === 0 ? "No successful model responses" : undefined);
|
|
77
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
68
78
|
return {
|
|
69
79
|
responses,
|
|
70
80
|
totalTime,
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { AIProviderFactory } from "../../core/factory.js";
|
|
6
6
|
import { logger } from "../../utils/logger.js";
|
|
7
|
+
import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../../observability/index.js";
|
|
7
8
|
import { MAX_REASONING_LENGTH } from "../config.js";
|
|
8
9
|
import { WorkflowError } from "../types.js";
|
|
9
10
|
const functionTag = "JudgeScorer";
|
|
@@ -18,6 +19,12 @@ const functionTag = "JudgeScorer";
|
|
|
18
19
|
export async function scoreEnsemble(options) {
|
|
19
20
|
const startTime = Date.now();
|
|
20
21
|
const { judges, responses, originalPrompt, systemPrompt, timeout, workflowDefaults, } = options;
|
|
22
|
+
const span = SpanSerializer.createSpan(SpanType.WORKFLOW, "workflow.judge", {
|
|
23
|
+
"workflow.operation": "judge",
|
|
24
|
+
"workflow.judge_count": judges.length,
|
|
25
|
+
"workflow.response_count": responses.length,
|
|
26
|
+
"workflow.pattern": judges.length > 1 ? "multi-judge" : "single-judge",
|
|
27
|
+
});
|
|
21
28
|
logger.info(`[${functionTag}] Starting judge scoring`, {
|
|
22
29
|
judgeCount: judges.length,
|
|
23
30
|
responseCount: responses.length,
|
|
@@ -36,17 +43,25 @@ export async function scoreEnsemble(options) {
|
|
|
36
43
|
if (judges.length === 1) {
|
|
37
44
|
// Single judge scoring
|
|
38
45
|
const judgeResult = await executeSingleJudge(judges[0], successfulResponses, originalPrompt, systemPrompt, timeout, workflowDefaults?.judgePrompt);
|
|
46
|
+
const judgeTime = Date.now() - startTime;
|
|
47
|
+
span.durationMs = judgeTime;
|
|
48
|
+
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
|
|
49
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
39
50
|
return {
|
|
40
51
|
scores: judgeResult,
|
|
41
|
-
judgeTime
|
|
52
|
+
judgeTime,
|
|
42
53
|
};
|
|
43
54
|
}
|
|
44
55
|
else {
|
|
45
56
|
// Multi-judge voting
|
|
46
57
|
const multiJudgeResult = await executeMultiJudge(judges, successfulResponses, originalPrompt, systemPrompt, timeout, workflowDefaults?.judgePrompt);
|
|
58
|
+
const judgeTime = Date.now() - startTime;
|
|
59
|
+
span.durationMs = judgeTime;
|
|
60
|
+
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
|
|
61
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
47
62
|
return {
|
|
48
63
|
scores: multiJudgeResult,
|
|
49
|
-
judgeTime
|
|
64
|
+
judgeTime,
|
|
50
65
|
};
|
|
51
66
|
}
|
|
52
67
|
}
|
|
@@ -55,6 +70,9 @@ export async function scoreEnsemble(options) {
|
|
|
55
70
|
logger.error(`[${functionTag}] Judge scoring failed`, {
|
|
56
71
|
error: err.message,
|
|
57
72
|
});
|
|
73
|
+
span.durationMs = Date.now() - startTime;
|
|
74
|
+
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR, err.message);
|
|
75
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
58
76
|
const workflowError = error instanceof WorkflowError
|
|
59
77
|
? error
|
|
60
78
|
: new WorkflowError(err.message, {
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* @module workflow/core/workflowRunner
|
|
13
13
|
*/
|
|
14
14
|
import { logger } from "../../utils/logger.js";
|
|
15
|
+
import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../../observability/index.js";
|
|
15
16
|
import { getModelGroups, PLACEHOLDER_MODEL, PLACEHOLDER_PROVIDER, usesModelGroups, } from "../config.js";
|
|
16
17
|
import { validateWorkflow } from "../utils/workflowValidation.js";
|
|
17
18
|
import { executeEnsemble, executeModelGroups } from "./ensembleExecutor.js";
|
|
@@ -45,9 +46,18 @@ import { conditionResponse } from "./responseConditioner.js";
|
|
|
45
46
|
*/
|
|
46
47
|
export async function runWorkflow(config, options) {
|
|
47
48
|
const startTime = Date.now();
|
|
49
|
+
const span = SpanSerializer.createSpan(SpanType.WORKFLOW, "workflow.run", {
|
|
50
|
+
"workflow.operation": "run",
|
|
51
|
+
"workflow.name": config.name,
|
|
52
|
+
"workflow.type": config.type,
|
|
53
|
+
"workflow.id": config.id,
|
|
54
|
+
});
|
|
48
55
|
// Validate configuration
|
|
49
56
|
const validation = validateWorkflow(config);
|
|
50
57
|
if (!validation.valid) {
|
|
58
|
+
span.durationMs = Date.now() - startTime;
|
|
59
|
+
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR, `Invalid workflow configuration: ${validation.errors.map((err) => err.message).join(", ")}`);
|
|
60
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
51
61
|
throw new Error(`Invalid workflow configuration: ${validation.errors.map((err) => err.message).join(", ")}`);
|
|
52
62
|
}
|
|
53
63
|
if (options.verbose) {
|
|
@@ -146,6 +156,9 @@ export async function runWorkflow(config, options) {
|
|
|
146
156
|
logger.debug(`[WorkflowRunner] Workflow complete in ${executionTime}ms`);
|
|
147
157
|
logger.debug(`[WorkflowRunner] Total tokens: ${result.usage?.totalTokens || 0}`);
|
|
148
158
|
}
|
|
159
|
+
span.durationMs = executionTime;
|
|
160
|
+
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
|
|
161
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
149
162
|
return result;
|
|
150
163
|
}
|
|
151
164
|
catch (error) {
|
|
@@ -154,6 +167,9 @@ export async function runWorkflow(config, options) {
|
|
|
154
167
|
if (options.verbose) {
|
|
155
168
|
logger.error(`[WorkflowRunner] Workflow failed:`, errorMessage);
|
|
156
169
|
}
|
|
170
|
+
span.durationMs = executionTime;
|
|
171
|
+
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR, errorMessage);
|
|
172
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
157
173
|
// Return error result with dummy data
|
|
158
174
|
const dummyResponse = {
|
|
159
175
|
provider: PLACEHOLDER_PROVIDER,
|
|
@@ -444,9 +460,18 @@ function calculateTotalTokens(responses) {
|
|
|
444
460
|
*/
|
|
445
461
|
export async function* runWorkflowWithStreaming(config, options) {
|
|
446
462
|
const startTime = Date.now();
|
|
463
|
+
const span = SpanSerializer.createSpan(SpanType.WORKFLOW, "workflow.run.streaming", {
|
|
464
|
+
"workflow.operation": "run.streaming",
|
|
465
|
+
"workflow.name": config.name,
|
|
466
|
+
"workflow.type": config.type,
|
|
467
|
+
"workflow.id": config.id,
|
|
468
|
+
});
|
|
447
469
|
// Validate configuration
|
|
448
470
|
const validation = validateWorkflow(config);
|
|
449
471
|
if (!validation.valid) {
|
|
472
|
+
span.durationMs = Date.now() - startTime;
|
|
473
|
+
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR, `Invalid workflow configuration: ${validation.errors.map((err) => err.message).join(", ")}`);
|
|
474
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
450
475
|
throw new Error(`Invalid workflow configuration: ${validation.errors.map((err) => err.message).join(", ")}`);
|
|
451
476
|
}
|
|
452
477
|
if (options.verbose) {
|
|
@@ -543,10 +568,18 @@ export async function* runWorkflowWithStreaming(config, options) {
|
|
|
543
568
|
timestamp: new Date().toISOString(),
|
|
544
569
|
},
|
|
545
570
|
};
|
|
571
|
+
span.durationMs = executionTime;
|
|
572
|
+
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.OK);
|
|
573
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
546
574
|
}
|
|
547
575
|
catch (error) {
|
|
576
|
+
const executionTime = Date.now() - startTime;
|
|
577
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
578
|
+
span.durationMs = executionTime;
|
|
579
|
+
const endedSpan = SpanSerializer.endSpan(span, SpanStatus.ERROR, errorMessage);
|
|
580
|
+
getMetricsAggregator().recordSpan(endedSpan);
|
|
548
581
|
logger.error(`[WorkflowRunner] Streaming workflow failed`, {
|
|
549
|
-
error:
|
|
582
|
+
error: errorMessage,
|
|
550
583
|
});
|
|
551
584
|
throw error;
|
|
552
585
|
}
|