@memberjunction/ai-prompts 2.77.0 → 2.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -1
- package/dist/AIPromptRunner.d.ts +3 -1
- package/dist/AIPromptRunner.d.ts.map +1 -1
- package/dist/AIPromptRunner.js +239 -36
- package/dist/AIPromptRunner.js.map +1 -1
- package/dist/ExecutionPlanner.d.ts +1 -0
- package/dist/ExecutionPlanner.d.ts.map +1 -1
- package/dist/ExecutionPlanner.js +43 -0
- package/dist/ExecutionPlanner.js.map +1 -1
- package/dist/ParallelExecution.d.ts +3 -0
- package/dist/ParallelExecution.d.ts.map +1 -1
- package/dist/ParallelExecutionCoordinator.d.ts.map +1 -1
- package/dist/ParallelExecutionCoordinator.js +12 -4
- package/dist/ParallelExecutionCoordinator.js.map +1 -1
- package/package.json +9 -9
package/README.md
CHANGED
|
@@ -2847,4 +2847,73 @@ WHERE ID = @PromptID;
|
|
|
2847
2847
|
5. **Use configuration parameters** for environment-specific settings beyond just model selection
|
|
2848
2848
|
6. **Assign models explicitly to configurations** to ensure clear separation between environments
|
|
2849
2849
|
|
|
2850
|
-
For more details on API key management, see the [AI Core API Keys documentation](../Core/README.md#api-key-management).
|
|
2850
|
+
For more details on API key management, see the [AI Core API Keys documentation](../Core/README.md#api-key-management).
|
|
2851
|
+
|
|
2852
|
+
## Model Selection Tracking (v2.78+)
|
|
2853
|
+
|
|
2854
|
+
The AIPromptRunner now provides comprehensive tracking of model selection decisions through the `modelSelectionInfo` property in `AIPromptRunResult`. This feature helps developers understand:
|
|
2855
|
+
|
|
2856
|
+
- Which models were considered during selection
|
|
2857
|
+
- Why specific models were or weren't available
|
|
2858
|
+
- Which configuration influenced the selection
|
|
2859
|
+
- What selection strategy was used
|
|
2860
|
+
|
|
2861
|
+
### Enhanced Model Selection Information
|
|
2862
|
+
|
|
2863
|
+
```typescript
|
|
2864
|
+
const result = await promptRunner.ExecutePrompt(params);
|
|
2865
|
+
|
|
2866
|
+
if (result.modelSelectionInfo) {
|
|
2867
|
+
// Access the configuration that was used
|
|
2868
|
+
const config = result.modelSelectionInfo.aiConfiguration;
|
|
2869
|
+
console.log(`Configuration: ${config?.Name || 'Default'}`);
|
|
2870
|
+
|
|
2871
|
+
// See all models that were considered
|
|
2872
|
+
for (const candidate of result.modelSelectionInfo.modelsConsidered) {
|
|
2873
|
+
console.log(`Model: ${candidate.model.Name}`);
|
|
2874
|
+
console.log(` Vendor: ${candidate.vendor?.Name || 'default'}`);
|
|
2875
|
+
console.log(` Priority: ${candidate.priority}`);
|
|
2876
|
+
console.log(` Available: ${candidate.available}`);
|
|
2877
|
+
if (!candidate.available) {
|
|
2878
|
+
console.log(` Reason: ${candidate.unavailableReason}`);
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
|
|
2882
|
+
// Understand the final selection
|
|
2883
|
+
console.log(`Selected: ${result.modelSelectionInfo.modelSelected.Name}`);
|
|
2884
|
+
console.log(`Vendor: ${result.modelSelectionInfo.vendorSelected?.Name}`);
|
|
2885
|
+
console.log(`Reason: ${result.modelSelectionInfo.selectionReason}`);
|
|
2886
|
+
console.log(`Strategy: ${result.modelSelectionInfo.selectionStrategy}`);
|
|
2887
|
+
}
|
|
2888
|
+
```
|
|
2889
|
+
|
|
2890
|
+
### Database Storage
|
|
2891
|
+
|
|
2892
|
+
Model selection information is stored in the `AIPromptRun.ModelSelection` field as JSON, containing:
|
|
2893
|
+
- Model and vendor IDs (not full entities)
|
|
2894
|
+
- Configuration ID and name
|
|
2895
|
+
- Array of considered models with their availability status
|
|
2896
|
+
- Selection reason and strategy
|
|
2897
|
+
|
|
2898
|
+
Additional fields track:
|
|
2899
|
+
- `SelectionStrategy`: The strategy used ('Default', 'Specific', 'ByPower')
|
|
2900
|
+
- `ModelPowerRank`: Power rank of the selected model
|
|
2901
|
+
- `Status`: Execution status ('Pending', 'Running', 'Completed', 'Failed', 'Cancelled')
|
|
2902
|
+
- `Cancelled`: Boolean flag for cancellation
|
|
2903
|
+
- `CancellationReason`: Why execution was cancelled
|
|
2904
|
+
- `ErrorDetails`: Detailed error information for failures
|
|
2905
|
+
|
|
2906
|
+
### Benefits
|
|
2907
|
+
|
|
2908
|
+
- **Debugging**: Understand why a specific model was selected
|
|
2909
|
+
- **Monitoring**: Track which models are being used across prompts
|
|
2910
|
+
- **Optimization**: Identify models that are frequently unavailable
|
|
2911
|
+
- **Compliance**: Audit model selection for regulatory requirements
|
|
2912
|
+
|
|
2913
|
+
## Version History
|
|
2914
|
+
|
|
2915
|
+
- **2.78.0** - Added model selection tracking with full entity objects in results
|
|
2916
|
+
- **2.77.0** - Enhanced status tracking and cancellation support
|
|
2917
|
+
- **2.76.0** - Added intelligent failover system
|
|
2918
|
+
- **2.75.0** - Introduced dynamic template composition
|
|
2919
|
+
- **2.50.0** - Initial release with core prompt execution
|
package/dist/AIPromptRunner.d.ts
CHANGED
|
@@ -45,6 +45,7 @@ export declare class AIPromptRunner {
|
|
|
45
45
|
model?: AIModelEntityExtended;
|
|
46
46
|
severity?: 'warning' | 'error' | 'critical';
|
|
47
47
|
}): void;
|
|
48
|
+
private isInferenceProvider;
|
|
48
49
|
ExecutePrompt<T = unknown>(params: AIPromptParams): Promise<AIPromptRunResult<T>>;
|
|
49
50
|
private executeSinglePrompt;
|
|
50
51
|
private executePromptInParallel;
|
|
@@ -53,10 +54,11 @@ export declare class AIPromptRunner {
|
|
|
53
54
|
private renderPromptWithChildTemplates;
|
|
54
55
|
private selectModel;
|
|
55
56
|
private buildModelVendorCandidates;
|
|
57
|
+
private selectModelWithAPIKeyTracked;
|
|
56
58
|
private selectModelWithAPIKey;
|
|
57
59
|
private createPromptRun;
|
|
58
60
|
private renderPromptTemplate;
|
|
59
|
-
protected executeModelWithFailover(model: AIModelEntityExtended, renderedPrompt: string, prompt: AIPromptEntity, params: AIPromptParams, vendorId: string | null, conversationMessages?: ChatMessage[], templateMessageRole?: TemplateMessageRole, cancellationToken?: AbortSignal, allCandidates?: ModelVendorCandidate[], promptRun?: AIPromptRunEntity): Promise<ChatResult>;
|
|
61
|
+
protected executeModelWithFailover(model: AIModelEntityExtended, renderedPrompt: string, prompt: AIPromptEntity, params: AIPromptParams, vendorId: string | null, conversationMessages?: ChatMessage[], templateMessageRole?: TemplateMessageRole, cancellationToken?: AbortSignal, allCandidates?: ModelVendorCandidate[], promptRun?: AIPromptRunEntity, vendorDriverClass?: string, vendorApiName?: string): Promise<ChatResult>;
|
|
60
62
|
protected buildFailoverCandidates(prompt: AIPromptEntity): Promise<ModelVendorCandidate[]>;
|
|
61
63
|
protected createCandidatesFromModels(models: AIModelEntityExtended[]): ModelVendorCandidate[];
|
|
62
64
|
protected updatePromptRunWithFailoverSuccess(promptRun: AIPromptRunEntity, failoverAttempts: FailoverAttempt[], currentModel: AIModelEntityExtended, currentVendorId: string | null): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AIPromptRunner.d.ts","sourceRoot":"","sources":["../src/AIPromptRunner.ts"],"names":[],"mappings":";AAAA,OAAO,EAAuB,UAAU,EAAmB,WAAW,EAA8B,MAAM,oBAAoB,CAAC;AAC/H,OAAO,EAAqB,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAGpF,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,iBAAiB,
|
|
1
|
+
{"version":3,"file":"AIPromptRunner.d.ts","sourceRoot":"","sources":["../src/AIPromptRunner.ts"],"names":[],"mappings":";AAAA,OAAO,EAAuB,UAAU,EAAmB,WAAW,EAA8B,MAAM,oBAAoB,CAAC;AAC/H,OAAO,EAAqB,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAGpF,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,iBAAiB,EAAmF,MAAM,+BAA+B,CAAC;AAQ1L,OAAO,EACH,mBAAmB,EAEnB,cAAc,EACjB,MAAM,8BAA8B,CAAC;AAkEtC,UAAU,oBAAoB;IAC5B,KAAK,EAAE,qBAAqB,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,UAAU,GAAG,cAAc,GAAG,YAAY,GAAG,YAAY,CAAC;CACnE;AAMD,UAAU,qBAAqB;IAC7B,QAAQ,EAAE,0BAA0B,GAAG,eAAe,GAAG,WAAW,GAAG,MAAM,CAAC;IAC9E,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,iBAAiB,GAAG,sBAAsB,GAAG,kBAAkB,CAAC;IAChF,UAAU,CAAC,EAAE,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,kBAAkB,CAAC;CAC3E;AAKD,UAAU,eAAe;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,iBAAiB,CAAmB;IAC5C,OAAO,CAAC,oBAAoB,CAA+B;IAC3D,OAAO,CAAC,cAAc,CAAgB;;IActC,OAAO,CAAC,aAAa;IAgBrB,SAAS,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,GAAE,OAAe,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI;IAejG,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE;QAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC/B,MAAM,CAAC,EAAE,cAAc,CAAC;QACxB,KAAK,CAAC,EAAE,qBAAqB,CAAC;QAC9B,QAAQ,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,UAAU,CAAC;KAC7C,GAAG,IAAI;IAkCR,OAAO,CAAC,mBAAmB;IA2Cd,aAAa,CAAC,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;YA6MhF,mBAAmB;YAmGnB,uBAAuB;YAsPvB,YAAY;YA0BZ,0BAA0B;YAoJ1B,8BAA8B;YAuE9B,WAAW;IA+KzB,OAAO,CAAC,0BAA0B;YA+NpB,4BAA4B;YAuF5B,qBAAqB;YAgErB,eAAe;YAwOf,oBAAoB;cAqDlB,wBAAwB,CACtC,KAAK,EAAE,qBAAqB,EAC5B,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,oBAAoB,CAAC,EAAE,WAAW,EAAE,EACpC,mBAAmB,GAAE,mBAA8B,EACnD,iBAAiB,CAAC,EAAE,WAAW,EAC/B,aAAa,CAAC,EAAE,oBAAoB,EAAE,EACtC,SAAS,CAAC,EAAE,iBAAiB,EAC7B,iBAAiB,CAAC,EAAE,MAAM,EAC1B,aAAa,CAAC,EAAE,MAAM,GACrB,OAAO,CAAC,UAAU,CAAC;cAwIN,uBAAuB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;IA2BhG,SAAS,CAAC,0BAA0B,CAAC,MAAM,EAAE,qBAAqB,EAAE,GAAG,oBAAoB,EAAE;IAwC7F,SAAS,CAAC,kCAAkC,CAC1C,SAAS,EAAE,iBAAiB,EAC5B,gBAAgB,EAAE,eAAe,EAAE,EACnC,YAAY,EAAE,qBAAqB,EACnC,eAAe,EAAE,MAAM,GAAG,IAAI,GAC7B,IAAI;IAiCP,OAAO,CAAC,kCAAkC;IA4B1C,OAAO,CAAC,yBAAyB;YAmBnB,YAAY;IA+K1B,OAAO,CAAC,iBAAiB;YAwCX,4BAA4B;YA4J5B,eAAe;IA0B7B,OAAO,CAAC,gCAAgC;IA0BxC,OAAO,CAAC,yBAAyB;IA8CjC,OAAO,CAAC,mBAAmB;IAqB3B,OAAO,CAAC,yBAAyB;IAuBjC,OAAO,CAAC,sBAAsB;YAyChB,8BAA8B;IAqI5C,OAAO,CAAC,iBAAiB;IAazB,OAAO,CAAC,iBAAiB;IA0BzB,OAAO,CAAC,kBAAkB;IA2B1B,OAAO,CAAC,eAAe;YA0BT,iBAAiB;YA+CjB,iBAAiB;YAiGjB,qBAAqB;YAuGrB,eAAe;IAwM7B,SAAS,CAAC,wBAAwB,CAAC,MAAM,EAAE,cAAc,GAAG,qBAAqB;IAgCjF,SAAS,CAAC,qBAAqB,CAC7B,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,qBAAqB,EAC7B,aAAa,EAAE,MAAM,GACpB,OAAO;IA0CV,SAAS,CAAC,sBAAsB,CAC9B,aAAa,EAAE,MAAM,EACrB,gBAAgB,EAAE,MAAM,GACvB,MAAM;IAgCT,SAAS,CAAC,wBAAwB,CAChC,YAAY,EAAE,qBAAqB,EACnC,eAAe,EAAE,MAAM,GAAG,SAAS,EACnC,QAAQ,EAAE,qBAAqB,CAAC,UAAU,CAAC,EAC3C,aAAa,EAAE,qBAAqB,CAAC,eAAe,CAAC,EACrD,aAAa,EAAE,oBAAoB,EAAE,EACrC,cAAc,EAAE,eAAe,EAAE,GAChC,oBAAoB,EAAE;IA4HzB,SAAS,CAAC,kBAAkB,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,eAAe,EACxB,SAAS,EAAE,OAAO,GACjB,IAAI;CA6BR;AAED,wBAAgB,kBAAkB,SAEjC"}
|
package/dist/AIPromptRunner.js
CHANGED
|
@@ -82,13 +82,21 @@ class AIPromptRunner {
|
|
|
82
82
|
metadata: Object.keys(metadata).length > 0 ? metadata : undefined
|
|
83
83
|
});
|
|
84
84
|
}
|
|
85
|
+
isInferenceProvider(modelVendor) {
|
|
86
|
+
const inferenceProviderType = aiengine_1.AIEngine.Instance.VendorTypeDefinitions.find(vt => vt.Name === 'Inference Provider');
|
|
87
|
+
if (!inferenceProviderType) {
|
|
88
|
+
const modelDeveloperType = aiengine_1.AIEngine.Instance.VendorTypeDefinitions.find(vt => vt.Name === 'Model Developer');
|
|
89
|
+
return modelVendor.TypeID !== modelDeveloperType?.ID;
|
|
90
|
+
}
|
|
91
|
+
return modelVendor.TypeID === inferenceProviderType.ID;
|
|
92
|
+
}
|
|
85
93
|
async ExecutePrompt(params) {
|
|
86
94
|
const startTime = new Date();
|
|
87
95
|
const promptRun = null;
|
|
88
96
|
if (params.cancellationToken?.aborted) {
|
|
89
97
|
const result = {
|
|
90
98
|
success: false,
|
|
91
|
-
status: '
|
|
99
|
+
status: 'Cancelled',
|
|
92
100
|
cancelled: true,
|
|
93
101
|
cancellationReason: 'user_requested',
|
|
94
102
|
errorMessage: 'Prompt execution was cancelled before starting',
|
|
@@ -111,12 +119,15 @@ class AIPromptRunner {
|
|
|
111
119
|
let parentPromptRun;
|
|
112
120
|
let selectedModel;
|
|
113
121
|
let childTemplateRenderingResult;
|
|
122
|
+
let modelSelectionInfo;
|
|
114
123
|
if (params.childPrompts && params.childPrompts.length > 0) {
|
|
115
124
|
let modelSelectionPrompt = prompt;
|
|
116
125
|
if (params.modelSelectionPrompt) {
|
|
117
126
|
modelSelectionPrompt = params.modelSelectionPrompt;
|
|
118
127
|
}
|
|
119
|
-
|
|
128
|
+
const modelResult = await this.selectModel(modelSelectionPrompt, params.override?.modelId, params.contextUser, params.configurationId, params.override?.vendorId, params);
|
|
129
|
+
selectedModel = modelResult.model;
|
|
130
|
+
modelSelectionInfo = modelResult.selectionInfo;
|
|
120
131
|
if (!selectedModel) {
|
|
121
132
|
throw new Error(`No suitable model found for prompt ${modelSelectionPrompt.Name}`);
|
|
122
133
|
}
|
|
@@ -128,7 +139,7 @@ class AIPromptRunner {
|
|
|
128
139
|
childTemplateRenderingResult = await this.renderChildPromptTemplates(params.childPrompts, params, params.cancellationToken);
|
|
129
140
|
renderedPromptText = await this.renderPromptWithChildTemplates(prompt, params, childTemplateRenderingResult.renderedTemplates);
|
|
130
141
|
}
|
|
131
|
-
parentPromptRun = await this.createPromptRun(prompt, selectedModel, params, renderedPromptText, startTime, params.override?.vendorId);
|
|
142
|
+
parentPromptRun = await this.createPromptRun(prompt, selectedModel, params, renderedPromptText, startTime, params.override?.vendorId, modelSelectionInfo);
|
|
132
143
|
}
|
|
133
144
|
else if (prompt.TemplateID && (!params.conversationMessages || params.templateMessageRole !== 'none')) {
|
|
134
145
|
if (params.systemPromptOverride) {
|
|
@@ -157,7 +168,9 @@ class AIPromptRunner {
|
|
|
157
168
|
modelSelectionPrompt = params.modelSelectionPrompt;
|
|
158
169
|
this.logStatus(`🎯 Using prompt "${modelSelectionPrompt.Name}" for model selection instead of main prompt`, true, params);
|
|
159
170
|
}
|
|
160
|
-
|
|
171
|
+
const modelResult = await this.selectModel(modelSelectionPrompt, params.override?.modelId, params.contextUser, params.configurationId, params.override?.vendorId, params);
|
|
172
|
+
selectedModel = modelResult.model;
|
|
173
|
+
modelSelectionInfo = modelResult.selectionInfo;
|
|
161
174
|
if (!selectedModel) {
|
|
162
175
|
throw new Error(`No suitable model found for prompt ${modelSelectionPrompt.Name}`);
|
|
163
176
|
}
|
|
@@ -170,6 +183,9 @@ class AIPromptRunner {
|
|
|
170
183
|
else {
|
|
171
184
|
result = await this.executeSinglePrompt(prompt, renderedPromptText, params, startTime, parentPromptRun, selectedModel);
|
|
172
185
|
}
|
|
186
|
+
if (modelSelectionInfo) {
|
|
187
|
+
result.modelSelectionInfo = modelSelectionInfo;
|
|
188
|
+
}
|
|
173
189
|
return result;
|
|
174
190
|
}
|
|
175
191
|
catch (error) {
|
|
@@ -186,6 +202,15 @@ class AIPromptRunner {
|
|
|
186
202
|
promptRun.CompletedAt = endTime;
|
|
187
203
|
promptRun.ExecutionTimeMS = executionTimeMS;
|
|
188
204
|
promptRun.Result = `ERROR: ${error.message}`;
|
|
205
|
+
if (error.message.includes('cancelled')) {
|
|
206
|
+
promptRun.Status = 'Cancelled';
|
|
207
|
+
promptRun.Cancelled = true;
|
|
208
|
+
promptRun.CancellationReason = 'user_requested';
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
promptRun.Status = 'Failed';
|
|
212
|
+
promptRun.ErrorDetails = error.message;
|
|
213
|
+
}
|
|
189
214
|
const saveResult = await promptRun.Save();
|
|
190
215
|
if (!saveResult) {
|
|
191
216
|
this.logError(`Failed to save error to AIPromptRun: ${promptRun.LatestResult?.Message || 'Unknown error'}`, {
|
|
@@ -214,13 +239,20 @@ class AIPromptRunner {
|
|
|
214
239
|
throw new Error('Prompt execution was cancelled before model selection');
|
|
215
240
|
}
|
|
216
241
|
let selectedModel = existingModel;
|
|
242
|
+
let modelSelectionInfo;
|
|
243
|
+
let vendorDriverClass;
|
|
244
|
+
let vendorApiName;
|
|
217
245
|
if (!selectedModel) {
|
|
218
246
|
let modelSelectionPrompt = prompt;
|
|
219
247
|
if (params.modelSelectionPrompt) {
|
|
220
248
|
modelSelectionPrompt = params.modelSelectionPrompt;
|
|
221
249
|
this.logStatus(` Using prompt "${modelSelectionPrompt.Name}" for model selection instead of main prompt`, true, params);
|
|
222
250
|
}
|
|
223
|
-
|
|
251
|
+
const modelResult = await this.selectModel(modelSelectionPrompt, params.override?.modelId, params.contextUser, params.configurationId, params.override?.vendorId, params);
|
|
252
|
+
selectedModel = modelResult.model;
|
|
253
|
+
vendorDriverClass = modelResult.vendorDriverClass;
|
|
254
|
+
vendorApiName = modelResult.vendorApiName;
|
|
255
|
+
modelSelectionInfo = modelResult.selectionInfo;
|
|
224
256
|
if (!selectedModel) {
|
|
225
257
|
throw new Error(`No suitable model found for prompt ${modelSelectionPrompt.Name}`);
|
|
226
258
|
}
|
|
@@ -228,11 +260,11 @@ class AIPromptRunner {
|
|
|
228
260
|
if (params.cancellationToken?.aborted) {
|
|
229
261
|
throw new Error('Prompt execution was cancelled after model selection');
|
|
230
262
|
}
|
|
231
|
-
const promptRun = existingPromptRun || await this.createPromptRun(prompt, selectedModel, params, renderedPromptText, startTime, params.override?.vendorId);
|
|
263
|
+
const promptRun = existingPromptRun || await this.createPromptRun(prompt, selectedModel, params, renderedPromptText, startTime, params.override?.vendorId, modelSelectionInfo);
|
|
232
264
|
if (params.cancellationToken?.aborted) {
|
|
233
265
|
throw new Error('Prompt execution was cancelled before model execution');
|
|
234
266
|
}
|
|
235
|
-
const { modelResult, parsedResult, validationAttempts, cumulativeTokens } = await this.executeWithValidationRetries(selectedModel, renderedPromptText, prompt, params, promptRun);
|
|
267
|
+
const { modelResult, parsedResult, validationAttempts, cumulativeTokens } = await this.executeWithValidationRetries(selectedModel, renderedPromptText, prompt, params, promptRun, vendorDriverClass, vendorApiName);
|
|
236
268
|
const endTime = new Date();
|
|
237
269
|
const executionTimeMS = endTime.getTime() - startTime.getTime();
|
|
238
270
|
await this.updatePromptRun(promptRun, prompt, modelResult, parsedResult, endTime, executionTimeMS, validationAttempts, cumulativeTokens);
|
|
@@ -309,7 +341,7 @@ class AIPromptRunner {
|
|
|
309
341
|
}
|
|
310
342
|
}
|
|
311
343
|
}
|
|
312
|
-
const consolidatedPromptRun = existingPromptRun || await this.createPromptRun(prompt, selectedResult.task.model, params, renderedPromptText, startTime, params.override?.vendorId);
|
|
344
|
+
const consolidatedPromptRun = existingPromptRun || await this.createPromptRun(prompt, selectedResult.task.model, params, renderedPromptText, startTime, params.override?.vendorId, undefined);
|
|
313
345
|
const endTime = new Date();
|
|
314
346
|
consolidatedPromptRun.CompletedAt = endTime;
|
|
315
347
|
consolidatedPromptRun.ExecutionTimeMS = parallelResult.totalExecutionTimeMS;
|
|
@@ -348,6 +380,8 @@ class AIPromptRunner {
|
|
|
348
380
|
if (hasCost) {
|
|
349
381
|
consolidatedPromptRun.TotalCost = totalCost;
|
|
350
382
|
}
|
|
383
|
+
consolidatedPromptRun.Status = parallelResult.successCount > 0 ? 'Completed' : 'Failed';
|
|
384
|
+
consolidatedPromptRun.WasSelectedResult = true;
|
|
351
385
|
const saveResult = await consolidatedPromptRun.Save();
|
|
352
386
|
if (!saveResult) {
|
|
353
387
|
this.logError(`Failed to save consolidated AIPromptRun: ${consolidatedPromptRun.LatestResult?.Message || 'Unknown error'}`, {
|
|
@@ -571,33 +605,115 @@ class AIPromptRunner {
|
|
|
571
605
|
}
|
|
572
606
|
}
|
|
573
607
|
async selectModel(prompt, explicitModelId, contextUser, configurationId, vendorId, params) {
|
|
608
|
+
let configurationName;
|
|
609
|
+
let configuration;
|
|
574
610
|
try {
|
|
575
611
|
await aiengine_1.AIEngine.Instance.Config(false, contextUser);
|
|
612
|
+
let selectionStrategy = 'Default';
|
|
613
|
+
if (explicitModelId) {
|
|
614
|
+
selectionStrategy = 'Specific';
|
|
615
|
+
}
|
|
616
|
+
else if (prompt.SelectionStrategy === 'Specific') {
|
|
617
|
+
selectionStrategy = 'Specific';
|
|
618
|
+
}
|
|
619
|
+
else if (prompt.SelectionStrategy === 'ByPower' || prompt.MinPowerRank != null) {
|
|
620
|
+
selectionStrategy = 'ByPower';
|
|
621
|
+
}
|
|
622
|
+
if (configurationId) {
|
|
623
|
+
configuration = aiengine_1.AIEngine.Instance.Configurations.find(c => c.ID === configurationId);
|
|
624
|
+
configurationName = configuration?.Name;
|
|
625
|
+
}
|
|
576
626
|
const candidates = this.buildModelVendorCandidates(prompt, explicitModelId, configurationId, vendorId);
|
|
627
|
+
const modelsConsidered = [];
|
|
577
628
|
if (candidates.length === 0) {
|
|
578
629
|
this.logError(`No suitable model candidates found for prompt ${prompt.Name}`, {
|
|
579
630
|
category: 'ModelSelection',
|
|
580
631
|
prompt: prompt,
|
|
581
632
|
severity: 'critical'
|
|
582
633
|
});
|
|
583
|
-
return
|
|
634
|
+
return {
|
|
635
|
+
model: null,
|
|
636
|
+
selectionInfo: {
|
|
637
|
+
aiConfiguration: configuration,
|
|
638
|
+
modelsConsidered: [],
|
|
639
|
+
modelSelected: undefined,
|
|
640
|
+
selectionReason: 'No suitable model candidates found',
|
|
641
|
+
fallbackUsed: false,
|
|
642
|
+
selectionStrategy
|
|
643
|
+
}
|
|
644
|
+
};
|
|
584
645
|
}
|
|
585
|
-
const selected = await this.
|
|
646
|
+
const { selected, consideredModels } = await this.selectModelWithAPIKeyTracked(candidates, params);
|
|
647
|
+
modelsConsidered.push(...consideredModels);
|
|
586
648
|
if (!selected) {
|
|
587
|
-
return
|
|
649
|
+
return {
|
|
650
|
+
model: null,
|
|
651
|
+
vendorDriverClass: undefined,
|
|
652
|
+
vendorApiName: undefined,
|
|
653
|
+
selectionInfo: {
|
|
654
|
+
aiConfiguration: configuration,
|
|
655
|
+
modelsConsidered,
|
|
656
|
+
modelSelected: undefined,
|
|
657
|
+
selectionReason: 'No API keys found for any model-vendor combination',
|
|
658
|
+
fallbackUsed: false,
|
|
659
|
+
selectionStrategy
|
|
660
|
+
}
|
|
661
|
+
};
|
|
588
662
|
}
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
663
|
+
let selectionReason = `Selected ${selected.model.Name} via ${selected.vendorName || 'default vendor'}`;
|
|
664
|
+
if (selected.source === 'explicit') {
|
|
665
|
+
selectionReason = `Explicitly requested model ${selected.model.Name}`;
|
|
666
|
+
}
|
|
667
|
+
else if (selected.source === 'prompt-model') {
|
|
668
|
+
selectionReason = `Selected from prompt-specific models (priority: ${selected.priority})`;
|
|
669
|
+
}
|
|
670
|
+
else if (selected.source === 'model-type') {
|
|
671
|
+
selectionReason = `Selected based on model type filtering`;
|
|
672
|
+
}
|
|
673
|
+
else if (selected.source === 'power-rank') {
|
|
674
|
+
selectionReason = `Selected by power rank (${selected.model.PowerRank || 0})`;
|
|
675
|
+
}
|
|
676
|
+
if (selected.isPreferredVendor) {
|
|
677
|
+
selectionReason += ' using preferred vendor';
|
|
678
|
+
}
|
|
679
|
+
const fallbackUsed = candidates.indexOf(selected) > 0;
|
|
680
|
+
let selectedVendor;
|
|
681
|
+
if (selected.vendorId) {
|
|
682
|
+
selectedVendor = aiengine_1.AIEngine.Instance.Vendors.find(v => v.ID === selected.vendorId);
|
|
683
|
+
}
|
|
684
|
+
return {
|
|
685
|
+
model: selected.model,
|
|
686
|
+
vendorDriverClass: selected.driverClass,
|
|
687
|
+
vendorApiName: selected.apiName,
|
|
688
|
+
selectionInfo: {
|
|
689
|
+
aiConfiguration: configuration,
|
|
690
|
+
modelsConsidered,
|
|
691
|
+
modelSelected: selected.model,
|
|
692
|
+
vendorSelected: selectedVendor,
|
|
693
|
+
selectionReason,
|
|
694
|
+
fallbackUsed,
|
|
695
|
+
selectionStrategy
|
|
696
|
+
}
|
|
697
|
+
};
|
|
594
698
|
}
|
|
595
699
|
catch (error) {
|
|
596
700
|
this.logError(error, {
|
|
597
701
|
category: 'ModelSelection',
|
|
598
702
|
prompt: prompt
|
|
599
703
|
});
|
|
600
|
-
return
|
|
704
|
+
return {
|
|
705
|
+
model: null,
|
|
706
|
+
vendorDriverClass: undefined,
|
|
707
|
+
vendorApiName: undefined,
|
|
708
|
+
selectionInfo: {
|
|
709
|
+
aiConfiguration: configuration,
|
|
710
|
+
modelsConsidered: [],
|
|
711
|
+
modelSelected: undefined,
|
|
712
|
+
selectionReason: `Error during model selection: ${error.message}`,
|
|
713
|
+
fallbackUsed: false,
|
|
714
|
+
selectionStrategy: 'Default'
|
|
715
|
+
}
|
|
716
|
+
};
|
|
601
717
|
}
|
|
602
718
|
}
|
|
603
719
|
buildModelVendorCandidates(prompt, explicitModelId, configurationId, preferredVendorId) {
|
|
@@ -607,7 +723,7 @@ class AIPromptRunner {
|
|
|
607
723
|
const createCandidatesForModel = (model, basePriority, source, promptModelPriority) => {
|
|
608
724
|
const modelCandidates = [];
|
|
609
725
|
const modelVendors = aiengine_1.AIEngine.Instance.ModelVendors
|
|
610
|
-
.filter(mv => mv.ModelID === model.ID && mv.Status === 'Active')
|
|
726
|
+
.filter(mv => mv.ModelID === model.ID && mv.Status === 'Active' && this.isInferenceProvider(mv))
|
|
611
727
|
.sort((a, b) => b.Priority - a.Priority);
|
|
612
728
|
if (preferredVendorId) {
|
|
613
729
|
const preferredVendor = modelVendors.find(mv => mv.VendorID === preferredVendorId);
|
|
@@ -685,7 +801,7 @@ class AIPromptRunner {
|
|
|
685
801
|
if (model && model.IsActive) {
|
|
686
802
|
const pmPreferredVendorId = pm.VendorID || preferredVendorId;
|
|
687
803
|
if (pmPreferredVendorId && pmPreferredVendorId !== preferredVendorId) {
|
|
688
|
-
const vendor = aiengine_1.AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === model.ID && mv.VendorID === pmPreferredVendorId && mv.Status === 'Active');
|
|
804
|
+
const vendor = aiengine_1.AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === model.ID && mv.VendorID === pmPreferredVendorId && mv.Status === 'Active' && this.isInferenceProvider(mv));
|
|
689
805
|
if (vendor) {
|
|
690
806
|
candidates.push({
|
|
691
807
|
model,
|
|
@@ -711,7 +827,8 @@ class AIPromptRunner {
|
|
|
711
827
|
(!preferredVendorName ||
|
|
712
828
|
aiengine_1.AIEngine.Instance.ModelVendors.some(mv => mv.ModelID === m.ID &&
|
|
713
829
|
mv.Status === 'Active' &&
|
|
714
|
-
mv.Vendor === preferredVendorName
|
|
830
|
+
mv.Vendor === preferredVendorName &&
|
|
831
|
+
this.isInferenceProvider(mv))));
|
|
715
832
|
switch (prompt.SelectionStrategy) {
|
|
716
833
|
case 'ByPower':
|
|
717
834
|
switch (prompt.PowerPreference) {
|
|
@@ -742,6 +859,52 @@ class AIPromptRunner {
|
|
|
742
859
|
candidates.sort((a, b) => b.priority - a.priority);
|
|
743
860
|
return candidates;
|
|
744
861
|
}
|
|
862
|
+
async selectModelWithAPIKeyTracked(candidates, params) {
|
|
863
|
+
const checkedDrivers = new Map();
|
|
864
|
+
const consideredModels = [];
|
|
865
|
+
let attemptCount = 0;
|
|
866
|
+
for (const candidate of candidates) {
|
|
867
|
+
attemptCount++;
|
|
868
|
+
let hasKey;
|
|
869
|
+
if (checkedDrivers.has(candidate.driverClass)) {
|
|
870
|
+
hasKey = checkedDrivers.get(candidate.driverClass);
|
|
871
|
+
}
|
|
872
|
+
else {
|
|
873
|
+
const apiKey = (0, ai_1.GetAIAPIKey)(candidate.driverClass);
|
|
874
|
+
hasKey = this.isValidAPIKey(apiKey);
|
|
875
|
+
checkedDrivers.set(candidate.driverClass, hasKey);
|
|
876
|
+
}
|
|
877
|
+
let vendorEntity;
|
|
878
|
+
if (candidate.vendorId) {
|
|
879
|
+
vendorEntity = aiengine_1.AIEngine.Instance.Vendors.find(v => v.ID === candidate.vendorId);
|
|
880
|
+
}
|
|
881
|
+
consideredModels.push({
|
|
882
|
+
model: candidate.model,
|
|
883
|
+
vendor: vendorEntity,
|
|
884
|
+
priority: candidate.priority,
|
|
885
|
+
available: hasKey,
|
|
886
|
+
unavailableReason: hasKey ? undefined : `No API key for driver ${candidate.driverClass}`
|
|
887
|
+
});
|
|
888
|
+
if (hasKey) {
|
|
889
|
+
(0, core_1.LogStatus)(` Selected model ${candidate.model.Name} with ${candidate.vendorName || 'default'} vendor (driver: ${candidate.driverClass})`);
|
|
890
|
+
if (candidate.isPreferredVendor) {
|
|
891
|
+
this.logStatus(` Using preferred vendor${candidate.vendorId ? ` (${candidate.vendorName})` : ''}`, true, params);
|
|
892
|
+
}
|
|
893
|
+
this.logStatus(` Checked ${attemptCount} candidate(s) before finding valid API key`, true, params);
|
|
894
|
+
return { selected: candidate, consideredModels };
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
const triedSummary = candidates.slice(0, 5).map(c => `${c.model.Name}/${c.vendorName || 'default'}(${c.driverClass})`).join(', ');
|
|
898
|
+
this.logError(`No API keys found for any model-vendor combination. Tried: ${triedSummary}${candidates.length > 5 ? `... (${candidates.length} total)` : ''}`, {
|
|
899
|
+
category: 'APIKeyValidation',
|
|
900
|
+
severity: 'critical',
|
|
901
|
+
metadata: {
|
|
902
|
+
candidatesChecked: candidates.length,
|
|
903
|
+
modelsChecked: consideredModels.length
|
|
904
|
+
}
|
|
905
|
+
});
|
|
906
|
+
return { selected: null, consideredModels };
|
|
907
|
+
}
|
|
745
908
|
async selectModelWithAPIKey(candidates, params) {
|
|
746
909
|
const checkedDrivers = new Map();
|
|
747
910
|
let attemptCount = 0;
|
|
@@ -784,12 +947,42 @@ class AIPromptRunner {
|
|
|
784
947
|
});
|
|
785
948
|
return null;
|
|
786
949
|
}
|
|
787
|
-
async createPromptRun(prompt, model, params, systemPromptText, startTime, vendorId) {
|
|
950
|
+
async createPromptRun(prompt, model, params, systemPromptText, startTime, vendorId, modelSelectionInfo) {
|
|
788
951
|
const promptRun = await this._metadata.GetEntityObject('MJ: AI Prompt Runs', params.contextUser);
|
|
789
952
|
try {
|
|
790
953
|
promptRun.NewRecord();
|
|
791
954
|
promptRun.PromptID = prompt.ID;
|
|
792
955
|
promptRun.ModelID = model.ID;
|
|
956
|
+
promptRun.Status = 'Running';
|
|
957
|
+
promptRun.Cancelled = false;
|
|
958
|
+
promptRun.CacheHit = false;
|
|
959
|
+
promptRun.StreamingEnabled = false;
|
|
960
|
+
promptRun.WasSelectedResult = false;
|
|
961
|
+
if (modelSelectionInfo) {
|
|
962
|
+
const dbSelectionInfo = {
|
|
963
|
+
configurationId: modelSelectionInfo.aiConfiguration?.ID,
|
|
964
|
+
configurationName: modelSelectionInfo.aiConfiguration?.Name,
|
|
965
|
+
modelsConsidered: modelSelectionInfo.modelsConsidered.map(mc => ({
|
|
966
|
+
modelId: mc.model.ID,
|
|
967
|
+
modelName: mc.model.Name,
|
|
968
|
+
vendorId: mc.vendor?.ID,
|
|
969
|
+
vendorName: mc.vendor?.Name || 'default',
|
|
970
|
+
priority: mc.priority,
|
|
971
|
+
available: mc.available,
|
|
972
|
+
unavailableReason: mc.unavailableReason
|
|
973
|
+
})),
|
|
974
|
+
modelSelected: modelSelectionInfo.modelSelected?.ID,
|
|
975
|
+
vendorSelected: modelSelectionInfo.vendorSelected?.ID,
|
|
976
|
+
selectionReason: modelSelectionInfo.selectionReason,
|
|
977
|
+
fallbackUsed: modelSelectionInfo.fallbackUsed,
|
|
978
|
+
selectionStrategy: modelSelectionInfo.selectionStrategy
|
|
979
|
+
};
|
|
980
|
+
promptRun.ModelSelection = JSON.stringify(dbSelectionInfo);
|
|
981
|
+
promptRun.SelectionStrategy = modelSelectionInfo.selectionStrategy || 'Default';
|
|
982
|
+
if (model.PowerRank != null) {
|
|
983
|
+
promptRun.ModelPowerRank = model.PowerRank;
|
|
984
|
+
}
|
|
985
|
+
}
|
|
793
986
|
const promptRunWithFailover = promptRun;
|
|
794
987
|
promptRunWithFailover.OriginalModelID = model.ID;
|
|
795
988
|
promptRunWithFailover.OriginalRequestStartTime = startTime;
|
|
@@ -806,7 +999,7 @@ class AIPromptRunner {
|
|
|
806
999
|
}
|
|
807
1000
|
else {
|
|
808
1001
|
const modelVendors = aiengine_1.AIEngine.Instance.ModelVendors
|
|
809
|
-
.filter((mv) => mv.ModelID === model.ID && mv.Status === 'Active')
|
|
1002
|
+
.filter((mv) => mv.ModelID === model.ID && mv.Status === 'Active' && this.isInferenceProvider(mv))
|
|
810
1003
|
.sort((a, b) => b.Priority - a.Priority);
|
|
811
1004
|
if (modelVendors.length > 0) {
|
|
812
1005
|
promptRun.VendorID = modelVendors[0].VendorID;
|
|
@@ -961,10 +1154,10 @@ class AIPromptRunner {
|
|
|
961
1154
|
throw error;
|
|
962
1155
|
}
|
|
963
1156
|
}
|
|
964
|
-
async executeModelWithFailover(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole = 'system', cancellationToken, allCandidates, promptRun) {
|
|
1157
|
+
async executeModelWithFailover(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole = 'system', cancellationToken, allCandidates, promptRun, vendorDriverClass, vendorApiName) {
|
|
965
1158
|
const failoverConfig = this.getFailoverConfiguration(prompt);
|
|
966
1159
|
if (failoverConfig.strategy === 'None') {
|
|
967
|
-
return this.executeModel(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole, cancellationToken);
|
|
1160
|
+
return this.executeModel(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole, cancellationToken, vendorDriverClass, vendorApiName);
|
|
968
1161
|
}
|
|
969
1162
|
const failoverAttempts = [];
|
|
970
1163
|
let lastError = null;
|
|
@@ -990,7 +1183,7 @@ class AIPromptRunner {
|
|
|
990
1183
|
}]
|
|
991
1184
|
});
|
|
992
1185
|
}
|
|
993
|
-
const result = await this.executeModel(currentModel, renderedPrompt, prompt, params, currentVendorId, conversationMessages, templateMessageRole, cancellationToken);
|
|
1186
|
+
const result = await this.executeModel(currentModel, renderedPrompt, prompt, params, currentVendorId, conversationMessages, templateMessageRole, cancellationToken, vendorDriverClass, vendorApiName);
|
|
994
1187
|
if (failoverAttempts.length > 0 && promptRun) {
|
|
995
1188
|
this.updatePromptRunWithFailoverSuccess(promptRun, failoverAttempts, currentModel, currentVendorId);
|
|
996
1189
|
}
|
|
@@ -1023,6 +1216,8 @@ class AIPromptRunner {
|
|
|
1023
1216
|
const nextCandidate = nextCandidates[0];
|
|
1024
1217
|
currentModel = nextCandidate.model;
|
|
1025
1218
|
currentVendorId = nextCandidate.vendorId;
|
|
1219
|
+
vendorDriverClass = nextCandidate.driverClass;
|
|
1220
|
+
vendorApiName = nextCandidate.apiName;
|
|
1026
1221
|
this.logFailoverAttempt(prompt.ID, failoverAttempt, true);
|
|
1027
1222
|
if (attemptNumber < failoverConfig.maxAttempts) {
|
|
1028
1223
|
const delay = this.calculateFailoverDelay(attemptNumber, failoverConfig.delaySeconds);
|
|
@@ -1127,30 +1322,29 @@ class AIPromptRunner {
|
|
|
1127
1322
|
data: null
|
|
1128
1323
|
};
|
|
1129
1324
|
}
|
|
1130
|
-
async executeModel(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole = 'system', cancellationToken) {
|
|
1325
|
+
async executeModel(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole = 'system', cancellationToken, vendorDriverClass, vendorApiName) {
|
|
1131
1326
|
let driverClass;
|
|
1132
1327
|
let apiName;
|
|
1133
1328
|
let llm;
|
|
1134
1329
|
let chatParams;
|
|
1135
1330
|
try {
|
|
1136
1331
|
const verbose = params.verbose === true || (0, core_1.IsVerboseLoggingEnabled)();
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
apiName = modelWithVendor._selectedApiName;
|
|
1141
|
-
delete modelWithVendor._selectedVendorId;
|
|
1142
|
-
delete modelWithVendor._selectedDriverClass;
|
|
1143
|
-
delete modelWithVendor._selectedApiName;
|
|
1332
|
+
if (vendorDriverClass && vendorApiName) {
|
|
1333
|
+
driverClass = vendorDriverClass;
|
|
1334
|
+
apiName = vendorApiName;
|
|
1144
1335
|
}
|
|
1145
1336
|
else {
|
|
1146
1337
|
driverClass = model.DriverClass;
|
|
1147
1338
|
apiName = model.APIName;
|
|
1148
1339
|
if (vendorId) {
|
|
1149
|
-
const modelVendor = aiengine_1.AIEngine.Instance.ModelVendors.find((mv) => mv.ModelID === model.ID && mv.VendorID === vendorId && mv.Status === 'Active');
|
|
1340
|
+
const modelVendor = aiengine_1.AIEngine.Instance.ModelVendors.find((mv) => mv.ModelID === model.ID && mv.VendorID === vendorId && mv.Status === 'Active' && this.isInferenceProvider(mv));
|
|
1150
1341
|
if (modelVendor) {
|
|
1151
1342
|
driverClass = modelVendor.DriverClass || driverClass;
|
|
1152
1343
|
apiName = modelVendor.APIName || apiName;
|
|
1153
1344
|
}
|
|
1345
|
+
else {
|
|
1346
|
+
this.logStatus(`⚠️ Vendor ${vendorId} not found or is not an inference provider for model ${model.Name}, using model defaults`, true, params);
|
|
1347
|
+
}
|
|
1154
1348
|
}
|
|
1155
1349
|
}
|
|
1156
1350
|
let apiKey;
|
|
@@ -1296,7 +1490,7 @@ class AIPromptRunner {
|
|
|
1296
1490
|
}
|
|
1297
1491
|
return messages;
|
|
1298
1492
|
}
|
|
1299
|
-
async executeWithValidationRetries(selectedModel, renderedPromptText, prompt, params, promptRun) {
|
|
1493
|
+
async executeWithValidationRetries(selectedModel, renderedPromptText, prompt, params, promptRun, vendorDriverClass, vendorApiName) {
|
|
1300
1494
|
const validationAttempts = [];
|
|
1301
1495
|
const maxRetries = Math.max(0, prompt.MaxRetries || 0);
|
|
1302
1496
|
let lastError = null;
|
|
@@ -1312,7 +1506,7 @@ class AIPromptRunner {
|
|
|
1312
1506
|
(0, core_1.LogStatus)(` 🔄 Retrying execution due to validation failure, attempt ${attempt + 1}/${maxRetries + 1}`);
|
|
1313
1507
|
await this.applyRetryDelay(prompt, attempt);
|
|
1314
1508
|
}
|
|
1315
|
-
const modelResult = await this.executeModelWithFailover(selectedModel, renderedPromptText, prompt, params, promptRun.VendorID, params.conversationMessages, params.templateMessageRole || 'system', params.cancellationToken, undefined, promptRun);
|
|
1509
|
+
const modelResult = await this.executeModelWithFailover(selectedModel, renderedPromptText, prompt, params, promptRun.VendorID, params.conversationMessages, params.templateMessageRole || 'system', params.cancellationToken, undefined, promptRun, vendorDriverClass, vendorApiName);
|
|
1316
1510
|
if (modelResult.data?.usage) {
|
|
1317
1511
|
cumulativePromptTokens += modelResult.data.usage.promptTokens || 0;
|
|
1318
1512
|
cumulativeCompletionTokens += modelResult.data.usage.completionTokens || 0;
|
|
@@ -1857,6 +2051,15 @@ class AIPromptRunner {
|
|
|
1857
2051
|
promptRun.TotalRetryDurationMS = 0;
|
|
1858
2052
|
}
|
|
1859
2053
|
promptRun.Success = modelResult.success && (parsedResult.validationResult?.Success !== false);
|
|
2054
|
+
promptRun.Status = promptRun.Success ? 'Completed' : 'Failed';
|
|
2055
|
+
if (!promptRun.Success) {
|
|
2056
|
+
if (!modelResult.success && modelResult.errorMessage) {
|
|
2057
|
+
promptRun.ErrorDetails = modelResult.errorMessage;
|
|
2058
|
+
}
|
|
2059
|
+
else if (parsedResult.validationResult?.Success === false) {
|
|
2060
|
+
promptRun.ErrorDetails = `Validation failed: ${parsedResult.validationResult.Errors?.map(e => e.Message).join(', ')}`;
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
1860
2063
|
promptRun.TokensPromptRollup = promptRun.TokensPrompt;
|
|
1861
2064
|
promptRun.TokensCompletionRollup = promptRun.TokensCompletion;
|
|
1862
2065
|
promptRun.TokensUsedRollup = promptRun.TokensUsed;
|