@memberjunction/ai-prompts 3.4.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +155 -3104
- package/dist/AIPromptRunner.d.ts +540 -2
- package/dist/AIPromptRunner.d.ts.map +1 -1
- package/dist/AIPromptRunner.js +1156 -232
- package/dist/AIPromptRunner.js.map +1 -1
- package/dist/ExecutionPlanner.d.ts +116 -1
- package/dist/ExecutionPlanner.d.ts.map +1 -1
- package/dist/ExecutionPlanner.js +171 -40
- package/dist/ExecutionPlanner.js.map +1 -1
- package/dist/ParallelExecution.d.ts +178 -1
- package/dist/ParallelExecution.d.ts.map +1 -1
- package/dist/ParallelExecution.js +1 -2
- package/dist/ParallelExecutionCoordinator.d.ts +182 -3
- package/dist/ParallelExecutionCoordinator.d.ts.map +1 -1
- package/dist/ParallelExecutionCoordinator.js +304 -79
- package/dist/ParallelExecutionCoordinator.js.map +1 -1
- package/dist/__tests__/AIPromptRunner.failover.test.d.ts +21 -0
- package/dist/__tests__/AIPromptRunner.failover.test.d.ts.map +1 -1
- package/dist/__tests__/AIPromptRunner.failover.test.js +92 -6
- package/dist/__tests__/AIPromptRunner.failover.test.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -19
- package/dist/index.js.map +1 -1
- package/package.json +21 -20
package/dist/AIPromptRunner.js
CHANGED
|
@@ -1,85 +1,77 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.LoadAIPromptRunner = exports.AIPromptRunner = void 0;
|
|
27
|
-
const ai_1 = require("@memberjunction/ai");
|
|
28
|
-
const ai_core_plus_1 = require("@memberjunction/ai-core-plus");
|
|
29
|
-
const core_1 = require("@memberjunction/core");
|
|
30
|
-
const global_1 = require("@memberjunction/global");
|
|
31
|
-
const credentials_1 = require("@memberjunction/credentials");
|
|
32
|
-
const templates_1 = require("@memberjunction/templates");
|
|
33
|
-
const ExecutionPlanner_1 = require("./ExecutionPlanner");
|
|
34
|
-
const ParallelExecutionCoordinator_1 = require("./ParallelExecutionCoordinator");
|
|
35
|
-
const aiengine_1 = require("@memberjunction/aiengine");
|
|
36
|
-
const ai_core_plus_2 = require("@memberjunction/ai-core-plus");
|
|
37
|
-
const JSON5 = __importStar(require("json5"));
|
|
38
|
-
class AIPromptRunner {
|
|
1
|
+
import { BaseLLM, ChatParams, ChatMessageRole, GetAIAPIKey, ErrorAnalyzer } from '@memberjunction/ai';
|
|
2
|
+
import { AIModelSelectionInfo } from '@memberjunction/ai-core-plus';
|
|
3
|
+
import { LogErrorEx, LogStatus, LogStatusEx, IsVerboseLoggingEnabled, Metadata } from '@memberjunction/core';
|
|
4
|
+
import { CleanJSON, MJGlobal, JSONValidator, ValidationResult, ValidationErrorInfo, ValidationErrorType } from '@memberjunction/global';
|
|
5
|
+
import { CredentialEngine } from '@memberjunction/credentials';
|
|
6
|
+
import { TemplateEngineServer } from '@memberjunction/templates';
|
|
7
|
+
import { ExecutionPlanner } from './ExecutionPlanner.js';
|
|
8
|
+
import { ParallelExecutionCoordinator } from './ParallelExecutionCoordinator.js';
|
|
9
|
+
import { AIEngine } from '@memberjunction/aiengine';
|
|
10
|
+
import { SystemPlaceholderManager } from '@memberjunction/ai-core-plus';
|
|
11
|
+
import * as JSON5 from 'json5';
|
|
12
|
+
export class AIPromptRunner {
|
|
39
13
|
constructor() {
|
|
40
|
-
this._metadata = new
|
|
41
|
-
this._templateEngine =
|
|
42
|
-
this._executionPlanner = new
|
|
43
|
-
this._parallelCoordinator = new
|
|
44
|
-
this._jsonValidator = new
|
|
45
|
-
}
|
|
14
|
+
this._metadata = new Metadata();
|
|
15
|
+
this._templateEngine = TemplateEngineServer.Instance;
|
|
16
|
+
this._executionPlanner = new ExecutionPlanner();
|
|
17
|
+
this._parallelCoordinator = new ParallelExecutionCoordinator();
|
|
18
|
+
this._jsonValidator = new JSONValidator();
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Performs robust validation of an API key
|
|
22
|
+
* @returns true if the API key is valid (not null, undefined, or empty/whitespace)
|
|
23
|
+
*/
|
|
46
24
|
isValidAPIKey(apiKey) {
|
|
47
25
|
if (apiKey === undefined || apiKey === null) {
|
|
48
26
|
return false;
|
|
49
27
|
}
|
|
28
|
+
// Check if it's just whitespace
|
|
50
29
|
const trimmed = apiKey.trim();
|
|
51
30
|
return trimmed.length > 0;
|
|
52
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* Internal logging helper that wraps LogStatusEx with verbose control
|
|
34
|
+
* @param message The message to log
|
|
35
|
+
* @param verboseOnly Whether this is a verbose-only message
|
|
36
|
+
* @param params Optional prompt parameters for custom verbose check
|
|
37
|
+
*/
|
|
53
38
|
logStatus(message, verboseOnly = false, params) {
|
|
54
39
|
if (verboseOnly) {
|
|
55
|
-
|
|
40
|
+
LogStatusEx({
|
|
56
41
|
message,
|
|
57
42
|
verboseOnly: true,
|
|
58
|
-
isVerboseEnabled: () => params?.verbose === true ||
|
|
43
|
+
isVerboseEnabled: () => params?.verbose === true || IsVerboseLoggingEnabled()
|
|
59
44
|
});
|
|
60
45
|
}
|
|
61
46
|
else {
|
|
62
|
-
|
|
47
|
+
LogStatus(message);
|
|
63
48
|
}
|
|
64
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* Helper method for enhanced error logging with metadata
|
|
52
|
+
*/
|
|
65
53
|
logError(error, options) {
|
|
66
54
|
let errorMessage = error instanceof Error ? error.message : error;
|
|
67
55
|
const errorObj = error instanceof Error ? error : undefined;
|
|
56
|
+
// Truncate extremely long error messages (like Groq's failed_generation JSON dumps)
|
|
57
|
+
// Only truncate if maxErrorLength is explicitly set
|
|
68
58
|
if (options?.maxErrorLength !== undefined && errorMessage.length > options.maxErrorLength) {
|
|
69
59
|
errorMessage = errorMessage.substring(0, options.maxErrorLength) + '... [truncated]';
|
|
70
60
|
}
|
|
71
61
|
const metadata = {
|
|
72
62
|
...options?.metadata
|
|
73
63
|
};
|
|
64
|
+
// Add prompt information if available
|
|
74
65
|
if (options?.prompt) {
|
|
75
66
|
metadata.promptId = options.prompt.ID;
|
|
76
67
|
metadata.promptName = options.prompt.Name;
|
|
77
68
|
}
|
|
69
|
+
// Add model information if available
|
|
78
70
|
if (options?.model) {
|
|
79
71
|
metadata.modelId = options.model.ID;
|
|
80
72
|
metadata.modelName = options.model.Name;
|
|
81
73
|
}
|
|
82
|
-
|
|
74
|
+
LogErrorEx({
|
|
83
75
|
message: errorMessage,
|
|
84
76
|
error: errorObj,
|
|
85
77
|
category: options?.category || 'AIPromptRunner',
|
|
@@ -87,46 +79,81 @@ class AIPromptRunner {
|
|
|
87
79
|
metadata: Object.keys(metadata).length > 0 ? metadata : undefined
|
|
88
80
|
});
|
|
89
81
|
}
|
|
82
|
+
/**
|
|
83
|
+
* Checks if a model vendor is configured as an inference provider
|
|
84
|
+
* @param modelVendor The model vendor to check
|
|
85
|
+
* @returns true if the vendor is an inference provider
|
|
86
|
+
*/
|
|
90
87
|
isInferenceProvider(modelVendor) {
|
|
91
|
-
|
|
88
|
+
// Find the inference provider type from cached vendor type definitions
|
|
89
|
+
const inferenceProviderType = AIEngine.Instance.VendorTypeDefinitions.find(vt => vt.Name === 'Inference Provider');
|
|
92
90
|
if (!inferenceProviderType) {
|
|
93
|
-
|
|
91
|
+
// Fallback to checking if it's not a model developer (should rarely happen)
|
|
92
|
+
const modelDeveloperType = AIEngine.Instance.VendorTypeDefinitions.find(vt => vt.Name === 'Model Developer');
|
|
94
93
|
return modelVendor.TypeID !== modelDeveloperType?.ID;
|
|
95
94
|
}
|
|
96
95
|
return modelVendor.TypeID === inferenceProviderType.ID;
|
|
97
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* Resolves credentials for AI model execution using a hierarchical resolution system.
|
|
99
|
+
*
|
|
100
|
+
* Resolution priority (highest to lowest):
|
|
101
|
+
* 1. Per-request override: params.credentialId
|
|
102
|
+
* 2. Prompt-Model specific: AIPromptModel.CredentialID
|
|
103
|
+
* 3. Model-Vendor specific: AIModelVendor.CredentialID
|
|
104
|
+
* 4. Vendor default: AIVendor.CredentialID
|
|
105
|
+
* 5. Legacy: params.apiKeys[] array
|
|
106
|
+
* 6. Legacy: AI_VENDOR_API_KEY__<DRIVER> environment variables
|
|
107
|
+
*
|
|
108
|
+
* IMPORTANT: When ANY credential ID is found (priorities 1-4), the system uses
|
|
109
|
+
* the Credentials path and ignores legacy methods (priorities 5-6).
|
|
110
|
+
*
|
|
111
|
+
* @param driverClass - The driver class name (e.g., 'OpenAILLM')
|
|
112
|
+
* @param promptId - The prompt ID for looking up AIPromptModel credentials
|
|
113
|
+
* @param modelId - The model ID for looking up AIPromptModel and AIModelVendor credentials
|
|
114
|
+
* @param vendorId - The vendor ID for looking up AIModelVendor and AIVendor credentials
|
|
115
|
+
* @param params - The prompt execution parameters containing contextUser and optional credentialId
|
|
116
|
+
* @returns The API key/configuration string to pass to the LLM constructor
|
|
117
|
+
*/
|
|
98
118
|
async resolveCredentialForExecution(driverClass, promptId, modelId, vendorId, params) {
|
|
99
|
-
const verbose = params.verbose === true ||
|
|
119
|
+
const verbose = params.verbose === true || IsVerboseLoggingEnabled();
|
|
120
|
+
// Priority 1: Per-request override - no failover, explicit choice
|
|
100
121
|
if (params.credentialId) {
|
|
101
122
|
return await this.resolveCredentialById(params.credentialId, 'per-request override', params, verbose);
|
|
102
123
|
}
|
|
103
|
-
|
|
124
|
+
// Ensure CredentialEngine is configured for binding lookups
|
|
125
|
+
await CredentialEngine.Instance.Config(false, params.contextUser);
|
|
126
|
+
// Priority 2: PromptModel bindings (most specific) - with failover
|
|
104
127
|
if (promptId && modelId) {
|
|
105
|
-
const promptModel =
|
|
128
|
+
const promptModel = AIEngine.Instance.PromptModels.find(pm => pm.PromptID === promptId && pm.ModelID === modelId);
|
|
106
129
|
if (promptModel) {
|
|
107
|
-
const bindings =
|
|
130
|
+
const bindings = AIEngine.Instance.GetCredentialBindingsForTarget('PromptModel', promptModel.ID);
|
|
108
131
|
const result = await this.tryCredentialBindingsWithFailover(bindings, 'AICredentialBinding(PromptModel)', params, verbose);
|
|
109
132
|
if (result)
|
|
110
133
|
return result;
|
|
111
134
|
}
|
|
112
135
|
}
|
|
136
|
+
// Priority 3: ModelVendor bindings - with failover
|
|
113
137
|
if (modelId && vendorId) {
|
|
114
|
-
const modelVendor =
|
|
138
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === modelId && mv.VendorID === vendorId && mv.Status === 'Active');
|
|
115
139
|
if (modelVendor) {
|
|
116
|
-
const bindings =
|
|
140
|
+
const bindings = AIEngine.Instance.GetCredentialBindingsForTarget('ModelVendor', modelVendor.ID);
|
|
117
141
|
const result = await this.tryCredentialBindingsWithFailover(bindings, 'AICredentialBinding(ModelVendor)', params, verbose);
|
|
118
142
|
if (result)
|
|
119
143
|
return result;
|
|
120
144
|
}
|
|
121
145
|
}
|
|
146
|
+
// Priority 4: Vendor bindings - with failover
|
|
122
147
|
if (vendorId) {
|
|
123
|
-
const bindings =
|
|
148
|
+
const bindings = AIEngine.Instance.GetCredentialBindingsForTarget('Vendor', vendorId);
|
|
124
149
|
const result = await this.tryCredentialBindingsWithFailover(bindings, 'AICredentialBinding(Vendor)', params, verbose);
|
|
125
150
|
if (result)
|
|
126
151
|
return result;
|
|
127
152
|
}
|
|
153
|
+
// Priority 5: Type-based default credential
|
|
154
|
+
// If the vendor declares a CredentialTypeID, try to find a default credential of that type
|
|
128
155
|
if (vendorId) {
|
|
129
|
-
const vendor =
|
|
156
|
+
const vendor = AIEngine.Instance.Vendors.find(v => v.ID === vendorId);
|
|
130
157
|
if (vendor?.CredentialTypeID) {
|
|
131
158
|
const defaultCredential = this.findDefaultCredentialByType(vendor.CredentialTypeID);
|
|
132
159
|
if (defaultCredential) {
|
|
@@ -136,31 +163,42 @@ class AIPromptRunner {
|
|
|
136
163
|
}
|
|
137
164
|
}
|
|
138
165
|
}
|
|
166
|
+
// No credential bindings found - fall back to legacy methods
|
|
139
167
|
if (verbose) {
|
|
140
168
|
this.logStatus(` Using legacy API key resolution for driver ${driverClass}`, true, params);
|
|
141
169
|
}
|
|
142
|
-
|
|
170
|
+
// Priority 6 & 7: Legacy apiKeys array and environment variables
|
|
171
|
+
return GetAIAPIKey(driverClass, params.apiKeys, verbose);
|
|
143
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* Attempts to resolve credentials from bindings with priority-based failover.
|
|
175
|
+
* Tries each binding in priority order until one succeeds.
|
|
176
|
+
*/
|
|
144
177
|
async tryCredentialBindingsWithFailover(bindings, source, params, verbose) {
|
|
145
178
|
if (bindings.length === 0)
|
|
146
179
|
return null;
|
|
147
180
|
for (let i = 0; i < bindings.length; i++) {
|
|
148
181
|
const binding = bindings[i];
|
|
149
|
-
const credential =
|
|
182
|
+
const credential = CredentialEngine.Instance.getCredentialById(binding.CredentialID);
|
|
150
183
|
if (!credential) {
|
|
151
184
|
if (verbose) {
|
|
152
185
|
this.logStatus(` ⚠️ Credential ${binding.CredentialID} not found (priority ${binding.Priority}), trying next...`, true, params);
|
|
153
186
|
}
|
|
154
187
|
continue;
|
|
155
188
|
}
|
|
156
|
-
const result = await this.tryResolveCredential(credential, `${source} priority ${binding.Priority}`, params, verbose, i < bindings.length - 1
|
|
189
|
+
const result = await this.tryResolveCredential(credential, `${source} priority ${binding.Priority}`, params, verbose, i < bindings.length - 1 // hasMoreBindings
|
|
190
|
+
);
|
|
157
191
|
if (result)
|
|
158
192
|
return result;
|
|
159
193
|
}
|
|
160
194
|
return null;
|
|
161
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* Attempts to resolve a single credential, returning null on failure for failover support.
|
|
198
|
+
*/
|
|
162
199
|
async tryResolveCredential(credential, source, params, verbose, hasMoreBindings = false) {
|
|
163
200
|
try {
|
|
201
|
+
// Check if credential is active and not expired
|
|
164
202
|
if (!credential.IsActive) {
|
|
165
203
|
if (verbose) {
|
|
166
204
|
this.logStatus(` ⚠️ Credential "${credential.Name}" is inactive, trying next...`, true, params);
|
|
@@ -173,7 +211,8 @@ class AIPromptRunner {
|
|
|
173
211
|
}
|
|
174
212
|
return null;
|
|
175
213
|
}
|
|
176
|
-
|
|
214
|
+
// Resolve the credential values
|
|
215
|
+
const resolved = await CredentialEngine.Instance.getCredential(credential.Name, {
|
|
177
216
|
credentialId: credential.ID,
|
|
178
217
|
contextUser: params.contextUser,
|
|
179
218
|
subsystem: 'AIPromptRunner'
|
|
@@ -185,12 +224,14 @@ class AIPromptRunner {
|
|
|
185
224
|
}
|
|
186
225
|
catch (error) {
|
|
187
226
|
if (hasMoreBindings) {
|
|
227
|
+
// More bindings to try - log warning and continue
|
|
188
228
|
if (verbose) {
|
|
189
229
|
this.logStatus(` ⚠️ Failed to resolve credential "${credential.Name}" from ${source}: ${error instanceof Error ? error.message : String(error)}, trying next...`, true, params);
|
|
190
230
|
}
|
|
191
231
|
return null;
|
|
192
232
|
}
|
|
193
233
|
else {
|
|
234
|
+
// No more bindings - log error but still return null for legacy fallback
|
|
194
235
|
this.logError(error instanceof Error ? error : new Error(String(error)), {
|
|
195
236
|
category: 'CredentialResolution',
|
|
196
237
|
severity: 'warning',
|
|
@@ -205,13 +246,17 @@ class AIPromptRunner {
|
|
|
205
246
|
}
|
|
206
247
|
}
|
|
207
248
|
}
|
|
249
|
+
/**
|
|
250
|
+
* Resolves a credential by its explicit ID (used for per-request override).
|
|
251
|
+
* This does not support failover since it's an explicit choice.
|
|
252
|
+
*/
|
|
208
253
|
async resolveCredentialById(credentialId, source, params, verbose) {
|
|
209
|
-
await
|
|
210
|
-
const credential =
|
|
254
|
+
await CredentialEngine.Instance.Config(false, params.contextUser);
|
|
255
|
+
const credential = CredentialEngine.Instance.getCredentialById(credentialId);
|
|
211
256
|
if (!credential) {
|
|
212
257
|
throw new Error(`Credential with ID ${credentialId} not found`);
|
|
213
258
|
}
|
|
214
|
-
const resolved = await
|
|
259
|
+
const resolved = await CredentialEngine.Instance.getCredential(credential.Name, {
|
|
215
260
|
credentialId,
|
|
216
261
|
contextUser: params.contextUser,
|
|
217
262
|
subsystem: 'AIPromptRunner'
|
|
@@ -221,36 +266,66 @@ class AIPromptRunner {
|
|
|
221
266
|
}
|
|
222
267
|
return JSON.stringify(resolved.values);
|
|
223
268
|
}
|
|
269
|
+
/**
|
|
270
|
+
* Finds a default credential matching a specific credential type.
|
|
271
|
+
*/
|
|
224
272
|
findDefaultCredentialByType(credentialTypeId) {
|
|
225
|
-
const credentials =
|
|
273
|
+
const credentials = CredentialEngine.Instance.Credentials;
|
|
226
274
|
return credentials.find(c => c.CredentialTypeID === credentialTypeId &&
|
|
227
275
|
c.IsDefault === true &&
|
|
228
276
|
c.IsActive === true &&
|
|
229
277
|
(!c.ExpiresAt || new Date(c.ExpiresAt) > new Date())) || null;
|
|
230
278
|
}
|
|
279
|
+
/**
|
|
280
|
+
* Checks if credentials are available for a given model-vendor combination.
|
|
281
|
+
* This is a pre-flight check used during model selection to determine which
|
|
282
|
+
* candidates have valid authentication configured.
|
|
283
|
+
*
|
|
284
|
+
* Checks the credential hierarchy:
|
|
285
|
+
* 1. Per-request override: params.credentialId
|
|
286
|
+
* 2. PromptModel bindings: AICredentialBinding WHERE BindingType='PromptModel'
|
|
287
|
+
* 3. ModelVendor bindings: AICredentialBinding WHERE BindingType='ModelVendor'
|
|
288
|
+
* 4. Vendor bindings: AICredentialBinding WHERE BindingType='Vendor'
|
|
289
|
+
* 5. Type-based default: Credential.IsDefault=1 matching AIVendor.CredentialTypeID
|
|
290
|
+
* 6. Legacy: params.apiKeys[] array
|
|
291
|
+
* 7. Legacy: AI_VENDOR_API_KEY__<DRIVER> environment variables
|
|
292
|
+
*
|
|
293
|
+
* @param driverClass - The driver class name (e.g., 'OpenAILLM')
|
|
294
|
+
* @param promptId - The prompt ID for looking up AIPromptModel bindings
|
|
295
|
+
* @param modelId - The model ID for looking up AIPromptModel and AIModelVendor bindings
|
|
296
|
+
* @param vendorId - The vendor ID for looking up AIModelVendor and AIVendor bindings
|
|
297
|
+
* @param params - The prompt execution parameters
|
|
298
|
+
* @returns true if credentials are available, false otherwise
|
|
299
|
+
*/
|
|
231
300
|
hasCredentialsAvailable(driverClass, promptId, modelId, vendorId, params) {
|
|
301
|
+
// Priority 1: Per-request override
|
|
232
302
|
if (params?.credentialId) {
|
|
303
|
+
// Assume valid if credential ID is provided - will be validated at execution time
|
|
233
304
|
return true;
|
|
234
305
|
}
|
|
306
|
+
// Priority 2: PromptModel bindings
|
|
235
307
|
if (promptId && modelId) {
|
|
236
|
-
const promptModel =
|
|
237
|
-
if (promptModel &&
|
|
308
|
+
const promptModel = AIEngine.Instance.PromptModels.find(pm => pm.PromptID === promptId && pm.ModelID === modelId);
|
|
309
|
+
if (promptModel && AIEngine.Instance.HasCredentialBindings('PromptModel', promptModel.ID)) {
|
|
238
310
|
return true;
|
|
239
311
|
}
|
|
240
312
|
}
|
|
313
|
+
// Priority 3: ModelVendor bindings
|
|
241
314
|
if (modelId && vendorId) {
|
|
242
|
-
const modelVendor =
|
|
243
|
-
if (modelVendor &&
|
|
315
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === modelId && mv.VendorID === vendorId && mv.Status === 'Active');
|
|
316
|
+
if (modelVendor && AIEngine.Instance.HasCredentialBindings('ModelVendor', modelVendor.ID)) {
|
|
244
317
|
return true;
|
|
245
318
|
}
|
|
246
319
|
}
|
|
320
|
+
// Priority 4: Vendor bindings
|
|
247
321
|
if (vendorId) {
|
|
248
|
-
if (
|
|
322
|
+
if (AIEngine.Instance.HasCredentialBindings('Vendor', vendorId)) {
|
|
249
323
|
return true;
|
|
250
324
|
}
|
|
251
325
|
}
|
|
326
|
+
// Priority 5: Type-based default credential
|
|
252
327
|
if (vendorId) {
|
|
253
|
-
const vendor =
|
|
328
|
+
const vendor = AIEngine.Instance.Vendors.find(v => v.ID === vendorId);
|
|
254
329
|
if (vendor?.CredentialTypeID) {
|
|
255
330
|
const defaultCredential = this.findDefaultCredentialByType(vendor.CredentialTypeID);
|
|
256
331
|
if (defaultCredential) {
|
|
@@ -258,12 +333,40 @@ class AIPromptRunner {
|
|
|
258
333
|
}
|
|
259
334
|
}
|
|
260
335
|
}
|
|
261
|
-
|
|
336
|
+
// Priority 6 & 7: Legacy methods - check if API key is available
|
|
337
|
+
const apiKey = GetAIAPIKey(driverClass, params?.apiKeys, params?.verbose);
|
|
262
338
|
return this.isValidAPIKey(apiKey);
|
|
263
339
|
}
|
|
340
|
+
/**
|
|
341
|
+
* Executes an AI prompt with full support for templates, model selection, and validation.
|
|
342
|
+
*
|
|
343
|
+
* @param params Parameters for prompt execution
|
|
344
|
+
* @returns Promise<AIPromptRunResult<T>> The execution result with tracking information
|
|
345
|
+
*
|
|
346
|
+
* @example
|
|
347
|
+
* ```typescript
|
|
348
|
+
* // Execute with specific result type
|
|
349
|
+
* interface AnalysisResult {
|
|
350
|
+
* sentiment: string;
|
|
351
|
+
* score: number;
|
|
352
|
+
* keywords: string[];
|
|
353
|
+
* }
|
|
354
|
+
*
|
|
355
|
+
* const result = await promptRunner.ExecutePrompt<AnalysisResult>({
|
|
356
|
+
* prompt: sentimentPrompt,
|
|
357
|
+
* data: { text: "Customer feedback text" }
|
|
358
|
+
* });
|
|
359
|
+
*
|
|
360
|
+
* if (result.success && result.result) {
|
|
361
|
+
* // result.result is typed as AnalysisResult
|
|
362
|
+
* console.log(`Sentiment: ${result.result.sentiment}, Score: ${result.result.score}`);
|
|
363
|
+
* }
|
|
364
|
+
* ```
|
|
365
|
+
*/
|
|
264
366
|
async ExecutePrompt(params) {
|
|
265
367
|
const startTime = new Date();
|
|
266
368
|
const promptRun = null;
|
|
369
|
+
// Check for cancellation at the start
|
|
267
370
|
if (params.cancellationToken?.aborted) {
|
|
268
371
|
const result = {
|
|
269
372
|
success: false,
|
|
@@ -279,6 +382,7 @@ class AIPromptRunner {
|
|
|
279
382
|
return result;
|
|
280
383
|
}
|
|
281
384
|
try {
|
|
385
|
+
// Use the prompt entity directly from params
|
|
282
386
|
const prompt = params.prompt;
|
|
283
387
|
if (!prompt) {
|
|
284
388
|
throw new Error(`Prompt entity is required`);
|
|
@@ -287,42 +391,60 @@ class AIPromptRunner {
|
|
|
287
391
|
throw new Error(`Prompt ${prompt.Name} is not active (Status: ${prompt.Status})`);
|
|
288
392
|
}
|
|
289
393
|
let renderedPromptText = '';
|
|
394
|
+
// For hierarchical prompts, we need to create the parent prompt run first to get its ID
|
|
290
395
|
let parentPromptRun;
|
|
291
396
|
let selectedModel;
|
|
292
397
|
let childTemplateRenderingResult;
|
|
293
398
|
let modelSelectionInfo;
|
|
399
|
+
// Handle different prompt execution modes
|
|
294
400
|
if (params.childPrompts && params.childPrompts.length > 0) {
|
|
401
|
+
// Hierarchical template composition mode - render child templates first, then compose
|
|
402
|
+
//this.logStatus(`🌳 Composing prompt with ${params.childPrompts.length} child templates in hierarchical mode`, true, params);
|
|
403
|
+
// Determine which prompt to use for model selection
|
|
295
404
|
let modelSelectionPrompt = prompt;
|
|
296
405
|
if (params.modelSelectionPrompt) {
|
|
297
406
|
modelSelectionPrompt = params.modelSelectionPrompt;
|
|
407
|
+
//this.logStatus(`🎯 Using prompt "${modelSelectionPrompt.Name}" for model selection instead of parent prompt`, true, params);
|
|
298
408
|
}
|
|
409
|
+
// Select model using the appropriate prompt
|
|
299
410
|
const modelResult = await this.selectModel(modelSelectionPrompt, params.override?.modelId, params.contextUser, params.configurationId, params.override?.vendorId, params);
|
|
300
411
|
selectedModel = modelResult.model;
|
|
301
412
|
modelSelectionInfo = modelResult.selectionInfo;
|
|
302
413
|
if (!selectedModel) {
|
|
303
414
|
throw new Error(`No suitable model found for prompt ${modelSelectionPrompt.Name}`);
|
|
304
415
|
}
|
|
416
|
+
// Check if we have a system prompt override
|
|
305
417
|
if (params.systemPromptOverride) {
|
|
418
|
+
// Use the override instead of rendering child templates and parent template
|
|
306
419
|
renderedPromptText = params.systemPromptOverride;
|
|
307
420
|
this.logStatus(` Using system prompt override for prompt "${prompt.Name}" (bypassing hierarchical template rendering)`, true, params);
|
|
308
421
|
}
|
|
309
422
|
else {
|
|
423
|
+
// Render all child prompt templates recursively
|
|
310
424
|
childTemplateRenderingResult = await this.renderChildPromptTemplates(params.childPrompts, params, params.cancellationToken);
|
|
425
|
+
// Render the parent prompt with child templates embedded
|
|
311
426
|
renderedPromptText = await this.renderPromptWithChildTemplates(prompt, params, childTemplateRenderingResult.renderedTemplates);
|
|
312
427
|
}
|
|
428
|
+
// Create parent prompt run for the final composed prompt execution
|
|
313
429
|
parentPromptRun = await this.createPromptRun(prompt, selectedModel, params, renderedPromptText, startTime, params.override?.vendorId, modelSelectionInfo);
|
|
314
430
|
}
|
|
315
431
|
else if (prompt.TemplateID && (!params.conversationMessages || params.templateMessageRole !== 'none')) {
|
|
432
|
+
// Check if we have a system prompt override
|
|
316
433
|
if (params.systemPromptOverride) {
|
|
434
|
+
// Use the override instead of rendering the template
|
|
317
435
|
renderedPromptText = params.systemPromptOverride;
|
|
318
436
|
this.logStatus(` Using system prompt override for prompt "${prompt.Name}" (bypassing template rendering)`, true, params);
|
|
319
437
|
}
|
|
320
438
|
else {
|
|
439
|
+
// Regular template rendering mode
|
|
440
|
+
// Initialize template engine
|
|
321
441
|
await this._templateEngine.Config(false, params.contextUser);
|
|
442
|
+
// Load the template for the prompt
|
|
322
443
|
const template = await this.loadTemplate(prompt.TemplateID, params.contextUser);
|
|
323
444
|
if (!template) {
|
|
324
445
|
throw new Error(`Template with ID ${prompt.TemplateID} not found for prompt ${prompt.Name}`);
|
|
325
446
|
}
|
|
447
|
+
// Render the template with full params context
|
|
326
448
|
const renderedPrompt = await this.renderPromptTemplate(template, params);
|
|
327
449
|
if (!renderedPrompt.Success) {
|
|
328
450
|
throw new Error(`Failed to render template for prompt ${prompt.Name}: ${renderedPrompt.Message}`);
|
|
@@ -330,9 +452,11 @@ class AIPromptRunner {
|
|
|
330
452
|
renderedPromptText = renderedPrompt.Output;
|
|
331
453
|
}
|
|
332
454
|
}
|
|
455
|
+
// Check for cancellation after template rendering
|
|
333
456
|
if (params.cancellationToken?.aborted) {
|
|
334
457
|
throw new Error('Prompt execution was cancelled during template rendering');
|
|
335
458
|
}
|
|
459
|
+
// If no model was selected yet (no template case), select one now
|
|
336
460
|
if (!selectedModel) {
|
|
337
461
|
let modelSelectionPrompt = prompt;
|
|
338
462
|
if (params.modelSelectionPrompt) {
|
|
@@ -346,14 +470,20 @@ class AIPromptRunner {
|
|
|
346
470
|
throw new Error(`No suitable model found for prompt ${modelSelectionPrompt.Name}`);
|
|
347
471
|
}
|
|
348
472
|
}
|
|
473
|
+
// Check if we need parallel execution based on ParallelizationMode
|
|
349
474
|
const shouldUseParallelExecution = prompt.ParallelizationMode && prompt.ParallelizationMode !== 'None';
|
|
350
475
|
let result;
|
|
351
476
|
if (shouldUseParallelExecution) {
|
|
477
|
+
// Use parallel execution path
|
|
352
478
|
result = await this.executePromptInParallel(prompt, renderedPromptText, params, startTime, parentPromptRun, selectedModel, modelSelectionInfo);
|
|
353
479
|
}
|
|
354
480
|
else {
|
|
481
|
+
// Use traditional single execution path
|
|
355
482
|
result = await this.executeSinglePrompt(prompt, renderedPromptText, params, startTime, parentPromptRun, selectedModel, modelSelectionInfo);
|
|
356
483
|
}
|
|
484
|
+
// Note: With template composition, we only execute once so no rollup calculations needed
|
|
485
|
+
// The final composed prompt is executed as a single operation
|
|
486
|
+
// Model selection info is now included in the result from both execution methods
|
|
357
487
|
return result;
|
|
358
488
|
}
|
|
359
489
|
catch (error) {
|
|
@@ -367,10 +497,12 @@ class AIPromptRunner {
|
|
|
367
497
|
});
|
|
368
498
|
const endTime = new Date();
|
|
369
499
|
const executionTimeMS = endTime.getTime() - startTime.getTime();
|
|
500
|
+
// Update prompt run with error if it was created
|
|
370
501
|
if (promptRun) {
|
|
371
502
|
promptRun.CompletedAt = endTime;
|
|
372
503
|
promptRun.ExecutionTimeMS = executionTimeMS;
|
|
373
504
|
promptRun.Result = `ERROR: ${error.message}`;
|
|
505
|
+
// Set Status and Cancelled based on error type
|
|
374
506
|
if (error.message.includes('cancelled')) {
|
|
375
507
|
promptRun.Status = 'Cancelled';
|
|
376
508
|
promptRun.Cancelled = true;
|
|
@@ -404,10 +536,21 @@ class AIPromptRunner {
|
|
|
404
536
|
return errorResult;
|
|
405
537
|
}
|
|
406
538
|
}
|
|
539
|
+
/**
|
|
540
|
+
* Executes a single prompt (non-parallel) using traditional model selection.
|
|
541
|
+
*
|
|
542
|
+
* @param prompt - The AI prompt to execute
|
|
543
|
+
* @param renderedPromptText - The rendered prompt text
|
|
544
|
+
* @param params - Original execution parameters
|
|
545
|
+
* @param startTime - Execution start time
|
|
546
|
+
* @returns Promise<AIPromptRunResult<T>> - The execution result
|
|
547
|
+
*/
|
|
407
548
|
async executeSinglePrompt(prompt, renderedPromptText, params, startTime, existingPromptRun, existingModel, existingModelSelectionInfo) {
|
|
549
|
+
// Check for cancellation before model selection
|
|
408
550
|
if (params.cancellationToken?.aborted) {
|
|
409
551
|
throw new Error('Prompt execution was cancelled before model selection');
|
|
410
552
|
}
|
|
553
|
+
// Use existing model if provided (hierarchical case) or select one
|
|
411
554
|
let selectedModel = existingModel;
|
|
412
555
|
let modelSelectionInfo = existingModelSelectionInfo;
|
|
413
556
|
let vendorDriverClass;
|
|
@@ -416,18 +559,21 @@ class AIPromptRunner {
|
|
|
416
559
|
let modelEffortLevel;
|
|
417
560
|
let allCandidates = [];
|
|
418
561
|
if (modelSelectionInfo) {
|
|
562
|
+
// we received model selection info, need to lookup vendor driver class and api name from there
|
|
419
563
|
const vendorID = modelSelectionInfo.vendorSelected?.ID;
|
|
420
564
|
const modelID = modelSelectionInfo.modelSelected.ID;
|
|
421
|
-
const modelVendor =
|
|
565
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find(mv => mv.VendorID === vendorID &&
|
|
422
566
|
mv.ModelID === modelID);
|
|
423
567
|
if (modelVendor) {
|
|
424
568
|
vendorDriverClass = modelVendor.DriverClass;
|
|
425
569
|
vendorApiName = modelVendor.APIName;
|
|
426
570
|
vendorSupportsEffortLevel = modelVendor.SupportsEffortLevel;
|
|
427
571
|
}
|
|
572
|
+
// Extract valid candidates from selection info for retry logic
|
|
428
573
|
allCandidates = this.buildCandidatesFromSelectionInfo(modelSelectionInfo);
|
|
429
574
|
}
|
|
430
575
|
if (!selectedModel) {
|
|
576
|
+
// Determine which prompt to use for model selection
|
|
431
577
|
let modelSelectionPrompt = prompt;
|
|
432
578
|
if (params.modelSelectionPrompt) {
|
|
433
579
|
modelSelectionPrompt = params.modelSelectionPrompt;
|
|
@@ -445,34 +591,46 @@ class AIPromptRunner {
|
|
|
445
591
|
throw new Error(`No suitable model found for prompt ${modelSelectionPrompt.Name}`);
|
|
446
592
|
}
|
|
447
593
|
}
|
|
594
|
+
// Check for cancellation after model selection
|
|
448
595
|
if (params.cancellationToken?.aborted) {
|
|
449
596
|
throw new Error('Prompt execution was cancelled after model selection');
|
|
450
597
|
}
|
|
598
|
+
// Use existing prompt run if provided (hierarchical case) or create new one
|
|
451
599
|
const promptRun = existingPromptRun || await this.createPromptRun(prompt, selectedModel, params, renderedPromptText, startTime, params.override?.vendorId, modelSelectionInfo);
|
|
600
|
+
// Check for cancellation before model execution
|
|
452
601
|
if (params.cancellationToken?.aborted) {
|
|
453
602
|
throw new Error('Prompt execution was cancelled before model execution');
|
|
454
603
|
}
|
|
455
|
-
|
|
604
|
+
// Execute with retry logic for validation failures
|
|
605
|
+
const { modelResult, parsedResult, validationAttempts, cumulativeTokens } = await this.executeWithValidationRetries(selectedModel, renderedPromptText, prompt, params, promptRun, allCandidates, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel // Pass model-specific effort level
|
|
606
|
+
);
|
|
607
|
+
// Calculate execution metrics
|
|
456
608
|
const endTime = new Date();
|
|
457
609
|
const executionTimeMS = endTime.getTime() - startTime.getTime();
|
|
610
|
+
// Update the prompt run with results including validation attempts and cumulative tokens
|
|
458
611
|
await this.updatePromptRun(promptRun, prompt, modelResult, parsedResult, endTime, executionTimeMS, validationAttempts, cumulativeTokens);
|
|
459
612
|
const chatResult = modelResult;
|
|
460
613
|
const usage = chatResult.data?.usage;
|
|
614
|
+
// CRITICAL: Populate errorMessage field when execution fails
|
|
615
|
+
// This ensures errors are properly propagated to BaseAgent and visible in AgentRunStep logs
|
|
461
616
|
let errorMessage;
|
|
462
617
|
if (!chatResult.success) {
|
|
618
|
+
// Model execution failed
|
|
463
619
|
errorMessage = chatResult.errorMessage;
|
|
464
620
|
}
|
|
465
621
|
else if (parsedResult.validationResult?.Success === false) {
|
|
622
|
+
// Validation failed (Warn or Strict mode)
|
|
466
623
|
errorMessage = `Validation failed: ${parsedResult.validationResult.Errors?.map(e => e.Message).join('; ')}`;
|
|
467
624
|
}
|
|
468
625
|
return {
|
|
469
626
|
success: chatResult.success,
|
|
470
627
|
rawResult: chatResult.data?.choices?.[0]?.message?.content,
|
|
471
628
|
result: parsedResult?.result ? parsedResult.result : parsedResult,
|
|
472
|
-
errorMessage,
|
|
629
|
+
errorMessage, // Include error message for proper error propagation
|
|
473
630
|
chatResult,
|
|
474
631
|
promptRun,
|
|
475
632
|
executionTimeMS,
|
|
633
|
+
// Use cumulative tokens if retries occurred, otherwise use single attempt tokens
|
|
476
634
|
promptTokens: cumulativeTokens.promptTokens || usage?.promptTokens,
|
|
477
635
|
completionTokens: cumulativeTokens.completionTokens || usage?.completionTokens,
|
|
478
636
|
tokensUsed: (cumulativeTokens.promptTokens + cumulativeTokens.completionTokens) || ((usage?.promptTokens || 0) + (usage?.completionTokens || 0)),
|
|
@@ -481,21 +639,35 @@ class AIPromptRunner {
|
|
|
481
639
|
validationResult: parsedResult.validationResult,
|
|
482
640
|
validationAttempts,
|
|
483
641
|
combinedTokensUsed: (cumulativeTokens.promptTokens + cumulativeTokens.completionTokens) || ((usage?.promptTokens || 0) + (usage?.completionTokens || 0)),
|
|
484
|
-
modelSelectionInfo
|
|
642
|
+
modelSelectionInfo // Include model selection info if available
|
|
485
643
|
};
|
|
486
644
|
}
|
|
645
|
+
/**
|
|
646
|
+
* Executes a prompt using parallel execution with multiple models/tasks.
|
|
647
|
+
*
|
|
648
|
+
* @param prompt - The AI prompt to execute
|
|
649
|
+
* @param renderedPromptText - The rendered prompt text
|
|
650
|
+
* @param params - Original execution parameters
|
|
651
|
+
* @param startTime - Execution start time
|
|
652
|
+
* @returns Promise<AIPromptRunResult<T>> - The aggregated execution result
|
|
653
|
+
*/
|
|
487
654
|
async executePromptInParallel(prompt, renderedPromptText, params, startTime, existingPromptRun, existingModel, existingModelSelectionInfo) {
|
|
655
|
+
// Check for cancellation before starting parallel execution
|
|
488
656
|
if (params.cancellationToken?.aborted) {
|
|
489
657
|
throw new Error('Parallel execution was cancelled before starting');
|
|
490
658
|
}
|
|
491
|
-
|
|
659
|
+
// Load AI Engine to get models and prompt models
|
|
660
|
+
await AIEngine.Instance.Config(false, params.contextUser);
|
|
492
661
|
let executionTasks;
|
|
662
|
+
// If a model is already selected (from hierarchical template composition),
|
|
663
|
+
// create a single task with that model instead of using the planner
|
|
493
664
|
if (existingModel) {
|
|
665
|
+
// Create a single execution task with the pre-selected model
|
|
494
666
|
executionTasks = [{
|
|
495
667
|
taskId: 'pre-selected',
|
|
496
668
|
model: existingModel,
|
|
497
|
-
vendorDriverClass: undefined,
|
|
498
|
-
vendorApiName: existingModel.Vendor,
|
|
669
|
+
vendorDriverClass: undefined, // Would need to look up vendor entity for this
|
|
670
|
+
vendorApiName: existingModel.Vendor, // Vendor is already the name string
|
|
499
671
|
messages: params.conversationMessages || [],
|
|
500
672
|
promptText: renderedPromptText,
|
|
501
673
|
templateMessageRole: params.templateMessageRole || 'system',
|
|
@@ -504,31 +676,39 @@ class AIPromptRunner {
|
|
|
504
676
|
this.logStatus(` Using pre-selected model "${existingModel.Name}" for parallel execution`, true, params);
|
|
505
677
|
}
|
|
506
678
|
else {
|
|
679
|
+
// Normal parallel execution path - let the planner decide
|
|
680
|
+
// Determine which prompt to use for model selection
|
|
507
681
|
let modelSelectionPrompt = prompt;
|
|
508
682
|
if (params.modelSelectionPrompt) {
|
|
509
683
|
modelSelectionPrompt = params.modelSelectionPrompt;
|
|
510
684
|
this.logStatus(` Using prompt "${modelSelectionPrompt.Name}" for model selection in parallel execution`, true, params);
|
|
511
685
|
}
|
|
512
|
-
|
|
686
|
+
// Get prompt-specific model associations using the model selection prompt
|
|
687
|
+
const promptModels = AIEngine.Instance.PromptModels.filter((pm) => pm.PromptID === modelSelectionPrompt.ID &&
|
|
513
688
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
514
689
|
(!params.configurationId || !pm.ConfigurationID || pm.ConfigurationID === params.configurationId));
|
|
515
|
-
|
|
690
|
+
// Create execution plan using the modelSelectionPrompt for model configurations
|
|
691
|
+
executionTasks = this._executionPlanner.createExecutionPlan(modelSelectionPrompt, promptModels, AIEngine.Instance.Models, renderedPromptText, params.contextUser, params.configurationId, params.conversationMessages, params.templateMessageRole || 'system');
|
|
516
692
|
}
|
|
517
693
|
if (executionTasks.length === 0) {
|
|
518
694
|
throw new Error(`No execution tasks created for parallel execution of prompt ${prompt.Name}`);
|
|
519
695
|
}
|
|
696
|
+
// Check for cancellation before executing tasks
|
|
520
697
|
if (params.cancellationToken?.aborted) {
|
|
521
698
|
throw new Error('Parallel execution was cancelled before task execution');
|
|
522
699
|
}
|
|
700
|
+
// Execute tasks in parallel
|
|
523
701
|
const parallelResult = await this._parallelCoordinator.executeTasksInParallel(params, executionTasks, undefined, undefined, params.cancellationToken, undefined, params.agentRunId);
|
|
524
702
|
if (!parallelResult.success) {
|
|
525
703
|
throw new Error(`Parallel execution failed: ${parallelResult.errors.join(', ')}`);
|
|
526
704
|
}
|
|
705
|
+
// Select best result if multiple successful results
|
|
527
706
|
const successfulResults = parallelResult.taskResults.filter((r) => r.success);
|
|
528
707
|
if (successfulResults.length === 0) {
|
|
529
708
|
throw new Error(`No successful results from parallel execution`);
|
|
530
709
|
}
|
|
531
|
-
let selectedResult = successfulResults[0];
|
|
710
|
+
let selectedResult = successfulResults[0]; // Default to first
|
|
711
|
+
// Use result selector if configured
|
|
532
712
|
if (successfulResults.length > 1 && prompt.ResultSelectorPromptID) {
|
|
533
713
|
const selectionConfig = {
|
|
534
714
|
method: 'PromptSelector',
|
|
@@ -539,6 +719,7 @@ class AIPromptRunner {
|
|
|
539
719
|
selectedResult = aiSelectedResult;
|
|
540
720
|
}
|
|
541
721
|
}
|
|
722
|
+
// Calculate total tokens and costs from all parallel executions
|
|
542
723
|
let totalPromptTokens = 0;
|
|
543
724
|
let totalCompletionTokens = 0;
|
|
544
725
|
let totalCost = 0;
|
|
@@ -554,12 +735,16 @@ class AIPromptRunner {
|
|
|
554
735
|
}
|
|
555
736
|
}
|
|
556
737
|
}
|
|
738
|
+
// Use existing prompt run if provided (hierarchical case) or create new one
|
|
739
|
+
// Use the model selection info if provided (from hierarchical execution)
|
|
557
740
|
const consolidatedPromptRun = existingPromptRun || await this.createPromptRun(prompt, selectedResult.task.model, params, renderedPromptText, startTime, params.override?.vendorId, existingModelSelectionInfo);
|
|
741
|
+
// Update with parallel execution metadata
|
|
558
742
|
const endTime = new Date();
|
|
559
743
|
consolidatedPromptRun.CompletedAt = endTime;
|
|
560
744
|
consolidatedPromptRun.ExecutionTimeMS = parallelResult.totalExecutionTimeMS;
|
|
561
745
|
consolidatedPromptRun.Result = selectedResult.rawResult || '';
|
|
562
746
|
consolidatedPromptRun.TokensUsed = parallelResult.totalTokensUsed;
|
|
747
|
+
// Extract token and cost info from selected result
|
|
563
748
|
const selectedResultUsage = selectedResult.modelResult?.data?.usage;
|
|
564
749
|
if (selectedResultUsage) {
|
|
565
750
|
consolidatedPromptRun.TokensPrompt = selectedResultUsage.promptTokens;
|
|
@@ -571,6 +756,7 @@ class AIPromptRunner {
|
|
|
571
756
|
consolidatedPromptRun.CostCurrency = selectedResultUsage.costCurrency;
|
|
572
757
|
}
|
|
573
758
|
}
|
|
759
|
+
// Add parallel execution metadata to Messages field
|
|
574
760
|
const parallelMetadata = {
|
|
575
761
|
parallelizationMode: prompt.ParallelizationMode,
|
|
576
762
|
totalTasks: executionTasks.length,
|
|
@@ -587,14 +773,16 @@ class AIPromptRunner {
|
|
|
587
773
|
messages: params.conversationMessages || [],
|
|
588
774
|
});
|
|
589
775
|
}
|
|
776
|
+
// For parallel execution, set rollup fields to match totals (no child execution to roll up)
|
|
590
777
|
consolidatedPromptRun.TokensPromptRollup = totalPromptTokens;
|
|
591
778
|
consolidatedPromptRun.TokensCompletionRollup = totalCompletionTokens;
|
|
592
779
|
consolidatedPromptRun.TokensUsedRollup = totalPromptTokens + totalCompletionTokens;
|
|
593
780
|
if (hasCost) {
|
|
594
781
|
consolidatedPromptRun.TotalCost = totalCost;
|
|
595
782
|
}
|
|
783
|
+
// Set Status and WasSelectedResult for parallel execution
|
|
596
784
|
consolidatedPromptRun.Status = parallelResult.successCount > 0 ? 'Completed' : 'Failed';
|
|
597
|
-
consolidatedPromptRun.WasSelectedResult = true;
|
|
785
|
+
consolidatedPromptRun.WasSelectedResult = true; // This is the consolidated result chosen by judge
|
|
598
786
|
const saveResult = await consolidatedPromptRun.Save();
|
|
599
787
|
if (!saveResult) {
|
|
600
788
|
this.logError(`Failed to save consolidated AIPromptRun: ${consolidatedPromptRun.LatestResult?.CompleteMessage || 'Unknown error'}`, {
|
|
@@ -607,7 +795,9 @@ class AIPromptRunner {
|
|
|
607
795
|
maxErrorLength: params.maxErrorLength
|
|
608
796
|
});
|
|
609
797
|
}
|
|
798
|
+
// Create additional results from all other successful results (excluding the best one)
|
|
610
799
|
const additionalResults = [];
|
|
800
|
+
// Sort successful results by ranking (if available) or keep original order
|
|
611
801
|
const sortedResults = successfulResults.sort((a, b) => {
|
|
612
802
|
if (a.ranking && b.ranking) {
|
|
613
803
|
return a.ranking - b.ranking;
|
|
@@ -616,6 +806,7 @@ class AIPromptRunner {
|
|
|
616
806
|
});
|
|
617
807
|
for (const result of sortedResults) {
|
|
618
808
|
if (result.task.taskId !== selectedResult.task.taskId) {
|
|
809
|
+
// Parse and validate this result
|
|
619
810
|
const { result: parsedResultData, validationResult } = await this.parseAndValidateResultEnhanced(result.modelResult, prompt, params.skipValidation, params.cleanValidationSyntax, consolidatedPromptRun, params);
|
|
620
811
|
const parsedResult = { result: parsedResultData, validationResult };
|
|
621
812
|
const resultUsage = result.modelResult?.data?.usage;
|
|
@@ -636,13 +827,14 @@ class AIPromptRunner {
|
|
|
636
827
|
modelInfo: {
|
|
637
828
|
modelId: result.task.model.ID,
|
|
638
829
|
modelName: result.task.model.Name,
|
|
639
|
-
vendorId: undefined,
|
|
830
|
+
vendorId: undefined, // VendorID not directly available on AIModel
|
|
640
831
|
vendorName: result.task.model.Vendor,
|
|
641
832
|
},
|
|
642
833
|
combinedTokensUsed: (resultUsage?.promptTokens || 0) + (resultUsage?.completionTokens || 0)
|
|
643
834
|
});
|
|
644
835
|
}
|
|
645
836
|
}
|
|
837
|
+
// Parse and validate the selected result
|
|
646
838
|
const { result: selectedResultData, validationResult: selectedValidationResult } = await this.parseAndValidateResultEnhanced(selectedResult.modelResult, prompt, params.skipValidation, params.cleanValidationSyntax, consolidatedPromptRun, params);
|
|
647
839
|
const selectedParsedResult = { result: selectedResultData, validationResult: selectedValidationResult };
|
|
648
840
|
const selectedUsage = selectedResult.modelResult?.data?.usage;
|
|
@@ -658,6 +850,7 @@ class AIPromptRunner {
|
|
|
658
850
|
tokensUsed: (selectedUsage?.promptTokens || 0) + (selectedUsage?.completionTokens || 0),
|
|
659
851
|
cost: selectedUsage?.cost,
|
|
660
852
|
costCurrency: selectedUsage?.costCurrency,
|
|
853
|
+
// Combined totals for parallel execution
|
|
661
854
|
combinedPromptTokens: totalPromptTokens,
|
|
662
855
|
combinedCompletionTokens: totalCompletionTokens,
|
|
663
856
|
combinedTokensUsed: totalPromptTokens + totalCompletionTokens,
|
|
@@ -669,15 +862,19 @@ class AIPromptRunner {
|
|
|
669
862
|
modelInfo: {
|
|
670
863
|
modelId: selectedResult.task.model.ID,
|
|
671
864
|
modelName: selectedResult.task.model.Name,
|
|
672
|
-
vendorId: existingModelSelectionInfo?.vendorSelected?.ID,
|
|
865
|
+
vendorId: existingModelSelectionInfo?.vendorSelected?.ID, // VendorID not directly available on AIModel
|
|
673
866
|
vendorName: selectedResult.task.model.Vendor,
|
|
674
867
|
},
|
|
675
868
|
judgeMetadata: selectedResult.judgeMetadata,
|
|
676
|
-
modelSelectionInfo: existingModelSelectionInfo,
|
|
869
|
+
modelSelectionInfo: existingModelSelectionInfo, // Include model selection info if provided
|
|
677
870
|
};
|
|
678
871
|
}
|
|
872
|
+
/**
|
|
873
|
+
* Loads a template entity by ID
|
|
874
|
+
*/
|
|
679
875
|
async loadTemplate(templateId, _contextUser) {
|
|
680
876
|
try {
|
|
877
|
+
// Use the template engine to find the template
|
|
681
878
|
const template = this._templateEngine.Templates.find((t) => t.ID === templateId);
|
|
682
879
|
return template || null;
|
|
683
880
|
}
|
|
@@ -692,41 +889,61 @@ class AIPromptRunner {
|
|
|
692
889
|
return null;
|
|
693
890
|
}
|
|
694
891
|
}
|
|
892
|
+
/**
|
|
893
|
+
* Renders child prompt templates in a depth-first manner, composing them into a final template.
|
|
894
|
+
*
|
|
895
|
+
* @param childPrompts - Array of child prompts to render templates for
|
|
896
|
+
* @param params - Original execution parameters for context
|
|
897
|
+
* @param cancellationToken - Cancellation token for aborting rendering
|
|
898
|
+
* @returns Promise with rendered templates map
|
|
899
|
+
*/
|
|
695
900
|
async renderChildPromptTemplates(childPrompts, params, cancellationToken) {
|
|
696
901
|
if (!childPrompts || childPrompts.length === 0) {
|
|
697
902
|
return {
|
|
698
903
|
renderedTemplates: {}
|
|
699
904
|
};
|
|
700
905
|
}
|
|
906
|
+
// Check for cancellation
|
|
701
907
|
if (cancellationToken?.aborted) {
|
|
702
908
|
throw new Error('Child prompt execution was cancelled');
|
|
703
909
|
}
|
|
910
|
+
//this.logStatus(`🔄 Rendering ${childPrompts.length} child prompt templates in parallel`, true, params);
|
|
911
|
+
// Render all child prompt templates in parallel at this level
|
|
704
912
|
const childRenderingPromises = childPrompts.map(async (childParam) => {
|
|
705
913
|
try {
|
|
914
|
+
// Check for cancellation before each child rendering
|
|
706
915
|
if (cancellationToken?.aborted) {
|
|
707
916
|
throw new Error('Child prompt template rendering was cancelled');
|
|
708
917
|
}
|
|
918
|
+
// First, recursively render any grandchild prompt templates
|
|
709
919
|
let childData = { ...childParam.childPrompt.data };
|
|
710
920
|
if (childParam.childPrompt.childPrompts && childParam.childPrompt.childPrompts.length > 0) {
|
|
711
921
|
const grandchildResults = await this.renderChildPromptTemplates(childParam.childPrompt.childPrompts, params, cancellationToken);
|
|
922
|
+
// Merge grandchild rendered templates into the child's data context
|
|
712
923
|
childData = { ...childData, ...grandchildResults.renderedTemplates };
|
|
713
924
|
}
|
|
925
|
+
// Render the child prompt template with merged data
|
|
926
|
+
//this.logStatus(` 🔹 Rendering child prompt template: ${childParam.childPrompt.prompt.Name} -> ${childParam.parentPlaceholder}`, true, params);
|
|
714
927
|
const childPrompt = childParam.childPrompt.prompt;
|
|
715
928
|
let renderedChildTemplate = '';
|
|
716
929
|
if (childPrompt.TemplateID) {
|
|
930
|
+
// Initialize template engine if not already done
|
|
717
931
|
await this._templateEngine.Config(false, params.contextUser);
|
|
932
|
+
// Load the template for the child prompt
|
|
718
933
|
const template = await this.loadTemplate(childPrompt.TemplateID, params.contextUser);
|
|
719
934
|
if (!template) {
|
|
720
935
|
throw new Error(`Template with ID ${childPrompt.TemplateID} not found for child prompt ${childPrompt.Name}`);
|
|
721
936
|
}
|
|
937
|
+
// Merge child data with original params context
|
|
722
938
|
const mergedChildData = {
|
|
723
|
-
...params.data,
|
|
724
|
-
...childData,
|
|
725
|
-
...childParam.childPrompt.templateData
|
|
939
|
+
...params.data, // Original context
|
|
940
|
+
...childData, // Child-specific data with grandchildren
|
|
941
|
+
...childParam.childPrompt.templateData // Child template data
|
|
726
942
|
};
|
|
943
|
+
// Render the child template
|
|
727
944
|
const childRenderResult = await this.renderPromptTemplate(template, {
|
|
728
|
-
...params,
|
|
729
|
-
prompt: childPrompt,
|
|
945
|
+
...params, // spread original params
|
|
946
|
+
prompt: childPrompt, // THEN, override the prompt for child so we get child related OUTPUT_EXAMPLE and anything else along those lines
|
|
730
947
|
data: mergedChildData,
|
|
731
948
|
templateData: childParam.childPrompt.templateData
|
|
732
949
|
});
|
|
@@ -736,8 +953,10 @@ class AIPromptRunner {
|
|
|
736
953
|
renderedChildTemplate = childRenderResult.Output;
|
|
737
954
|
}
|
|
738
955
|
else {
|
|
956
|
+
// If no template, use empty string (child might be using conversation messages)
|
|
739
957
|
renderedChildTemplate = '';
|
|
740
958
|
}
|
|
959
|
+
// Return the placeholder name and rendered template
|
|
741
960
|
return {
|
|
742
961
|
placeholder: childParam.parentPlaceholder,
|
|
743
962
|
renderedTemplate: renderedChildTemplate,
|
|
@@ -752,6 +971,7 @@ class AIPromptRunner {
|
|
|
752
971
|
},
|
|
753
972
|
maxErrorLength: params.maxErrorLength
|
|
754
973
|
});
|
|
974
|
+
// Return error result but allow other children to continue
|
|
755
975
|
return {
|
|
756
976
|
placeholder: childParam.parentPlaceholder,
|
|
757
977
|
renderedTemplate: `ERROR: ${error.message}`,
|
|
@@ -759,7 +979,9 @@ class AIPromptRunner {
|
|
|
759
979
|
};
|
|
760
980
|
}
|
|
761
981
|
});
|
|
982
|
+
// Wait for all child template rendering to complete
|
|
762
983
|
const childResults = await Promise.all(childRenderingPromises);
|
|
984
|
+
// Check if any critical errors occurred
|
|
763
985
|
const failedChildren = childResults.filter(r => !r.success);
|
|
764
986
|
if (failedChildren.length > 0) {
|
|
765
987
|
this.logError(`${failedChildren.length} out of ${childResults.length} child prompt templates failed to render`, {
|
|
@@ -772,37 +994,57 @@ class AIPromptRunner {
|
|
|
772
994
|
},
|
|
773
995
|
maxErrorLength: params.maxErrorLength
|
|
774
996
|
});
|
|
997
|
+
// any child render failure means we must throw an error
|
|
775
998
|
throw new Error(`Failed to render ${failedChildren.length} child prompt templates: ${failedChildren.map(fc => fc.placeholder).join(', ')}`);
|
|
776
999
|
}
|
|
1000
|
+
// Build rendered templates map
|
|
777
1001
|
const renderedTemplatesMap = {};
|
|
778
1002
|
for (const childResult of childResults) {
|
|
779
1003
|
renderedTemplatesMap[childResult.placeholder] = childResult.renderedTemplate;
|
|
780
1004
|
}
|
|
1005
|
+
//this.logStatus(`✅ Completed rendering of ${childResults.length} child prompt templates`, true, params);
|
|
781
1006
|
return {
|
|
782
1007
|
renderedTemplates: renderedTemplatesMap
|
|
783
1008
|
};
|
|
784
1009
|
}
|
|
1010
|
+
/**
|
|
1011
|
+
* Renders a prompt template with child prompt templates merged into the data context.
|
|
1012
|
+
*
|
|
1013
|
+
* @param prompt - The AI prompt to render
|
|
1014
|
+
* @param params - Original execution parameters
|
|
1015
|
+
* @param childTemplates - Map of placeholder names to rendered child prompt templates
|
|
1016
|
+
* @returns Promise<string> - The rendered prompt text with child templates embedded
|
|
1017
|
+
*/
|
|
785
1018
|
async renderPromptWithChildTemplates(prompt, params, childTemplates) {
|
|
786
1019
|
if (!prompt.TemplateID) {
|
|
1020
|
+
// If no template, return empty string (will be handled by conversation messages)
|
|
787
1021
|
return '';
|
|
788
1022
|
}
|
|
789
1023
|
try {
|
|
1024
|
+
// Initialize template engine
|
|
790
1025
|
await this._templateEngine.Config(false, params.contextUser);
|
|
1026
|
+
// Load the template for the prompt
|
|
791
1027
|
const template = await this.loadTemplate(prompt.TemplateID, params.contextUser);
|
|
792
1028
|
if (!template) {
|
|
793
1029
|
throw new Error(`Template with ID ${prompt.TemplateID} not found for prompt ${prompt.Name}`);
|
|
794
1030
|
}
|
|
795
|
-
|
|
1031
|
+
// Resolve system placeholders with full prompt context
|
|
1032
|
+
const systemPlaceholders = await SystemPlaceholderManager.resolveAllPlaceholders(params);
|
|
1033
|
+
// Merge all data sources with proper priority order
|
|
796
1034
|
const mergedData = {
|
|
797
|
-
...systemPlaceholders,
|
|
798
|
-
...params.data,
|
|
799
|
-
...childTemplates,
|
|
800
|
-
...params.templateData
|
|
1035
|
+
...systemPlaceholders, // System placeholders (lowest priority)
|
|
1036
|
+
...params.data, // Original data context
|
|
1037
|
+
...childTemplates, // Child prompt templates with placeholder names as keys
|
|
1038
|
+
...params.templateData // Additional template data (highest priority)
|
|
801
1039
|
};
|
|
802
1040
|
this.logStatus(` 🔧 ${prompt.Name} [Rendering Prompt Template]`, true, params);
|
|
1041
|
+
// Log placeholder replacement for debugging
|
|
803
1042
|
for (const [placeholder, template] of Object.entries(childTemplates)) {
|
|
804
1043
|
const truncatedTemplate = template.length > 100 ? template.substring(0, 100) + '...' : template;
|
|
1044
|
+
//this.logStatus(` 📝 ${placeholder} -> ${truncatedTemplate}`, true, params);
|
|
805
1045
|
}
|
|
1046
|
+
// Render the template with the full params context
|
|
1047
|
+
// We already have system placeholders resolved, so we'll render directly
|
|
806
1048
|
const renderedPrompt = await this._templateEngine.RenderTemplate(template, template.GetHighestPriorityContent(), mergedData);
|
|
807
1049
|
if (!renderedPrompt.Success) {
|
|
808
1050
|
throw new Error(`Failed to render template for prompt ${prompt.Name}: ${renderedPrompt.Message}`);
|
|
@@ -822,11 +1064,19 @@ class AIPromptRunner {
|
|
|
822
1064
|
throw error;
|
|
823
1065
|
}
|
|
824
1066
|
}
|
|
1067
|
+
/**
|
|
1068
|
+
* Selects the appropriate AI model based on prompt configuration and parameters.
|
|
1069
|
+
* Uses the unified buildModelVendorCandidates method to create an ordered list of candidates,
|
|
1070
|
+
* then selects the first one with an available API key.
|
|
1071
|
+
*/
|
|
825
1072
|
async selectModel(prompt, explicitModelId, contextUser, configurationId, vendorId, params) {
|
|
1073
|
+
// Declare variables outside try block for catch block access
|
|
826
1074
|
let configurationName;
|
|
827
1075
|
let configuration;
|
|
828
1076
|
try {
|
|
829
|
-
|
|
1077
|
+
// Load AI Engine to access cached models and prompt models
|
|
1078
|
+
await AIEngine.Instance.Config(false, contextUser);
|
|
1079
|
+
// Determine selection strategy
|
|
830
1080
|
let selectionStrategy = 'Default';
|
|
831
1081
|
if (explicitModelId) {
|
|
832
1082
|
selectionStrategy = 'Specific';
|
|
@@ -837,11 +1087,14 @@ class AIPromptRunner {
|
|
|
837
1087
|
else if (prompt.SelectionStrategy === 'ByPower' || prompt.MinPowerRank != null) {
|
|
838
1088
|
selectionStrategy = 'ByPower';
|
|
839
1089
|
}
|
|
1090
|
+
// Get configuration info if provided
|
|
840
1091
|
if (configurationId) {
|
|
841
|
-
configuration =
|
|
1092
|
+
configuration = AIEngine.Instance.Configurations.find(c => c.ID === configurationId);
|
|
842
1093
|
configurationName = configuration?.Name;
|
|
843
1094
|
}
|
|
1095
|
+
// Build unified list of model-vendor candidates
|
|
844
1096
|
const candidates = this.buildModelVendorCandidates(prompt, explicitModelId, configurationId, vendorId, params.verbose);
|
|
1097
|
+
// Track all models considered for selection info
|
|
845
1098
|
const modelsConsidered = [];
|
|
846
1099
|
if (candidates.length === 0) {
|
|
847
1100
|
this.logError(`No suitable model candidates found for prompt ${prompt.Name}`, {
|
|
@@ -859,16 +1112,25 @@ class AIPromptRunner {
|
|
|
859
1112
|
selectionInfo: this.createSelectionInfo({
|
|
860
1113
|
aiConfiguration: configuration,
|
|
861
1114
|
modelsConsidered: [],
|
|
862
|
-
modelSelected: undefined,
|
|
1115
|
+
modelSelected: undefined, // Type requirement, but null model means no selection
|
|
863
1116
|
selectionReason: 'No suitable model candidates found',
|
|
864
1117
|
fallbackUsed: false,
|
|
865
1118
|
selectionStrategy
|
|
866
1119
|
})
|
|
867
1120
|
};
|
|
868
1121
|
}
|
|
1122
|
+
// this.logStatus(`🔍 Found ${candidates.length} model-vendor candidates for prompt ${prompt.Name}`, true, params);
|
|
1123
|
+
// if (candidates.length <= 5) {
|
|
1124
|
+
// candidates.forEach((c, i) => {
|
|
1125
|
+
// this.logStatus(` ${i + 1}. ${c.model.Name} via ${c.vendorName || 'default'} (${c.driverClass}) - Priority: ${c.priority}${c.isPreferredVendor ? ' [PREFERRED]' : ''}`, true, params);
|
|
1126
|
+
// });
|
|
1127
|
+
// }
|
|
1128
|
+
// Select the first candidate with available credentials and track all attempts
|
|
869
1129
|
const { selected, consideredModels } = await this.selectModelWithAPIKeyTracked(candidates, prompt.ID, params);
|
|
1130
|
+
// Merge considered models into our tracking
|
|
870
1131
|
modelsConsidered.push(...consideredModels);
|
|
871
1132
|
if (!selected) {
|
|
1133
|
+
// No models with API keys found
|
|
872
1134
|
return {
|
|
873
1135
|
model: null,
|
|
874
1136
|
vendorDriverClass: undefined,
|
|
@@ -879,13 +1141,14 @@ class AIPromptRunner {
|
|
|
879
1141
|
selectionInfo: this.createSelectionInfo({
|
|
880
1142
|
aiConfiguration: configuration,
|
|
881
1143
|
modelsConsidered,
|
|
882
|
-
modelSelected: undefined,
|
|
1144
|
+
modelSelected: undefined, // Type requirement, but null model means no selection
|
|
883
1145
|
selectionReason: 'No API keys found for any model-vendor combination',
|
|
884
1146
|
fallbackUsed: false,
|
|
885
1147
|
selectionStrategy
|
|
886
1148
|
})
|
|
887
1149
|
};
|
|
888
1150
|
}
|
|
1151
|
+
// Determine selection reason
|
|
889
1152
|
let selectionReason = `Selected ${selected.model.Name} via ${selected.vendorName || 'default vendor'}`;
|
|
890
1153
|
if (selected.source === 'explicit') {
|
|
891
1154
|
selectionReason = `Explicitly requested model ${selected.model.Name}`;
|
|
@@ -902,17 +1165,19 @@ class AIPromptRunner {
|
|
|
902
1165
|
if (selected.isPreferredVendor) {
|
|
903
1166
|
selectionReason += ' using preferred vendor';
|
|
904
1167
|
}
|
|
1168
|
+
// Check if fallback was used (not the first candidate)
|
|
905
1169
|
const fallbackUsed = candidates.indexOf(selected) > 0;
|
|
1170
|
+
// Get selected vendor entity
|
|
906
1171
|
let selectedVendor;
|
|
907
1172
|
if (selected.vendorId) {
|
|
908
|
-
selectedVendor =
|
|
1173
|
+
selectedVendor = AIEngine.Instance.Vendors.find(v => v.ID === selected.vendorId);
|
|
909
1174
|
}
|
|
910
1175
|
return {
|
|
911
1176
|
model: selected.model,
|
|
912
1177
|
vendorDriverClass: selected.driverClass,
|
|
913
1178
|
vendorApiName: selected.apiName,
|
|
914
1179
|
vendorSupportsEffortLevel: selected.supportsEffortLevel,
|
|
915
|
-
modelEffortLevel: selected.effortLevel,
|
|
1180
|
+
modelEffortLevel: selected.effortLevel, // Pass through model-specific effort level
|
|
916
1181
|
allCandidates: candidates,
|
|
917
1182
|
selectionInfo: this.createSelectionInfo({
|
|
918
1183
|
aiConfiguration: configuration,
|
|
@@ -941,7 +1206,7 @@ class AIPromptRunner {
|
|
|
941
1206
|
selectionInfo: this.createSelectionInfo({
|
|
942
1207
|
aiConfiguration: configuration,
|
|
943
1208
|
modelsConsidered: [],
|
|
944
|
-
modelSelected: undefined,
|
|
1209
|
+
modelSelected: undefined, // Type requirement, but null model means no selection
|
|
945
1210
|
selectionReason: `Error during model selection: ${error.message}`,
|
|
946
1211
|
fallbackUsed: false,
|
|
947
1212
|
selectionStrategy: 'Default'
|
|
@@ -949,20 +1214,43 @@ class AIPromptRunner {
|
|
|
949
1214
|
};
|
|
950
1215
|
}
|
|
951
1216
|
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Builds a unified, ordered list of model-vendor candidates based on all selection criteria.
|
|
1219
|
+
* Uses a 3-phase approach to properly handle SelectionStrategy='Specific' with AIPromptModel priorities.
|
|
1220
|
+
*
|
|
1221
|
+
* Phase 1: Handle explicit model ID (highest priority)
|
|
1222
|
+
* Phase 2: Check if SelectionStrategy='Specific' with AIPromptModel entries - use ONLY those with AIPromptModel priorities
|
|
1223
|
+
* Phase 3: Use general selection strategy (fallback) - blended priorities from legacy behavior
|
|
1224
|
+
*
|
|
1225
|
+
* @param prompt - The AI prompt with selection criteria
|
|
1226
|
+
* @param explicitModelId - Explicitly specified model ID (highest priority)
|
|
1227
|
+
* @param configurationId - Configuration ID for filtering
|
|
1228
|
+
* @param preferredVendorId - Preferred vendor ID
|
|
1229
|
+
* @returns Ordered array of model-vendor candidates (highest priority first)
|
|
1230
|
+
*/
|
|
952
1231
|
buildModelVendorCandidates(prompt, explicitModelId, configurationId, preferredVendorId, verbose) {
|
|
1232
|
+
// PHASE 1: Handle explicit model ID (highest priority)
|
|
953
1233
|
if (explicitModelId) {
|
|
954
1234
|
return this.buildCandidatesForExplicitModel(explicitModelId, prompt, preferredVendorId);
|
|
955
1235
|
}
|
|
1236
|
+
// PHASE 2: SelectionStrategy='Specific' - Use explicit AIPromptModel configuration
|
|
956
1237
|
if (prompt.SelectionStrategy === 'Specific') {
|
|
957
1238
|
return this.buildCandidatesForSpecificStrategy(prompt, configurationId, verbose);
|
|
958
1239
|
}
|
|
1240
|
+
// PHASE 3: Build candidates with configuration-aware fallback hierarchy
|
|
1241
|
+
// (SelectionStrategy='Default' or 'ByPower')
|
|
959
1242
|
return this.buildCandidatesForGeneralSelection(prompt, configurationId, preferredVendorId, verbose);
|
|
960
1243
|
}
|
|
1244
|
+
/**
|
|
1245
|
+
* PHASE 1: Build candidates for explicitly specified model ID.
|
|
1246
|
+
* Returns candidates for the single model if it's active and compatible.
|
|
1247
|
+
*/
|
|
961
1248
|
buildCandidatesForExplicitModel(explicitModelId, prompt, preferredVendorId) {
|
|
962
|
-
const model =
|
|
1249
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === explicitModelId);
|
|
963
1250
|
if (!model || !model.IsActive) {
|
|
964
1251
|
return [];
|
|
965
1252
|
}
|
|
1253
|
+
// Check model type compatibility
|
|
966
1254
|
if (prompt.AIModelTypeID && model.AIModelTypeID !== prompt.AIModelTypeID) {
|
|
967
1255
|
return [];
|
|
968
1256
|
}
|
|
@@ -970,85 +1258,130 @@ class AIPromptRunner {
|
|
|
970
1258
|
candidates.sort((a, b) => b.priority - a.priority);
|
|
971
1259
|
return candidates;
|
|
972
1260
|
}
|
|
1261
|
+
/**
|
|
1262
|
+
* PHASE 2: Build candidates for 'Specific' selection strategy.
|
|
1263
|
+
* Uses AIPromptModel configuration with clean ranking:
|
|
1264
|
+
* 1. Config-matching models first (by priority DESC)
|
|
1265
|
+
* 2. Then universal (null config) models (by priority DESC)
|
|
1266
|
+
*/
|
|
973
1267
|
buildCandidatesForSpecificStrategy(prompt, configurationId, verbose) {
|
|
974
|
-
|
|
1268
|
+
// Get all active AIPromptModel records for this prompt
|
|
1269
|
+
const allPromptModels = AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID && (pm.Status === 'Active' || pm.Status === 'Preview'));
|
|
1270
|
+
// Filter by configuration matching rules
|
|
975
1271
|
const promptModels = this.filterPromptModelsByConfiguration(allPromptModels, configurationId);
|
|
1272
|
+
// Sort: config-specific before universal, then by priority DESC within each group
|
|
976
1273
|
const sortedPromptModels = this.sortPromptModelsForSpecificStrategy(promptModels, configurationId);
|
|
1274
|
+
// Build candidates maintaining order
|
|
977
1275
|
const candidates = this.buildCandidatesFromPromptModels(sortedPromptModels);
|
|
1276
|
+
// Strategy='Specific' requires explicit configuration
|
|
978
1277
|
if (candidates.length === 0) {
|
|
979
1278
|
const configInfo = configurationId ? ` with configuration "${configurationId}"` : '';
|
|
980
1279
|
throw new Error(`SelectionStrategy is 'Specific' but no valid AIPromptModel candidates found for prompt "${prompt.Name}"${configInfo}. ` +
|
|
981
1280
|
`Please configure AIPromptModel records for this prompt.`);
|
|
982
1281
|
}
|
|
983
1282
|
if (verbose) {
|
|
984
|
-
|
|
1283
|
+
LogStatus(`Using SelectionStrategy='Specific' with ${sortedPromptModels.length} AIPromptModel entries, generated ${candidates.length} candidates`);
|
|
985
1284
|
}
|
|
986
1285
|
return candidates;
|
|
987
1286
|
}
|
|
1287
|
+
/**
|
|
1288
|
+
* PHASE 3: Build candidates for general selection strategies ('Default' or 'ByPower').
|
|
1289
|
+
* Uses configuration-aware fallback hierarchy with legacy blended priority calculation.
|
|
1290
|
+
*/
|
|
988
1291
|
buildCandidatesForGeneralSelection(prompt, configurationId, preferredVendorId, verbose) {
|
|
989
1292
|
const preferredVendorName = preferredVendorId ?
|
|
990
|
-
|
|
1293
|
+
AIEngine.Instance.Vendors.find(v => v.ID === preferredVendorId)?.Name : undefined;
|
|
1294
|
+
// Get prompt models for configuration
|
|
991
1295
|
const promptModels = this.getPromptModelsForConfiguration(prompt, configurationId);
|
|
992
1296
|
const candidates = [];
|
|
993
1297
|
if (promptModels.length > 0) {
|
|
1298
|
+
// Use prompt-specific models with blended priorities
|
|
994
1299
|
this.addPromptSpecificCandidates(candidates, promptModels, preferredVendorId);
|
|
1300
|
+
// Add configuration fallback candidates if needed
|
|
995
1301
|
if (configurationId) {
|
|
996
1302
|
this.addConfigurationFallbackCandidates(candidates, prompt, configurationId, preferredVendorId, verbose);
|
|
997
1303
|
}
|
|
998
1304
|
}
|
|
999
1305
|
else {
|
|
1306
|
+
// No prompt-specific models, use selection strategy
|
|
1000
1307
|
this.addStrategyBasedCandidates(candidates, prompt, preferredVendorName);
|
|
1001
1308
|
}
|
|
1309
|
+
// Sort all candidates by priority (highest first)
|
|
1002
1310
|
candidates.sort((a, b) => b.priority - a.priority);
|
|
1003
1311
|
return candidates;
|
|
1004
1312
|
}
|
|
1313
|
+
/**
|
|
1314
|
+
* Helper: Filter prompt models by configuration matching rules.
|
|
1315
|
+
* Supports configuration inheritance - includes models from the entire inheritance chain.
|
|
1316
|
+
*/
|
|
1005
1317
|
filterPromptModelsByConfiguration(allPromptModels, configurationId) {
|
|
1006
1318
|
if (configurationId) {
|
|
1007
|
-
|
|
1319
|
+
// Get the configuration inheritance chain
|
|
1320
|
+
const chain = AIEngine.Instance.GetConfigurationChain(configurationId);
|
|
1008
1321
|
const chainIds = new Set(chain.map(c => c.ID));
|
|
1322
|
+
// Include models matching any config in the chain, plus null-config (universal fallback)
|
|
1009
1323
|
return allPromptModels.filter(pm => (pm.ConfigurationID && chainIds.has(pm.ConfigurationID)) ||
|
|
1010
1324
|
pm.ConfigurationID === null);
|
|
1011
1325
|
}
|
|
1012
1326
|
else {
|
|
1327
|
+
// No config specified - only include null-config models
|
|
1013
1328
|
return allPromptModels.filter(pm => pm.ConfigurationID === null);
|
|
1014
1329
|
}
|
|
1015
1330
|
}
|
|
1331
|
+
/**
|
|
1332
|
+
* Helper: Sort prompt models for 'Specific' strategy.
|
|
1333
|
+
* Respects configuration inheritance chain - child configs first, then parents, then null-config.
|
|
1334
|
+
* Within each config level, sorts by priority DESC.
|
|
1335
|
+
*/
|
|
1016
1336
|
sortPromptModelsForSpecificStrategy(promptModels, configurationId) {
|
|
1017
1337
|
if (!configurationId) {
|
|
1338
|
+
// No config specified - just sort by priority
|
|
1018
1339
|
return promptModels.sort((a, b) => (b.Priority || 0) - (a.Priority || 0));
|
|
1019
1340
|
}
|
|
1020
|
-
|
|
1341
|
+
// Get the configuration inheritance chain and create position map
|
|
1342
|
+
const chain = AIEngine.Instance.GetConfigurationChain(configurationId);
|
|
1021
1343
|
const chainOrder = new Map(chain.map((c, index) => [c.ID, index]));
|
|
1022
1344
|
return promptModels.sort((a, b) => {
|
|
1345
|
+
// Primary: Chain position (lower index = higher priority, null config = last)
|
|
1023
1346
|
const aChainPos = a.ConfigurationID ? (chainOrder.get(a.ConfigurationID) ?? 999) : 1000;
|
|
1024
1347
|
const bChainPos = b.ConfigurationID ? (chainOrder.get(b.ConfigurationID) ?? 999) : 1000;
|
|
1025
1348
|
if (aChainPos !== bChainPos) {
|
|
1026
|
-
return aChainPos - bChainPos;
|
|
1349
|
+
return aChainPos - bChainPos; // Lower chain position first (child before parent)
|
|
1027
1350
|
}
|
|
1351
|
+
// Secondary: Higher priority first within same config level
|
|
1028
1352
|
return (b.Priority || 0) - (a.Priority || 0);
|
|
1029
1353
|
});
|
|
1030
1354
|
}
|
|
1355
|
+
/**
|
|
1356
|
+
* Helper: Build candidates from sorted AIPromptModel records.
|
|
1357
|
+
* Expands VendorID=null to all vendors for that model.
|
|
1358
|
+
*/
|
|
1031
1359
|
buildCandidatesFromPromptModels(promptModels) {
|
|
1032
1360
|
const candidates = [];
|
|
1033
1361
|
for (const pm of promptModels) {
|
|
1034
|
-
const model =
|
|
1362
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === pm.ModelID);
|
|
1035
1363
|
if (!model || !model.IsActive)
|
|
1036
1364
|
continue;
|
|
1037
1365
|
if (pm.VendorID) {
|
|
1366
|
+
// Specific vendor specified - create single candidate
|
|
1038
1367
|
const candidate = this.createCandidateForSpecificVendor(model, pm);
|
|
1039
1368
|
if (candidate) {
|
|
1040
1369
|
candidates.push(candidate);
|
|
1041
1370
|
}
|
|
1042
1371
|
}
|
|
1043
1372
|
else {
|
|
1373
|
+
// No vendor specified - create candidates for all vendors
|
|
1044
1374
|
const vendorCandidates = this.createCandidatesForAllVendors(model);
|
|
1045
1375
|
candidates.push(...vendorCandidates);
|
|
1046
1376
|
}
|
|
1047
1377
|
}
|
|
1048
1378
|
return candidates;
|
|
1049
1379
|
}
|
|
1380
|
+
/**
|
|
1381
|
+
* Helper: Create candidate for specific vendor from AIPromptModel.
|
|
1382
|
+
*/
|
|
1050
1383
|
createCandidateForSpecificVendor(model, promptModel) {
|
|
1051
|
-
const modelVendor =
|
|
1384
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === promptModel.ModelID &&
|
|
1052
1385
|
mv.VendorID === promptModel.VendorID &&
|
|
1053
1386
|
mv.Status === 'Active' &&
|
|
1054
1387
|
this.isInferenceProvider(mv));
|
|
@@ -1061,14 +1394,17 @@ class AIPromptRunner {
|
|
|
1061
1394
|
driverClass: modelVendor.DriverClass || model.DriverClass,
|
|
1062
1395
|
apiName: modelVendor.APIName || model.APIName,
|
|
1063
1396
|
supportsEffortLevel: modelVendor.SupportsEffortLevel ?? model.SupportsEffortLevel ?? false,
|
|
1064
|
-
effortLevel: promptModel.EffortLevel ?? undefined,
|
|
1397
|
+
effortLevel: promptModel.EffortLevel ?? undefined, // Model-specific effort level override
|
|
1065
1398
|
isPreferredVendor: false,
|
|
1066
|
-
priority: 0,
|
|
1399
|
+
priority: 0, // Order is determined by promptModels sort
|
|
1067
1400
|
source: 'prompt-model'
|
|
1068
1401
|
};
|
|
1069
1402
|
}
|
|
1403
|
+
/**
|
|
1404
|
+
* Helper: Create candidates for all vendors of a model, sorted by vendor priority.
|
|
1405
|
+
*/
|
|
1070
1406
|
createCandidatesForAllVendors(model) {
|
|
1071
|
-
const vendors =
|
|
1407
|
+
const vendors = AIEngine.Instance.ModelVendors
|
|
1072
1408
|
.filter(mv => mv.ModelID === model.ID &&
|
|
1073
1409
|
mv.Status === 'Active' &&
|
|
1074
1410
|
this.isInferenceProvider(mv))
|
|
@@ -1083,10 +1419,11 @@ class AIPromptRunner {
|
|
|
1083
1419
|
apiName: vendor.APIName || model.APIName,
|
|
1084
1420
|
supportsEffortLevel: vendor.SupportsEffortLevel ?? model.SupportsEffortLevel ?? false,
|
|
1085
1421
|
isPreferredVendor: false,
|
|
1086
|
-
priority: 0,
|
|
1422
|
+
priority: 0, // Order is determined by promptModels sort
|
|
1087
1423
|
source: 'prompt-model'
|
|
1088
1424
|
});
|
|
1089
1425
|
}
|
|
1426
|
+
// If no vendors found, use model defaults
|
|
1090
1427
|
if (candidates.length === 0 && model.DriverClass) {
|
|
1091
1428
|
candidates.push({
|
|
1092
1429
|
model,
|
|
@@ -1100,94 +1437,128 @@ class AIPromptRunner {
|
|
|
1100
1437
|
}
|
|
1101
1438
|
return candidates;
|
|
1102
1439
|
}
|
|
1440
|
+
/**
|
|
1441
|
+
* Helper: Get prompt models for configuration with inheritance chain fallback.
|
|
1442
|
+
* Walks the configuration inheritance chain looking for prompt models.
|
|
1443
|
+
* Returns models from the first config in the chain that has any, or falls back to null-config.
|
|
1444
|
+
*/
|
|
1103
1445
|
getPromptModelsForConfiguration(prompt, configurationId) {
|
|
1104
1446
|
if (configurationId) {
|
|
1105
|
-
|
|
1447
|
+
// Get the configuration inheritance chain (child -> parent -> grandparent -> ...)
|
|
1448
|
+
const chain = AIEngine.Instance.GetConfigurationChain(configurationId);
|
|
1449
|
+
// Walk the chain looking for prompt models
|
|
1106
1450
|
for (const config of chain) {
|
|
1107
|
-
const promptModels =
|
|
1451
|
+
const promptModels = AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID &&
|
|
1108
1452
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
1109
1453
|
pm.ConfigurationID === config.ID);
|
|
1110
1454
|
if (promptModels.length > 0) {
|
|
1111
1455
|
return promptModels;
|
|
1112
1456
|
}
|
|
1113
1457
|
}
|
|
1114
|
-
|
|
1458
|
+
// No match in chain, fall back to NULL config models
|
|
1459
|
+
LogStatus(`No models found in configuration chain for "${configurationId}", falling back to default models`);
|
|
1115
1460
|
}
|
|
1116
|
-
|
|
1461
|
+
// Return null-config (universal) models
|
|
1462
|
+
return AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID &&
|
|
1117
1463
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
1118
1464
|
!pm.ConfigurationID);
|
|
1119
1465
|
}
|
|
1466
|
+
/**
|
|
1467
|
+
* Helper: Add prompt-specific candidates with blended priorities (legacy behavior).
|
|
1468
|
+
*/
|
|
1120
1469
|
addPromptSpecificCandidates(candidates, promptModels, preferredVendorId) {
|
|
1121
1470
|
for (const pm of promptModels) {
|
|
1122
|
-
const model =
|
|
1471
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === pm.ModelID);
|
|
1123
1472
|
if (model && model.IsActive) {
|
|
1124
1473
|
const modelCandidates = this.createCandidatesForModel(model, 5000, 'prompt-model', preferredVendorId, pm.Priority);
|
|
1125
1474
|
candidates.push(...modelCandidates);
|
|
1126
1475
|
}
|
|
1127
1476
|
}
|
|
1128
1477
|
}
|
|
1478
|
+
/**
|
|
1479
|
+
* Helper: Add configuration fallback candidates from the inheritance chain.
|
|
1480
|
+
* Adds models from parent configs (with decreasing priority) and null-config models as final fallback.
|
|
1481
|
+
*/
|
|
1129
1482
|
addConfigurationFallbackCandidates(candidates, prompt, configurationId, preferredVendorId, verbose) {
|
|
1130
|
-
const chain =
|
|
1483
|
+
const chain = AIEngine.Instance.GetConfigurationChain(configurationId);
|
|
1484
|
+
// Add models from parent configs (skip index 0 which is the direct config, already handled)
|
|
1131
1485
|
for (let i = 1; i < chain.length; i++) {
|
|
1132
1486
|
const parentConfig = chain[i];
|
|
1133
|
-
const parentModels =
|
|
1487
|
+
const parentModels = AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID &&
|
|
1134
1488
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
1135
1489
|
pm.ConfigurationID === parentConfig.ID);
|
|
1136
1490
|
if (parentModels.length > 0 && verbose) {
|
|
1137
|
-
|
|
1491
|
+
LogStatus(`Adding ${parentModels.length} models from parent config "${parentConfig.Name}" as fallback`);
|
|
1138
1492
|
}
|
|
1139
1493
|
for (const pm of parentModels) {
|
|
1140
|
-
const model =
|
|
1494
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === pm.ModelID);
|
|
1141
1495
|
if (model && model.IsActive) {
|
|
1496
|
+
// Decrease base priority for each level up the chain (3000, 2500, 2000, etc.)
|
|
1142
1497
|
const basePriority = 3000 - (i * 500);
|
|
1143
1498
|
const modelCandidates = this.createCandidatesForModel(model, basePriority, 'prompt-model', preferredVendorId, pm.Priority);
|
|
1144
1499
|
candidates.push(...modelCandidates);
|
|
1145
1500
|
}
|
|
1146
1501
|
}
|
|
1147
1502
|
}
|
|
1148
|
-
|
|
1503
|
+
// Finally add NULL config models (universal fallback) with lowest priority
|
|
1504
|
+
const nullConfigModels = AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID &&
|
|
1149
1505
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
1150
1506
|
!pm.ConfigurationID);
|
|
1151
1507
|
if (nullConfigModels.length > 0 && verbose) {
|
|
1152
|
-
|
|
1508
|
+
LogStatus(`Adding ${nullConfigModels.length} NULL configuration models as universal fallback`);
|
|
1153
1509
|
}
|
|
1154
1510
|
for (const pm of nullConfigModels) {
|
|
1155
|
-
const model =
|
|
1511
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === pm.ModelID);
|
|
1156
1512
|
if (model && model.IsActive) {
|
|
1157
|
-
const modelCandidates = this.createCandidatesForModel(model, 1000,
|
|
1513
|
+
const modelCandidates = this.createCandidatesForModel(model, 1000, // Lowest priority tier
|
|
1514
|
+
'prompt-model', preferredVendorId, pm.Priority);
|
|
1158
1515
|
candidates.push(...modelCandidates);
|
|
1159
1516
|
}
|
|
1160
1517
|
}
|
|
1161
1518
|
}
|
|
1519
|
+
/**
|
|
1520
|
+
* Helper: Add strategy-based candidates when no prompt models exist.
|
|
1521
|
+
*/
|
|
1162
1522
|
addStrategyBasedCandidates(candidates, prompt, preferredVendorName) {
|
|
1163
1523
|
let modelPool = this.getModelPoolForStrategy(prompt, preferredVendorName);
|
|
1164
1524
|
modelPool = this.sortModelPoolByStrategy(modelPool, prompt);
|
|
1525
|
+
// Create candidates for each model in the pool
|
|
1165
1526
|
modelPool.forEach((model, index) => {
|
|
1166
|
-
const basePriority = 1000 - index * 10;
|
|
1527
|
+
const basePriority = 1000 - index * 10; // Decrease priority by position
|
|
1167
1528
|
const source = prompt.SelectionStrategy === 'ByPower' ? 'power-rank' : 'model-type';
|
|
1168
1529
|
candidates.push(...this.createCandidatesForModel(model, basePriority, source));
|
|
1169
1530
|
});
|
|
1170
1531
|
}
|
|
1532
|
+
/**
|
|
1533
|
+
* Helper: Get model pool filtered for strategy.
|
|
1534
|
+
*/
|
|
1171
1535
|
getModelPoolForStrategy(prompt, preferredVendorName) {
|
|
1172
|
-
return
|
|
1536
|
+
return AIEngine.Instance.Models.filter(m => m.IsActive &&
|
|
1173
1537
|
(!prompt.AIModelTypeID || m.AIModelTypeID === prompt.AIModelTypeID) &&
|
|
1174
1538
|
(!preferredVendorName ||
|
|
1175
|
-
|
|
1539
|
+
AIEngine.Instance.ModelVendors.some(mv => mv.ModelID === m.ID &&
|
|
1176
1540
|
mv.Status === 'Active' &&
|
|
1177
1541
|
mv.Vendor === preferredVendorName &&
|
|
1178
1542
|
this.isInferenceProvider(mv))));
|
|
1179
1543
|
}
|
|
1544
|
+
/**
|
|
1545
|
+
* Helper: Sort model pool by selection strategy.
|
|
1546
|
+
*/
|
|
1180
1547
|
sortModelPoolByStrategy(modelPool, prompt) {
|
|
1181
1548
|
if (prompt.SelectionStrategy === 'ByPower') {
|
|
1182
1549
|
return this.sortByPowerPreference(modelPool, prompt.PowerPreference);
|
|
1183
1550
|
}
|
|
1184
1551
|
else {
|
|
1552
|
+
// Default strategy
|
|
1185
1553
|
const minPowerRank = prompt.MinPowerRank || 0;
|
|
1186
1554
|
return modelPool
|
|
1187
1555
|
.filter(m => m.PowerRank >= minPowerRank)
|
|
1188
1556
|
.sort((a, b) => b.PowerRank - a.PowerRank);
|
|
1189
1557
|
}
|
|
1190
1558
|
}
|
|
1559
|
+
/**
|
|
1560
|
+
* Helper: Sort models by power preference.
|
|
1561
|
+
*/
|
|
1191
1562
|
sortByPowerPreference(modelPool, powerPreference) {
|
|
1192
1563
|
const pool = [...modelPool];
|
|
1193
1564
|
switch (powerPreference) {
|
|
@@ -1202,11 +1573,16 @@ class AIPromptRunner {
|
|
|
1202
1573
|
return pool.sort((a, b) => b.PowerRank - a.PowerRank);
|
|
1203
1574
|
}
|
|
1204
1575
|
}
|
|
1576
|
+
/**
|
|
1577
|
+
* Helper: Create candidates for a model with AIModelVendor priorities (legacy behavior).
|
|
1578
|
+
*/
|
|
1205
1579
|
createCandidatesForModel(model, basePriority, source, preferredVendorId, promptModelPriority) {
|
|
1206
1580
|
const modelCandidates = [];
|
|
1207
|
-
|
|
1581
|
+
// Get all vendors for this model - filter for inference providers only
|
|
1582
|
+
const modelVendors = AIEngine.Instance.ModelVendors
|
|
1208
1583
|
.filter(mv => mv.ModelID === model.ID && mv.Status === 'Active' && this.isInferenceProvider(mv))
|
|
1209
1584
|
.sort((a, b) => b.Priority - a.Priority);
|
|
1585
|
+
// First, add preferred vendor if it exists
|
|
1210
1586
|
if (preferredVendorId) {
|
|
1211
1587
|
const preferredVendor = modelVendors.find(mv => mv.VendorID === preferredVendorId);
|
|
1212
1588
|
if (preferredVendor) {
|
|
@@ -1218,11 +1594,12 @@ class AIPromptRunner {
|
|
|
1218
1594
|
apiName: preferredVendor.APIName || model.APIName,
|
|
1219
1595
|
supportsEffortLevel: preferredVendor.SupportsEffortLevel ?? model.SupportsEffortLevel ?? false,
|
|
1220
1596
|
isPreferredVendor: true,
|
|
1221
|
-
priority: basePriority + 1000,
|
|
1597
|
+
priority: basePriority + 1000, // Boost priority for preferred vendor
|
|
1222
1598
|
source
|
|
1223
1599
|
});
|
|
1224
1600
|
}
|
|
1225
1601
|
}
|
|
1602
|
+
// Then add other vendors in priority order
|
|
1226
1603
|
for (const vendor of modelVendors) {
|
|
1227
1604
|
if (vendor.VendorID !== preferredVendorId) {
|
|
1228
1605
|
modelCandidates.push({
|
|
@@ -1238,6 +1615,7 @@ class AIPromptRunner {
|
|
|
1238
1615
|
});
|
|
1239
1616
|
}
|
|
1240
1617
|
}
|
|
1618
|
+
// If no vendors found, add model with its default driver
|
|
1241
1619
|
if (modelCandidates.length === 0 && model.DriverClass) {
|
|
1242
1620
|
modelCandidates.push({
|
|
1243
1621
|
model,
|
|
@@ -1249,21 +1627,34 @@ class AIPromptRunner {
|
|
|
1249
1627
|
source
|
|
1250
1628
|
});
|
|
1251
1629
|
}
|
|
1630
|
+
// Apply prompt model priority if provided (legacy blended approach)
|
|
1252
1631
|
if (promptModelPriority !== undefined) {
|
|
1253
1632
|
modelCandidates.forEach(c => c.priority += promptModelPriority * 10);
|
|
1254
1633
|
}
|
|
1255
1634
|
return modelCandidates;
|
|
1256
1635
|
}
|
|
1636
|
+
/**
|
|
1637
|
+
* Creates a properly typed AIModelSelectionInfo instance.
|
|
1638
|
+
* TypeScript requires instantiating the class to get the getValidCandidates() method.
|
|
1639
|
+
*/
|
|
1257
1640
|
createSelectionInfo(data) {
|
|
1258
|
-
const info = new
|
|
1641
|
+
const info = new AIModelSelectionInfo();
|
|
1259
1642
|
Object.assign(info, data);
|
|
1260
1643
|
return info;
|
|
1261
1644
|
}
|
|
1645
|
+
/**
|
|
1646
|
+
* Converts model selection info into ModelVendorCandidate array for retry logic.
|
|
1647
|
+
* Extracts only the valid candidates (those with available API keys) from the selection info.
|
|
1648
|
+
*
|
|
1649
|
+
* @param selectionInfo - Model selection information containing considered models
|
|
1650
|
+
* @returns Array of valid model-vendor candidates sorted by priority
|
|
1651
|
+
*/
|
|
1262
1652
|
buildCandidatesFromSelectionInfo(selectionInfo) {
|
|
1263
1653
|
const validModels = selectionInfo.extractValidCandidates();
|
|
1264
1654
|
return validModels.map(considered => {
|
|
1655
|
+
// Find matching model vendor for driver and API info
|
|
1265
1656
|
const modelVendor = considered.vendor
|
|
1266
|
-
?
|
|
1657
|
+
? AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === considered.model.ID &&
|
|
1267
1658
|
mv.VendorID === considered.vendor.ID)
|
|
1268
1659
|
: undefined;
|
|
1269
1660
|
return {
|
|
@@ -1273,29 +1664,47 @@ class AIPromptRunner {
|
|
|
1273
1664
|
driverClass: modelVendor?.DriverClass || considered.model.DriverClass,
|
|
1274
1665
|
apiName: modelVendor?.APIName || considered.model.APIName,
|
|
1275
1666
|
supportsEffortLevel: modelVendor?.SupportsEffortLevel ?? considered.model.SupportsEffortLevel ?? false,
|
|
1276
|
-
isPreferredVendor: false,
|
|
1667
|
+
isPreferredVendor: false, // Can't determine from selection info alone
|
|
1277
1668
|
priority: considered.priority,
|
|
1278
1669
|
source: (selectionInfo.selectionStrategy === 'ByPower' ? 'power-rank' : 'model-type')
|
|
1279
1670
|
};
|
|
1280
|
-
}).sort((a, b) => b.priority - a.priority);
|
|
1281
|
-
}
|
|
1671
|
+
}).sort((a, b) => b.priority - a.priority); // Sort by priority descending
|
|
1672
|
+
}
|
|
1673
|
+
/**
|
|
1674
|
+
* Enhanced version of selectModelWithAPIKey that tracks all considered models
|
|
1675
|
+
* for model selection reporting. Uses the hierarchical credential resolution
|
|
1676
|
+
* system to check for available credentials.
|
|
1677
|
+
*
|
|
1678
|
+
* @param candidates - Ordered array of model-vendor candidates
|
|
1679
|
+
* @param promptId - The prompt ID for credential resolution
|
|
1680
|
+
* @param params - Optional prompt parameters for verbose logging and credential override
|
|
1681
|
+
* @returns Object containing selected candidate and all considered models
|
|
1682
|
+
*/
|
|
1282
1683
|
async selectModelWithAPIKeyTracked(candidates, promptId, params) {
|
|
1684
|
+
// Cache for credential availability checks
|
|
1685
|
+
// Key format: "driverClass:modelId:vendorId" to properly cache credential hierarchy
|
|
1283
1686
|
const credentialCache = new Map();
|
|
1284
1687
|
const consideredModels = [];
|
|
1688
|
+
// Check ALL candidates to build complete list of valid and invalid options
|
|
1285
1689
|
for (const candidate of candidates) {
|
|
1690
|
+
// Build cache key including model and vendor for proper credential resolution
|
|
1286
1691
|
const cacheKey = `${candidate.driverClass}:${candidate.model.ID}:${candidate.vendorId || 'default'}`;
|
|
1692
|
+
// Check cache first
|
|
1287
1693
|
let hasCredentials;
|
|
1288
1694
|
if (credentialCache.has(cacheKey)) {
|
|
1289
1695
|
hasCredentials = credentialCache.get(cacheKey);
|
|
1290
1696
|
}
|
|
1291
1697
|
else {
|
|
1698
|
+
// Check for credentials using hierarchical resolution
|
|
1292
1699
|
hasCredentials = this.hasCredentialsAvailable(candidate.driverClass, promptId, candidate.model.ID, candidate.vendorId, params);
|
|
1293
1700
|
credentialCache.set(cacheKey, hasCredentials);
|
|
1294
1701
|
}
|
|
1702
|
+
// Get vendor entity from AIEngine cache if vendorId is available
|
|
1295
1703
|
let vendorEntity;
|
|
1296
1704
|
if (candidate.vendorId) {
|
|
1297
|
-
vendorEntity =
|
|
1705
|
+
vendorEntity = AIEngine.Instance.Vendors.find(v => v.ID === candidate.vendorId);
|
|
1298
1706
|
}
|
|
1707
|
+
// Track this model as considered with availability status
|
|
1299
1708
|
consideredModels.push({
|
|
1300
1709
|
model: candidate.model,
|
|
1301
1710
|
vendor: vendorEntity,
|
|
@@ -1304,6 +1713,7 @@ class AIPromptRunner {
|
|
|
1304
1713
|
unavailableReason: hasCredentials ? undefined : `No credentials configured for driver ${candidate.driverClass}`
|
|
1305
1714
|
});
|
|
1306
1715
|
}
|
|
1716
|
+
// Select the first available candidate (highest priority with API key)
|
|
1307
1717
|
const selected = consideredModels.find(m => m.available);
|
|
1308
1718
|
const selectedCandidate = selected ? candidates.find(c => c.model.ID === selected.model.ID &&
|
|
1309
1719
|
c.vendorId === selected.vendor?.ID) : null;
|
|
@@ -1316,6 +1726,7 @@ class AIPromptRunner {
|
|
|
1316
1726
|
this.logStatus(` Found ${validCount} valid candidate(s) out of ${candidates.length} total`, true, params);
|
|
1317
1727
|
}
|
|
1318
1728
|
else {
|
|
1729
|
+
// Log what we tried
|
|
1319
1730
|
const triedSummary = candidates.slice(0, 5).map(c => `${c.model.Name}/${c.vendorName || 'default'}(${c.driverClass})`).join(', ');
|
|
1320
1731
|
this.logError(`No credentials found for any model-vendor combination. Tried: ${triedSummary}${candidates.length > 5 ? `... (${candidates.length} total)` : ''}`, {
|
|
1321
1732
|
category: 'CredentialValidation',
|
|
@@ -1329,18 +1740,24 @@ class AIPromptRunner {
|
|
|
1329
1740
|
}
|
|
1330
1741
|
return { selected: selectedCandidate, consideredModels };
|
|
1331
1742
|
}
|
|
1743
|
+
/**
|
|
1744
|
+
* Creates an AIPromptRun entity for execution tracking
|
|
1745
|
+
*/
|
|
1332
1746
|
async createPromptRun(prompt, model, params, systemPromptText, startTime, vendorId, modelSelectionInfo) {
|
|
1333
1747
|
const promptRun = await this._metadata.GetEntityObject('MJ: AI Prompt Runs', params.contextUser);
|
|
1334
1748
|
try {
|
|
1335
1749
|
promptRun.NewRecord();
|
|
1336
1750
|
promptRun.PromptID = prompt.ID;
|
|
1337
1751
|
promptRun.ModelID = model.ID;
|
|
1752
|
+
// Set initial status and tracking fields
|
|
1338
1753
|
promptRun.Status = 'Running';
|
|
1339
1754
|
promptRun.Cancelled = false;
|
|
1340
1755
|
promptRun.CacheHit = false;
|
|
1341
1756
|
promptRun.StreamingEnabled = false;
|
|
1342
1757
|
promptRun.WasSelectedResult = false;
|
|
1758
|
+
// Set model selection tracking fields
|
|
1343
1759
|
if (modelSelectionInfo) {
|
|
1760
|
+
// Convert the rich entity objects to simple IDs/names for database storage
|
|
1344
1761
|
const dbSelectionInfo = {
|
|
1345
1762
|
configurationId: modelSelectionInfo.aiConfiguration?.ID,
|
|
1346
1763
|
configurationName: modelSelectionInfo.aiConfiguration?.Name,
|
|
@@ -1361,28 +1778,35 @@ class AIPromptRunner {
|
|
|
1361
1778
|
};
|
|
1362
1779
|
promptRun.ModelSelection = JSON.stringify(dbSelectionInfo);
|
|
1363
1780
|
promptRun.SelectionStrategy = modelSelectionInfo.selectionStrategy || 'Default';
|
|
1781
|
+
// Set ModelPowerRank if available
|
|
1364
1782
|
if (model.PowerRank != null) {
|
|
1365
1783
|
promptRun.ModelPowerRank = model.PowerRank;
|
|
1366
1784
|
}
|
|
1367
1785
|
}
|
|
1786
|
+
// Set original model tracking for failover
|
|
1368
1787
|
promptRun.OriginalModelID = model.ID;
|
|
1369
1788
|
promptRun.OriginalRequestStartTime = startTime;
|
|
1789
|
+
// Initialize failover tracking fields
|
|
1370
1790
|
promptRun.FailoverAttempts = 0;
|
|
1371
1791
|
promptRun.FailoverErrors = null;
|
|
1372
1792
|
promptRun.FailoverDurations = null;
|
|
1373
1793
|
promptRun.TotalFailoverDuration = 0;
|
|
1794
|
+
// Check if model has pre-selected vendor info from selectModel
|
|
1374
1795
|
const modelWithVendor = model;
|
|
1375
1796
|
if (modelSelectionInfo) {
|
|
1376
1797
|
promptRun.VendorID = modelSelectionInfo.vendorSelected?.ID || vendorId || modelWithVendor._selectedVendorId;
|
|
1377
1798
|
}
|
|
1378
1799
|
else if (vendorId) {
|
|
1800
|
+
// Explicit vendor ID provided
|
|
1379
1801
|
promptRun.VendorID = vendorId;
|
|
1380
1802
|
}
|
|
1381
1803
|
else if (modelWithVendor._selectedVendorId) {
|
|
1804
|
+
// Use vendor selected during model selection (with API key verification)
|
|
1382
1805
|
promptRun.VendorID = modelWithVendor._selectedVendorId;
|
|
1383
1806
|
}
|
|
1384
1807
|
else {
|
|
1385
|
-
|
|
1808
|
+
// Fallback: grab the highest priority AI Model Vendor record for this model (inference providers only)
|
|
1809
|
+
const modelVendors = AIEngine.Instance.ModelVendors
|
|
1386
1810
|
.filter((mv) => mv.ModelID === model.ID && mv.Status === 'Active' && this.isInferenceProvider(mv))
|
|
1387
1811
|
.sort((a, b) => b.Priority - a.Priority);
|
|
1388
1812
|
if (modelVendors.length > 0) {
|
|
@@ -1391,24 +1815,32 @@ class AIPromptRunner {
|
|
|
1391
1815
|
}
|
|
1392
1816
|
promptRun.ConfigurationID = params.configurationId;
|
|
1393
1817
|
promptRun.RunAt = startTime;
|
|
1818
|
+
// Set AgentRunID if provided for agent-prompt execution tracking
|
|
1394
1819
|
if (params.agentRunId) {
|
|
1395
1820
|
promptRun.AgentRunID = params.agentRunId;
|
|
1396
1821
|
}
|
|
1822
|
+
// Resolve and save the effort level used (same precedence as ChatParams resolution)
|
|
1397
1823
|
if (params.effortLevel !== undefined && params.effortLevel !== null) {
|
|
1398
1824
|
promptRun.EffortLevel = params.effortLevel;
|
|
1399
1825
|
}
|
|
1400
1826
|
else if (prompt.EffortLevel !== undefined && prompt.EffortLevel !== null) {
|
|
1401
1827
|
promptRun.EffortLevel = prompt.EffortLevel;
|
|
1402
1828
|
}
|
|
1829
|
+
// If neither is set, EffortLevel remains null (provider default was used)
|
|
1830
|
+
// Set ParentID for hierarchical prompt execution tracking
|
|
1403
1831
|
if (params.parentPromptRunId) {
|
|
1404
1832
|
promptRun.ParentID = params.parentPromptRunId;
|
|
1405
1833
|
}
|
|
1834
|
+
// Set RerunFromPromptRunID if this is a rerun
|
|
1406
1835
|
if (params.rerunFromPromptRunID) {
|
|
1407
1836
|
promptRun.RerunFromPromptRunID = params.rerunFromPromptRunID;
|
|
1408
1837
|
}
|
|
1838
|
+
// Always save the response format from the prompt if it exists
|
|
1409
1839
|
if (prompt.ResponseFormat && prompt.ResponseFormat !== 'Any') {
|
|
1410
1840
|
promptRun.ResponseFormat = prompt.ResponseFormat;
|
|
1411
1841
|
}
|
|
1842
|
+
// Save the actual values that will be used (either from prompt defaults or additionalParameters)
|
|
1843
|
+
// First, apply defaults from prompt entity
|
|
1412
1844
|
if (prompt.Temperature != null)
|
|
1413
1845
|
promptRun.Temperature = prompt.Temperature;
|
|
1414
1846
|
if (prompt.TopP != null)
|
|
@@ -1429,6 +1861,7 @@ class AIPromptRunner {
|
|
|
1429
1861
|
promptRun.LogProbs = prompt.IncludeLogProbs;
|
|
1430
1862
|
if (prompt.TopLogProbs != null)
|
|
1431
1863
|
promptRun.TopLogProbs = prompt.TopLogProbs;
|
|
1864
|
+
// Then override with additionalParameters if provided
|
|
1432
1865
|
if (params.additionalParameters) {
|
|
1433
1866
|
if (params.additionalParameters.temperature !== undefined) {
|
|
1434
1867
|
promptRun.Temperature = params.additionalParameters.temperature;
|
|
@@ -1461,6 +1894,7 @@ class AIPromptRunner {
|
|
|
1461
1894
|
promptRun.TopLogProbs = params.additionalParameters.topLogProbs;
|
|
1462
1895
|
}
|
|
1463
1896
|
}
|
|
1897
|
+
// Store the input data/context as JSON in Messages field
|
|
1464
1898
|
if (params.data || params.templateData || systemPromptText) {
|
|
1465
1899
|
const messages = [];
|
|
1466
1900
|
if (systemPromptText) {
|
|
@@ -1476,13 +1910,14 @@ class AIPromptRunner {
|
|
|
1476
1910
|
messages: messages || [],
|
|
1477
1911
|
});
|
|
1478
1912
|
}
|
|
1913
|
+
// Populate new retry tracking columns with initial values
|
|
1479
1914
|
promptRun.ValidationBehavior = prompt.ValidationBehavior || 'Warn';
|
|
1480
1915
|
promptRun.RetryStrategy = prompt.RetryStrategy || 'Fixed';
|
|
1481
1916
|
promptRun.MaxRetriesConfigured = prompt.MaxRetries || 0;
|
|
1482
1917
|
promptRun.FirstAttemptAt = startTime;
|
|
1483
|
-
promptRun.ValidationAttemptCount = 0;
|
|
1918
|
+
promptRun.ValidationAttemptCount = 0; // Will be updated during execution
|
|
1484
1919
|
promptRun.SuccessfulValidationCount = 0;
|
|
1485
|
-
promptRun.FinalValidationPassed = false;
|
|
1920
|
+
promptRun.FinalValidationPassed = false; // Will be updated after execution
|
|
1486
1921
|
const saveResult = await promptRun.Save();
|
|
1487
1922
|
if (!saveResult) {
|
|
1488
1923
|
const error = `Failed to save AIPromptRun: ${promptRun.LatestResult?.CompleteMessage || 'Unknown error'}`;
|
|
@@ -1497,12 +1932,14 @@ class AIPromptRunner {
|
|
|
1497
1932
|
});
|
|
1498
1933
|
throw new Error(error);
|
|
1499
1934
|
}
|
|
1935
|
+
// Invoke callback if provided
|
|
1500
1936
|
if (params.onPromptRunCreated) {
|
|
1501
1937
|
try {
|
|
1502
1938
|
await params.onPromptRunCreated(promptRun.ID);
|
|
1503
1939
|
}
|
|
1504
1940
|
catch (callbackError) {
|
|
1505
|
-
|
|
1941
|
+
LogStatus(`Error in onPromptRunCreated callback: ${callbackError.message}`);
|
|
1942
|
+
// Don't fail the execution if callback fails
|
|
1506
1943
|
}
|
|
1507
1944
|
}
|
|
1508
1945
|
return promptRun;
|
|
@@ -1520,18 +1957,26 @@ class AIPromptRunner {
|
|
|
1520
1957
|
throw new Error(msg);
|
|
1521
1958
|
}
|
|
1522
1959
|
}
|
|
1960
|
+
/**
|
|
1961
|
+
* Renders the prompt template with provided data
|
|
1962
|
+
*/
|
|
1523
1963
|
async renderPromptTemplate(template, params) {
|
|
1524
1964
|
try {
|
|
1965
|
+
// Get the highest priority content for the template
|
|
1525
1966
|
const templateContent = template.GetHighestPriorityContent();
|
|
1526
1967
|
if (!templateContent) {
|
|
1527
1968
|
throw new Error(`No content found for template ${template.Name}`);
|
|
1528
1969
|
}
|
|
1529
|
-
|
|
1970
|
+
// Resolve system placeholders with full params context
|
|
1971
|
+
const systemPlaceholders = await SystemPlaceholderManager.resolveAllPlaceholders(params);
|
|
1972
|
+
// Merge data contexts with system placeholders having lowest priority
|
|
1530
1973
|
const mergedData = {
|
|
1531
|
-
...systemPlaceholders,
|
|
1532
|
-
...params.data,
|
|
1533
|
-
...params.templateData
|
|
1974
|
+
...systemPlaceholders, // System placeholders first (lowest priority)
|
|
1975
|
+
...params.data, // User data overrides system placeholders
|
|
1976
|
+
...params.templateData // Template data has highest priority
|
|
1534
1977
|
};
|
|
1978
|
+
//LogStatus(`🔧 Rendering template '${template.Name}' with ${Object.keys(systemPlaceholders).length} system placeholders`);
|
|
1979
|
+
// Render the template
|
|
1535
1980
|
return await this._templateEngine.RenderTemplate(template, templateContent, mergedData);
|
|
1536
1981
|
}
|
|
1537
1982
|
catch (error) {
|
|
@@ -1547,20 +1992,40 @@ class AIPromptRunner {
|
|
|
1547
1992
|
throw error;
|
|
1548
1993
|
}
|
|
1549
1994
|
}
|
|
1995
|
+
/**
|
|
1996
|
+
* Executes the AI model with failover support
|
|
1997
|
+
*
|
|
1998
|
+
* @remarks
|
|
1999
|
+
* This method wraps the core executeModel functionality with intelligent failover
|
|
2000
|
+
* capabilities. It will attempt to execute with different models/vendors according
|
|
2001
|
+
* to the configured failover strategy when errors occur.
|
|
2002
|
+
*
|
|
2003
|
+
* The method calls several smaller, focused helper methods:
|
|
2004
|
+
* - buildFailoverCandidates: Creates candidate models based on type restrictions
|
|
2005
|
+
* - createCandidatesFromModels: Converts models to vendor-specific candidates
|
|
2006
|
+
* - updatePromptRunWithFailoverSuccess: Records successful failover metadata
|
|
2007
|
+
* - updatePromptRunWithFailoverFailure: Records failed failover metadata
|
|
2008
|
+
* - createFailoverErrorResult: Creates standardized error response
|
|
2009
|
+
*/
|
|
1550
2010
|
async executeModelWithFailover(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole = 'system', cancellationToken, allCandidates, promptRun, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel) {
|
|
2011
|
+
// Get failover configuration (used for errorScope filtering)
|
|
1551
2012
|
const failoverConfig = this.getFailoverConfiguration(prompt);
|
|
2013
|
+
// If no candidates provided or failover disabled, execute normally with first model
|
|
1552
2014
|
if (!allCandidates || allCandidates.length === 0 || failoverConfig.strategy === 'None') {
|
|
1553
2015
|
return this.executeModel(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole, cancellationToken, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel);
|
|
1554
2016
|
}
|
|
2017
|
+
// Track failover attempts
|
|
1555
2018
|
const failoverAttempts = [];
|
|
1556
2019
|
let lastError = null;
|
|
2020
|
+
// Iterate through all candidates in priority order with instant failover
|
|
1557
2021
|
for (let i = 0; i < allCandidates.length; i++) {
|
|
1558
2022
|
const candidate = allCandidates[i];
|
|
1559
2023
|
const attemptStartTime = Date.now();
|
|
1560
2024
|
try {
|
|
2025
|
+
// Log the attempt if not the first one
|
|
1561
2026
|
if (i > 0) {
|
|
1562
2027
|
const vendorName = candidate.vendorName || 'default';
|
|
1563
|
-
|
|
2028
|
+
LogStatusEx({
|
|
1564
2029
|
message: `🔄 Trying candidate ${i + 1}/${allCandidates.length}: ${candidate.model.Name} via ${vendorName}`,
|
|
1565
2030
|
category: 'AI',
|
|
1566
2031
|
additionalArgs: [{
|
|
@@ -1573,20 +2038,29 @@ class AIPromptRunner {
|
|
|
1573
2038
|
}]
|
|
1574
2039
|
});
|
|
1575
2040
|
}
|
|
2041
|
+
// Execute the model with this candidate
|
|
1576
2042
|
const result = await this.executeModel(candidate.model, renderedPrompt, prompt, params, candidate.vendorId || null, conversationMessages, templateMessageRole, cancellationToken, candidate.driverClass, candidate.apiName, candidate.supportsEffortLevel, candidate.effortLevel);
|
|
2043
|
+
// CRITICAL FIX: Check if result failed but is retriable (network errors, rate limits, etc.)
|
|
2044
|
+
// Provider drivers (GeminiLLM, OpenAILLM, etc.) catch errors internally and return ChatResult{success: false}
|
|
2045
|
+
// instead of throwing, so we must check result.success here.
|
|
1577
2046
|
if (!result.success && result.errorInfo?.canFailover) {
|
|
1578
2047
|
lastError = result.exception || new Error(result.errorMessage || 'Model execution failed');
|
|
2048
|
+
// Use shared failover error handling logic
|
|
1579
2049
|
const decision = await this.processFailoverError(lastError, result.errorInfo, candidate, attemptStartTime, i, allCandidates, failoverAttempts, prompt, failoverConfig);
|
|
2050
|
+
// Update candidates list (may have been filtered)
|
|
1580
2051
|
allCandidates = decision.updatedCandidates;
|
|
1581
2052
|
if (decision.shouldRetry) {
|
|
1582
|
-
i--;
|
|
2053
|
+
i--; // Retry same model/vendor
|
|
1583
2054
|
continue;
|
|
1584
2055
|
}
|
|
1585
2056
|
if (decision.shouldContinue) {
|
|
1586
|
-
continue;
|
|
2057
|
+
continue; // Try next candidate
|
|
1587
2058
|
}
|
|
2059
|
+
// Otherwise break (fatal error or last candidate)
|
|
1588
2060
|
break;
|
|
1589
2061
|
}
|
|
2062
|
+
// If we reach here, the result was successful
|
|
2063
|
+
// Update promptRun with failover information if we had prior failures
|
|
1590
2064
|
if (failoverAttempts.length > 0 && promptRun) {
|
|
1591
2065
|
this.updatePromptRunWithFailoverSuccess(promptRun, failoverAttempts, candidate.model, candidate.vendorId || null);
|
|
1592
2066
|
}
|
|
@@ -1594,48 +2068,65 @@ class AIPromptRunner {
|
|
|
1594
2068
|
}
|
|
1595
2069
|
catch (error) {
|
|
1596
2070
|
lastError = error;
|
|
1597
|
-
|
|
2071
|
+
// Analyze error to get error info
|
|
2072
|
+
const errorInfo = ErrorAnalyzer.analyzeError(lastError);
|
|
2073
|
+
// Use shared failover error handling logic
|
|
1598
2074
|
const decision = await this.processFailoverError(lastError, errorInfo, candidate, attemptStartTime, i, allCandidates, failoverAttempts, prompt, failoverConfig);
|
|
2075
|
+
// Update candidates list (may have been filtered)
|
|
1599
2076
|
allCandidates = decision.updatedCandidates;
|
|
1600
2077
|
if (decision.shouldRetry) {
|
|
1601
|
-
i--;
|
|
2078
|
+
i--; // Retry same model/vendor
|
|
1602
2079
|
continue;
|
|
1603
2080
|
}
|
|
1604
2081
|
if (decision.shouldContinue) {
|
|
1605
|
-
continue;
|
|
2082
|
+
continue; // Try next candidate
|
|
1606
2083
|
}
|
|
2084
|
+
// Otherwise break (fatal error or last candidate)
|
|
1607
2085
|
break;
|
|
1608
2086
|
}
|
|
1609
2087
|
}
|
|
2088
|
+
// All candidates failed
|
|
1610
2089
|
if (promptRun && failoverAttempts.length > 0) {
|
|
1611
2090
|
this.updatePromptRunWithFailoverFailure(promptRun, failoverAttempts);
|
|
1612
2091
|
}
|
|
1613
2092
|
return this.createFailoverErrorResult(lastError, failoverAttempts);
|
|
1614
2093
|
}
|
|
2094
|
+
/**
|
|
2095
|
+
* Builds failover candidates for a prompt based on available models and type restrictions
|
|
2096
|
+
*/
|
|
1615
2097
|
async buildFailoverCandidates(prompt) {
|
|
1616
|
-
const aiEngine =
|
|
2098
|
+
const aiEngine = AIEngine.Instance;
|
|
2099
|
+
// Get all models, filtered by type if specified
|
|
1617
2100
|
let allModels;
|
|
1618
2101
|
if (prompt.AIModelTypeID) {
|
|
2102
|
+
// Find the model type from the prompt
|
|
1619
2103
|
const modelType = aiEngine.ModelTypes.find(mt => mt.ID === prompt.AIModelTypeID);
|
|
1620
2104
|
if (!modelType) {
|
|
1621
2105
|
throw new Error(`Model type ${prompt.AIModelTypeID} not found`);
|
|
1622
2106
|
}
|
|
2107
|
+
// Get all models of this specific type
|
|
1623
2108
|
const targetTypeName = modelType.Name.trim().toLowerCase();
|
|
1624
2109
|
allModels = aiEngine.Models.filter(m => {
|
|
2110
|
+
// Guard against AIModelType being non-string (defensive coding for data issues)
|
|
1625
2111
|
const mType = typeof m.AIModelType === 'string' ? m.AIModelType.trim().toLowerCase() : '';
|
|
1626
2112
|
return mType === targetTypeName;
|
|
1627
2113
|
});
|
|
1628
2114
|
}
|
|
1629
2115
|
else {
|
|
2116
|
+
// No type restriction - get all models
|
|
1630
2117
|
allModels = aiEngine.Models;
|
|
1631
2118
|
}
|
|
1632
2119
|
return this.createCandidatesFromModels(allModels);
|
|
1633
2120
|
}
|
|
2121
|
+
/**
|
|
2122
|
+
* Creates model-vendor candidates from a list of models
|
|
2123
|
+
*/
|
|
1634
2124
|
createCandidatesFromModels(models) {
|
|
1635
2125
|
const candidates = [];
|
|
1636
2126
|
for (const model of models) {
|
|
1637
2127
|
const vendors = model.ModelVendors || [];
|
|
1638
2128
|
if (vendors.length === 0) {
|
|
2129
|
+
// Model without specific vendors
|
|
1639
2130
|
candidates.push({
|
|
1640
2131
|
model: model,
|
|
1641
2132
|
vendorId: undefined,
|
|
@@ -1649,6 +2140,7 @@ class AIPromptRunner {
|
|
|
1649
2140
|
});
|
|
1650
2141
|
}
|
|
1651
2142
|
else {
|
|
2143
|
+
// Add each vendor as a separate candidate
|
|
1652
2144
|
for (const vendor of vendors) {
|
|
1653
2145
|
candidates.push({
|
|
1654
2146
|
model: model,
|
|
@@ -1666,6 +2158,9 @@ class AIPromptRunner {
|
|
|
1666
2158
|
}
|
|
1667
2159
|
return candidates;
|
|
1668
2160
|
}
|
|
2161
|
+
/**
|
|
2162
|
+
* Updates prompt run with successful failover tracking data
|
|
2163
|
+
*/
|
|
1669
2164
|
updatePromptRunWithFailoverSuccess(promptRun, failoverAttempts, currentModel, currentVendorId) {
|
|
1670
2165
|
promptRun.FailoverAttempts = failoverAttempts.length;
|
|
1671
2166
|
promptRun.FailoverErrors = JSON.stringify(failoverAttempts.map(a => ({
|
|
@@ -1676,6 +2171,7 @@ class AIPromptRunner {
|
|
|
1676
2171
|
})));
|
|
1677
2172
|
promptRun.FailoverDurations = JSON.stringify(failoverAttempts.map(a => a.duration));
|
|
1678
2173
|
promptRun.TotalFailoverDuration = failoverAttempts.reduce((sum, a) => sum + a.duration, 0);
|
|
2174
|
+
// Update ModelID if we ended up using a different model
|
|
1679
2175
|
if (currentModel.ID !== promptRun.OriginalModelID) {
|
|
1680
2176
|
promptRun.ModelID = currentModel.ID;
|
|
1681
2177
|
}
|
|
@@ -1683,6 +2179,9 @@ class AIPromptRunner {
|
|
|
1683
2179
|
promptRun.VendorID = currentVendorId;
|
|
1684
2180
|
}
|
|
1685
2181
|
}
|
|
2182
|
+
/**
|
|
2183
|
+
* Updates prompt run with failover failure tracking data
|
|
2184
|
+
*/
|
|
1686
2185
|
updatePromptRunWithFailoverFailure(promptRun, failoverAttempts) {
|
|
1687
2186
|
promptRun.FailoverAttempts = failoverAttempts.length;
|
|
1688
2187
|
promptRun.FailoverErrors = JSON.stringify(failoverAttempts.map(a => ({
|
|
@@ -1694,14 +2193,20 @@ class AIPromptRunner {
|
|
|
1694
2193
|
promptRun.FailoverDurations = JSON.stringify(failoverAttempts.map(a => a.duration));
|
|
1695
2194
|
promptRun.TotalFailoverDuration = failoverAttempts.reduce((sum, a) => sum + a.duration, 0);
|
|
1696
2195
|
}
|
|
2196
|
+
/**
|
|
2197
|
+
* Creates an error result for failed failover attempts
|
|
2198
|
+
*/
|
|
1697
2199
|
createFailoverErrorResult(lastError, failoverAttempts) {
|
|
1698
2200
|
const startTime = new Date();
|
|
1699
2201
|
const endTime = new Date();
|
|
2202
|
+
// Check if this is a ContextLengthExceeded error - if so, mark as Fatal
|
|
1700
2203
|
const hasContextLengthError = failoverAttempts.some(a => a.errorType === 'ContextLengthExceeded' ||
|
|
1701
|
-
|
|
2204
|
+
ErrorAnalyzer.analyzeError(a.error).errorType === 'ContextLengthExceeded');
|
|
2205
|
+
// If ContextLengthExceeded and all failover attempts failed, this is fatal
|
|
1702
2206
|
let errorInfo;
|
|
1703
2207
|
if (lastError) {
|
|
1704
|
-
errorInfo =
|
|
2208
|
+
errorInfo = ErrorAnalyzer.analyzeError(lastError);
|
|
2209
|
+
// Override severity to Fatal if context length exceeded and no larger models exist
|
|
1705
2210
|
if (hasContextLengthError && errorInfo.errorType === 'ContextLengthExceeded') {
|
|
1706
2211
|
errorInfo.severity = 'Fatal';
|
|
1707
2212
|
}
|
|
@@ -1718,43 +2223,63 @@ class AIPromptRunner {
|
|
|
1718
2223
|
data: null
|
|
1719
2224
|
};
|
|
1720
2225
|
}
|
|
2226
|
+
/**
|
|
2227
|
+
* Executes the AI model with the rendered prompt
|
|
2228
|
+
*/
|
|
1721
2229
|
async executeModel(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole = 'system', cancellationToken, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel) {
|
|
2230
|
+
// define these variables here to ensure they're available in the catch block
|
|
1722
2231
|
let driverClass;
|
|
1723
2232
|
let apiName;
|
|
1724
2233
|
let llm;
|
|
1725
2234
|
let chatParams;
|
|
1726
2235
|
try {
|
|
1727
|
-
|
|
2236
|
+
// Get verbose flag for logging
|
|
2237
|
+
const verbose = params.verbose === true || IsVerboseLoggingEnabled();
|
|
2238
|
+
// Determine if effort level is supported
|
|
1728
2239
|
let supportsEffortLevel = false;
|
|
2240
|
+
// Get vendor-specific configuration
|
|
2241
|
+
// Use passed vendor info if available, otherwise fall back to vendor lookup
|
|
1729
2242
|
if (vendorDriverClass && vendorApiName) {
|
|
2243
|
+
// Vendor info was provided by the caller (from model selection)
|
|
1730
2244
|
driverClass = vendorDriverClass;
|
|
1731
2245
|
apiName = vendorApiName;
|
|
2246
|
+
// Use provided vendorSupportsEffortLevel, or default to false
|
|
1732
2247
|
supportsEffortLevel = vendorSupportsEffortLevel ?? false;
|
|
1733
2248
|
}
|
|
1734
2249
|
else {
|
|
2250
|
+
// Fallback to model defaults or vendor lookup
|
|
1735
2251
|
driverClass = model.DriverClass;
|
|
1736
2252
|
apiName = model.APIName;
|
|
2253
|
+
// Start with model's SupportsEffortLevel setting
|
|
1737
2254
|
supportsEffortLevel = model.SupportsEffortLevel ?? false;
|
|
1738
2255
|
if (vendorId) {
|
|
1739
|
-
|
|
2256
|
+
// Find the AIModelVendor record for this specific vendor - must be an inference provider
|
|
2257
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find((mv) => mv.ModelID === model.ID && mv.VendorID === vendorId && mv.Status === 'Active' && this.isInferenceProvider(mv));
|
|
1740
2258
|
if (modelVendor) {
|
|
1741
2259
|
driverClass = modelVendor.DriverClass || driverClass;
|
|
1742
2260
|
apiName = modelVendor.APIName || apiName;
|
|
2261
|
+
// Use modelVendor's SupportsEffortLevel if available
|
|
1743
2262
|
supportsEffortLevel = modelVendor.SupportsEffortLevel ?? supportsEffortLevel;
|
|
1744
2263
|
}
|
|
1745
2264
|
else {
|
|
2265
|
+
// Log warning if vendor was specified but not found or not an inference provider
|
|
1746
2266
|
this.logStatus(`⚠️ Vendor ${vendorId} not found or is not an inference provider for model ${model.Name}, using model defaults`, true, params);
|
|
1747
2267
|
}
|
|
1748
2268
|
}
|
|
1749
2269
|
}
|
|
2270
|
+
// Resolve credentials using hierarchical resolution (Credentials system with legacy fallback)
|
|
1750
2271
|
const apiKey = await this.resolveCredentialForExecution(driverClass, prompt.ID, model.ID, vendorId ?? undefined, params);
|
|
1751
|
-
|
|
1752
|
-
|
|
2272
|
+
// Create LLM instance with vendor-specific driver class
|
|
2273
|
+
llm = MJGlobal.Instance.ClassFactory.CreateInstance(BaseLLM, driverClass, apiKey);
|
|
2274
|
+
// Prepare chat parameters
|
|
2275
|
+
chatParams = new ChatParams();
|
|
1753
2276
|
if (!apiName) {
|
|
1754
2277
|
throw new Error(`No API name found for model ${model.Name}. Please ensure the model or its vendor configuration includes an APIName.`);
|
|
1755
2278
|
}
|
|
1756
2279
|
chatParams.model = apiName;
|
|
1757
2280
|
chatParams.cancellationToken = cancellationToken;
|
|
2281
|
+
// Apply defaults from prompt entity first (if they exist)
|
|
2282
|
+
// These can be overridden by additionalParameters
|
|
1758
2283
|
if (prompt.Temperature != null)
|
|
1759
2284
|
chatParams.temperature = prompt.Temperature;
|
|
1760
2285
|
if (prompt.TopP != null)
|
|
@@ -1770,13 +2295,16 @@ class AIPromptRunner {
|
|
|
1770
2295
|
if (prompt.Seed != null)
|
|
1771
2296
|
chatParams.seed = prompt.Seed;
|
|
1772
2297
|
if (prompt.StopSequences) {
|
|
2298
|
+
// Parse comma-delimited stop sequences
|
|
1773
2299
|
chatParams.stopSequences = prompt.StopSequences.split(',').map((s) => s.trim()).filter((s) => s.length > 0);
|
|
1774
2300
|
}
|
|
1775
2301
|
if (prompt.IncludeLogProbs != null)
|
|
1776
2302
|
chatParams.includeLogProbs = prompt.IncludeLogProbs;
|
|
1777
2303
|
if (prompt.TopLogProbs != null)
|
|
1778
2304
|
chatParams.topLogProbs = prompt.TopLogProbs;
|
|
2305
|
+
// Apply additional parameters if provided (these override prompt defaults)
|
|
1779
2306
|
if (params.additionalParameters) {
|
|
2307
|
+
// Apply chat-specific parameters from additionalParameters
|
|
1780
2308
|
if (params.additionalParameters.temperature !== undefined) {
|
|
1781
2309
|
chatParams.temperature = params.additionalParameters.temperature;
|
|
1782
2310
|
}
|
|
@@ -1808,11 +2336,18 @@ class AIPromptRunner {
|
|
|
1808
2336
|
chatParams.topLogProbs = params.additionalParameters.topLogProbs;
|
|
1809
2337
|
}
|
|
1810
2338
|
}
|
|
2339
|
+
// Apply effortLevel with precedence hierarchy
|
|
2340
|
+
// 1. params.effortLevel (runtime override - highest priority)
|
|
2341
|
+
// 2. modelEffortLevel (model-specific override from AIPromptModel - second priority)
|
|
2342
|
+
// 3. Agent DefaultPromptEffortLevel (passed via params.effortLevel by BaseAgent - third priority)
|
|
2343
|
+
// 4. prompt.EffortLevel (prompt default - fourth priority)
|
|
2344
|
+
// 5. No effort level (provider default - lowest priority)
|
|
1811
2345
|
const hasEffortLevel = (params.effortLevel !== undefined && params.effortLevel !== null) ||
|
|
1812
2346
|
(modelEffortLevel !== undefined && modelEffortLevel !== null) ||
|
|
1813
2347
|
(prompt.EffortLevel !== undefined && prompt.EffortLevel !== null);
|
|
1814
2348
|
if (hasEffortLevel) {
|
|
1815
2349
|
if (supportsEffortLevel) {
|
|
2350
|
+
// Vendor/model supports effort level, apply it with precedence
|
|
1816
2351
|
if (params.effortLevel !== undefined && params.effortLevel !== null) {
|
|
1817
2352
|
chatParams.effortLevel = params.effortLevel.toString();
|
|
1818
2353
|
}
|
|
@@ -1824,18 +2359,25 @@ class AIPromptRunner {
|
|
|
1824
2359
|
}
|
|
1825
2360
|
}
|
|
1826
2361
|
else {
|
|
2362
|
+
// Vendor/model does not support effort level, log warning
|
|
1827
2363
|
const effortValue = params.effortLevel ?? modelEffortLevel ?? prompt.EffortLevel;
|
|
1828
2364
|
console.log(`⚠️ Effort Level ${effortValue} specified but will be ignored - model ${model.Name} does not support effort levels`);
|
|
1829
2365
|
}
|
|
1830
2366
|
}
|
|
2367
|
+
// If none are set, effortLevel remains undefined and providers use their defaults
|
|
2368
|
+
// Apply response format from prompt settings
|
|
1831
2369
|
if (prompt.ResponseFormat && prompt.ResponseFormat !== 'Any') {
|
|
1832
|
-
chatParams.responseFormat = prompt.ResponseFormat;
|
|
2370
|
+
chatParams.responseFormat = prompt.ResponseFormat; //as 'Any' | 'Text' | 'Markdown' | 'JSON' | 'ModelSpecific';
|
|
1833
2371
|
}
|
|
1834
2372
|
else {
|
|
2373
|
+
// if chatParams.responseFormat is not set or set to Any, stay silent on response format
|
|
1835
2374
|
chatParams.responseFormat = undefined;
|
|
1836
2375
|
}
|
|
2376
|
+
// Build message array with rendered prompt and conversation messages
|
|
1837
2377
|
chatParams.messages = this.buildMessageArray(renderedPrompt, conversationMessages, templateMessageRole);
|
|
2378
|
+
// Execute the model with cancellation support
|
|
1838
2379
|
if (cancellationToken) {
|
|
2380
|
+
// If cancellation token is provided, wrap the execution to handle cancellation
|
|
1839
2381
|
return await Promise.race([
|
|
1840
2382
|
llm.ChatCompletion(chatParams),
|
|
1841
2383
|
new Promise((_, reject) => {
|
|
@@ -1851,11 +2393,12 @@ class AIPromptRunner {
|
|
|
1851
2393
|
]);
|
|
1852
2394
|
}
|
|
1853
2395
|
else {
|
|
2396
|
+
// No cancellation token, execute normally
|
|
1854
2397
|
return await llm.ChatCompletion(chatParams);
|
|
1855
2398
|
}
|
|
1856
2399
|
}
|
|
1857
2400
|
catch (error) {
|
|
1858
|
-
const errorInfo =
|
|
2401
|
+
const errorInfo = ErrorAnalyzer.analyzeError(error, driverClass);
|
|
1859
2402
|
this.logError(error, {
|
|
1860
2403
|
category: 'ModelExecution',
|
|
1861
2404
|
model: model,
|
|
@@ -1868,51 +2411,70 @@ class AIPromptRunner {
|
|
|
1868
2411
|
throw error;
|
|
1869
2412
|
}
|
|
1870
2413
|
}
|
|
2414
|
+
/**
|
|
2415
|
+
* Builds the message array combining rendered prompt with conversation messages
|
|
2416
|
+
*/
|
|
1871
2417
|
buildMessageArray(renderedPrompt, conversationMessages, templateMessageRole = 'system') {
|
|
1872
2418
|
const messages = [];
|
|
2419
|
+
// Add rendered template as system or user message if not 'none'
|
|
1873
2420
|
if (renderedPrompt && templateMessageRole !== 'none') {
|
|
1874
2421
|
messages.push({
|
|
1875
|
-
role: templateMessageRole === 'system' ?
|
|
2422
|
+
role: templateMessageRole === 'system' ? ChatMessageRole.system : ChatMessageRole.user,
|
|
1876
2423
|
content: renderedPrompt,
|
|
1877
2424
|
});
|
|
1878
2425
|
}
|
|
2426
|
+
// Add conversation messages if provided
|
|
1879
2427
|
if (conversationMessages && conversationMessages.length > 0) {
|
|
1880
2428
|
messages.push(...conversationMessages);
|
|
1881
2429
|
}
|
|
2430
|
+
// If no conversation messages and no rendered prompt as user message,
|
|
2431
|
+
// add a default user message to ensure we have at least one user message
|
|
1882
2432
|
if ((!conversationMessages || conversationMessages.length === 0) && templateMessageRole !== 'user' && renderedPrompt) {
|
|
2433
|
+
// If we only have a system message, we need a user message too
|
|
1883
2434
|
if (templateMessageRole === 'system') {
|
|
1884
2435
|
messages.push({
|
|
1885
|
-
role:
|
|
2436
|
+
role: ChatMessageRole.user,
|
|
1886
2437
|
content: 'Please proceed with the above instructions.',
|
|
1887
2438
|
});
|
|
1888
2439
|
}
|
|
1889
2440
|
}
|
|
1890
2441
|
else if ((!conversationMessages || conversationMessages.length === 0) && !renderedPrompt) {
|
|
2442
|
+
// Fallback: if no conversation and no rendered prompt, add a basic user message
|
|
1891
2443
|
messages.push({
|
|
1892
|
-
role:
|
|
2444
|
+
role: ChatMessageRole.user,
|
|
1893
2445
|
content: 'Hello',
|
|
1894
2446
|
});
|
|
1895
2447
|
}
|
|
1896
2448
|
return messages;
|
|
1897
2449
|
}
|
|
2450
|
+
/**
|
|
2451
|
+
* Executes the model with retry logic for validation failures
|
|
2452
|
+
*/
|
|
1898
2453
|
async executeWithValidationRetries(selectedModel, renderedPromptText, prompt, params, promptRun, allCandidates, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel) {
|
|
1899
2454
|
const validationAttempts = [];
|
|
1900
2455
|
const maxRetries = Math.max(0, prompt.MaxRetries || 0);
|
|
1901
2456
|
let lastError = null;
|
|
2457
|
+
// Track cumulative token usage across all attempts
|
|
1902
2458
|
let cumulativePromptTokens = 0;
|
|
1903
2459
|
let cumulativeCompletionTokens = 0;
|
|
1904
2460
|
let cumulativeCost = 0;
|
|
1905
2461
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
1906
2462
|
try {
|
|
2463
|
+
// Check for cancellation before each attempt
|
|
1907
2464
|
if (params.cancellationToken?.aborted) {
|
|
1908
2465
|
throw new Error('Execution was cancelled during validation retries');
|
|
1909
2466
|
}
|
|
1910
2467
|
if (attempt > 0) {
|
|
1911
|
-
|
|
2468
|
+
LogStatus(` 🔄 Retrying execution due to validation failure, attempt ${attempt + 1}/${maxRetries + 1}`);
|
|
1912
2469
|
await this.applyRetryDelay(prompt, attempt);
|
|
1913
2470
|
}
|
|
1914
|
-
|
|
2471
|
+
// Execute the AI model with failover support
|
|
2472
|
+
const modelResult = await this.executeModelWithFailover(selectedModel, renderedPromptText, prompt, params, promptRun.VendorID, params.conversationMessages, params.templateMessageRole || 'system', params.cancellationToken, allCandidates, // Pass the candidates from initial selection
|
|
2473
|
+
promptRun, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel);
|
|
2474
|
+
// Check for fatal errors - don't attempt validation/retry on these
|
|
2475
|
+
// Fatal errors (like ContextLengthExceeded when all models exhausted) cannot be resolved by retrying
|
|
1915
2476
|
if (!modelResult.success && modelResult.errorInfo?.severity === 'Fatal') {
|
|
2477
|
+
// Record the fatal error attempt
|
|
1916
2478
|
const validationAttempt = {
|
|
1917
2479
|
attemptNumber: attempt + 1,
|
|
1918
2480
|
success: false,
|
|
@@ -1921,6 +2483,7 @@ class AIPromptRunner {
|
|
|
1921
2483
|
timestamp: new Date(),
|
|
1922
2484
|
};
|
|
1923
2485
|
validationAttempts.push(validationAttempt);
|
|
2486
|
+
// Return immediately - no point in validation or retries for fatal errors
|
|
1924
2487
|
return {
|
|
1925
2488
|
modelResult,
|
|
1926
2489
|
parsedResult: {
|
|
@@ -1935,12 +2498,15 @@ class AIPromptRunner {
|
|
|
1935
2498
|
},
|
|
1936
2499
|
};
|
|
1937
2500
|
}
|
|
2501
|
+
// Accumulate token usage from this attempt
|
|
1938
2502
|
if (modelResult.data?.usage) {
|
|
1939
2503
|
cumulativePromptTokens += modelResult.data.usage.promptTokens || 0;
|
|
1940
2504
|
cumulativeCompletionTokens += modelResult.data.usage.completionTokens || 0;
|
|
1941
2505
|
cumulativeCost += modelResult.data.usage.cost || 0;
|
|
1942
2506
|
}
|
|
2507
|
+
// Parse and validate the result
|
|
1943
2508
|
const { result, validationResult, validationErrors } = await this.parseAndValidateResultEnhanced(modelResult, prompt, params.skipValidation, params.cleanValidationSyntax, promptRun, params);
|
|
2509
|
+
// Record this validation attempt
|
|
1944
2510
|
const validationAttempt = {
|
|
1945
2511
|
attemptNumber: attempt + 1,
|
|
1946
2512
|
success: validationResult?.Success || false,
|
|
@@ -1952,6 +2518,7 @@ class AIPromptRunner {
|
|
|
1952
2518
|
};
|
|
1953
2519
|
validationAttempts.push(validationAttempt);
|
|
1954
2520
|
if (validationResult?.Success !== false) {
|
|
2521
|
+
// Validation succeeded, return the result
|
|
1955
2522
|
return {
|
|
1956
2523
|
modelResult,
|
|
1957
2524
|
parsedResult: { result, validationResult },
|
|
@@ -1963,16 +2530,19 @@ class AIPromptRunner {
|
|
|
1963
2530
|
},
|
|
1964
2531
|
};
|
|
1965
2532
|
}
|
|
2533
|
+
// Validation failed, check if we should retry
|
|
2534
|
+
// BUG FIX: Only retry in Strict mode, not in Warn or None modes
|
|
1966
2535
|
if (prompt.ValidationBehavior === 'Strict' && attempt < maxRetries) {
|
|
1967
2536
|
lastError = new Error(`Validation failed: ${validationErrors?.map(e => e.Message).join('; ')}`);
|
|
1968
|
-
|
|
1969
|
-
continue;
|
|
2537
|
+
LogStatus(` ⚠️ Validation failed on attempt ${attempt + 1}, will retry (Strict mode)`);
|
|
2538
|
+
continue; // Retry
|
|
1970
2539
|
}
|
|
1971
2540
|
else {
|
|
2541
|
+
// Either not strict mode or no more retries, return what we have
|
|
1972
2542
|
const reason = prompt.ValidationBehavior !== 'Strict'
|
|
1973
2543
|
? `${prompt.ValidationBehavior || 'None'} mode - continuing with invalid output (no retry)`
|
|
1974
2544
|
: 'max retries exceeded';
|
|
1975
|
-
|
|
2545
|
+
LogStatus(` ⚠️ Validation failed on attempt ${attempt + 1}, stopping retries (${reason})`);
|
|
1976
2546
|
return {
|
|
1977
2547
|
modelResult,
|
|
1978
2548
|
parsedResult: { result, validationResult },
|
|
@@ -1997,6 +2567,7 @@ class AIPromptRunner {
|
|
|
1997
2567
|
},
|
|
1998
2568
|
maxErrorLength: params.maxErrorLength
|
|
1999
2569
|
});
|
|
2570
|
+
// Record failed attempt
|
|
2000
2571
|
const validationAttempt = {
|
|
2001
2572
|
attemptNumber: attempt + 1,
|
|
2002
2573
|
success: false,
|
|
@@ -2006,17 +2577,26 @@ class AIPromptRunner {
|
|
|
2006
2577
|
};
|
|
2007
2578
|
validationAttempts.push(validationAttempt);
|
|
2008
2579
|
if (attempt === maxRetries) {
|
|
2009
|
-
throw error;
|
|
2580
|
+
throw error; // Last attempt, propagate error
|
|
2010
2581
|
}
|
|
2011
2582
|
}
|
|
2012
2583
|
}
|
|
2584
|
+
// Should not reach here, but just in case
|
|
2013
2585
|
throw lastError || new Error('Execution failed after all retry attempts');
|
|
2014
2586
|
}
|
|
2587
|
+
/**
|
|
2588
|
+
* Applies retry delay based on the prompt's retry strategy
|
|
2589
|
+
*/
|
|
2590
|
+
/**
|
|
2591
|
+
* Calculates retry delay for rate limit and other retriable errors.
|
|
2592
|
+
* Uses the prompt's RetryStrategy and can respect suggested delays from provider.
|
|
2593
|
+
*/
|
|
2015
2594
|
calculateRetryDelay(prompt, attemptNumber, suggestedDelaySeconds) {
|
|
2595
|
+
// Use provider's suggested delay if available
|
|
2016
2596
|
if (suggestedDelaySeconds && suggestedDelaySeconds > 0) {
|
|
2017
|
-
return suggestedDelaySeconds * 1000;
|
|
2597
|
+
return suggestedDelaySeconds * 1000; // Convert to milliseconds
|
|
2018
2598
|
}
|
|
2019
|
-
const baseDelay = prompt.RetryDelayMS || 1000;
|
|
2599
|
+
const baseDelay = prompt.RetryDelayMS || 1000; // Default 1 second
|
|
2020
2600
|
let delay = baseDelay;
|
|
2021
2601
|
switch (prompt.RetryStrategy) {
|
|
2022
2602
|
case 'Fixed':
|
|
@@ -2036,20 +2616,28 @@ class AIPromptRunner {
|
|
|
2036
2616
|
async applyRetryDelay(prompt, attemptNumber, suggestedDelaySeconds) {
|
|
2037
2617
|
const delay = this.calculateRetryDelay(prompt, attemptNumber, suggestedDelaySeconds);
|
|
2038
2618
|
const delaySeconds = (delay / 1000).toFixed(1);
|
|
2039
|
-
|
|
2619
|
+
LogStatus(` Waiting ${delaySeconds}s before retry (strategy: ${prompt.RetryStrategy || 'Fixed'})...`);
|
|
2040
2620
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
2041
2621
|
}
|
|
2622
|
+
/**
|
|
2623
|
+
* Filters out all candidates from a vendor when a vendor-level error occurs.
|
|
2624
|
+
* Vendor-level errors affect all models from that vendor:
|
|
2625
|
+
* - Authentication: Invalid API key
|
|
2626
|
+
* - VendorValidationError: API schema/validation requirements
|
|
2627
|
+
*/
|
|
2042
2628
|
filterVendorCandidates(errorType, currentVendorId, allCandidates) {
|
|
2043
2629
|
if (errorType !== 'Authentication' && errorType !== 'VendorValidationError') {
|
|
2044
|
-
return allCandidates;
|
|
2630
|
+
return allCandidates; // No filtering needed for non-vendor-level errors
|
|
2045
2631
|
}
|
|
2046
2632
|
const failedVendorId = currentVendorId || 'default';
|
|
2047
2633
|
const beforeCount = allCandidates.length;
|
|
2634
|
+
// Filter out ALL candidates from this vendor
|
|
2048
2635
|
const filteredCandidates = allCandidates.filter(c => (c.vendorId || 'default') !== failedVendorId);
|
|
2049
2636
|
const removedCount = beforeCount - filteredCandidates.length;
|
|
2050
2637
|
if (removedCount > 0) {
|
|
2051
|
-
const vendorName =
|
|
2638
|
+
const vendorName = AIEngine.Instance.Vendors.find(v => v.ID === failedVendorId)?.Name || failedVendorId;
|
|
2052
2639
|
const remainingCount = filteredCandidates.length;
|
|
2640
|
+
// Log appropriate message based on error type
|
|
2053
2641
|
let reason;
|
|
2054
2642
|
let icon;
|
|
2055
2643
|
if (errorType === 'Authentication') {
|
|
@@ -2068,32 +2656,47 @@ class AIPromptRunner {
|
|
|
2068
2656
|
}
|
|
2069
2657
|
return filteredCandidates;
|
|
2070
2658
|
}
|
|
2659
|
+
/**
|
|
2660
|
+
* Handles rate limit errors by retrying the same model/vendor with backoff.
|
|
2661
|
+
* Returns true if the caller should continue (retry), false if should proceed to failover.
|
|
2662
|
+
*/
|
|
2071
2663
|
async handleRateLimitRetry(errorAnalysis, currentModel, currentVendorId, failoverAttempts, prompt, attemptNumber, maxAttempts, failoverAttempt) {
|
|
2072
2664
|
const isRateLimit = errorAnalysis.errorType === 'RateLimit';
|
|
2073
2665
|
if (!isRateLimit) {
|
|
2074
|
-
return false;
|
|
2666
|
+
return false; // Not a rate limit error
|
|
2075
2667
|
}
|
|
2668
|
+
// Count how many times we've retried this specific model/vendor for rate limits
|
|
2076
2669
|
const rateLimitRetryCount = failoverAttempts.filter(a => a.modelId === currentModel.ID &&
|
|
2077
2670
|
a.vendorId === currentVendorId &&
|
|
2078
2671
|
a.errorType === 'RateLimit').length;
|
|
2672
|
+
// Use MaxRetries from prompt configuration, default to 3 if not set
|
|
2079
2673
|
const maxRetries = prompt.MaxRetries ?? 3;
|
|
2674
|
+
// Retry up to MaxRetries times before giving up and failing over
|
|
2080
2675
|
const shouldRetry = rateLimitRetryCount <= maxRetries;
|
|
2081
2676
|
if (shouldRetry) {
|
|
2082
2677
|
const modelName = currentModel.Name;
|
|
2083
2678
|
const vendorName = currentVendorId
|
|
2084
|
-
?
|
|
2679
|
+
? AIEngine.Instance.Vendors.find(v => v.ID === currentVendorId)?.Name || 'default'
|
|
2085
2680
|
: 'default';
|
|
2086
2681
|
this.logStatus(` ⏳ Rate limit hit - retrying ${modelName} (${vendorName}) with backoff (attempt ${rateLimitRetryCount}/${maxRetries})`, true);
|
|
2087
2682
|
this.logFailoverAttempt(prompt.ID, failoverAttempt, true);
|
|
2683
|
+
// Apply backoff delay before retry
|
|
2088
2684
|
if (attemptNumber < maxAttempts) {
|
|
2089
2685
|
await this.applyRetryDelay(prompt, rateLimitRetryCount, errorAnalysis.suggestedRetryDelaySeconds);
|
|
2090
2686
|
}
|
|
2091
|
-
return true;
|
|
2687
|
+
return true; // Signal to continue with same model/vendor
|
|
2092
2688
|
}
|
|
2093
|
-
return false;
|
|
2689
|
+
return false; // Too many retries, proceed to failover
|
|
2094
2690
|
}
|
|
2691
|
+
/**
|
|
2692
|
+
* Processes a failover error (either from catch block or from failed ChatResult).
|
|
2693
|
+
* Handles vendor filtering, rate limit retries, fatal error detection, and failover logic.
|
|
2694
|
+
*
|
|
2695
|
+
* @returns Decision object indicating whether to retry same model, continue to next candidate, or stop
|
|
2696
|
+
*/
|
|
2095
2697
|
async processFailoverError(error, errorInfo, candidate, attemptStartTime, attemptIndex, allCandidates, failoverAttempts, prompt, failoverConfig) {
|
|
2096
2698
|
const attemptDuration = Date.now() - attemptStartTime;
|
|
2699
|
+
// Create failover attempt record
|
|
2097
2700
|
const failoverAttempt = {
|
|
2098
2701
|
attemptNumber: attemptIndex + 1,
|
|
2099
2702
|
modelId: candidate.model.ID,
|
|
@@ -2104,17 +2707,20 @@ class AIPromptRunner {
|
|
|
2104
2707
|
timestamp: new Date()
|
|
2105
2708
|
};
|
|
2106
2709
|
failoverAttempts.push(failoverAttempt);
|
|
2710
|
+
// Vendor-level errors: filter out all candidates from this vendor
|
|
2107
2711
|
let updatedCandidates = allCandidates;
|
|
2108
2712
|
if (errorInfo.errorType === 'Authentication' || errorInfo.errorType === 'VendorValidationError') {
|
|
2109
2713
|
updatedCandidates = this.filterVendorCandidates(errorInfo.errorType, candidate.vendorId, allCandidates);
|
|
2110
2714
|
}
|
|
2111
2715
|
const isLastCandidate = attemptIndex === updatedCandidates.length - 1;
|
|
2716
|
+
// Fatal errors: stop immediately
|
|
2112
2717
|
if (errorInfo.severity === 'Fatal') {
|
|
2113
2718
|
const errorMessage = error?.message || 'Unknown error';
|
|
2114
|
-
|
|
2719
|
+
LogErrorEx(`Stopping failover: Fatal error (${errorInfo.errorType}): ${errorMessage}`);
|
|
2115
2720
|
this.logFailoverAttempt(prompt.ID, failoverAttempt, false);
|
|
2116
2721
|
return { shouldRetry: false, shouldContinue: false, updatedCandidates };
|
|
2117
2722
|
}
|
|
2723
|
+
// Check errorScope filter if configured
|
|
2118
2724
|
if (failoverConfig.errorScope && failoverConfig.errorScope !== 'All') {
|
|
2119
2725
|
const matchesScope = this.errorMatchesScope(errorInfo.errorType, failoverConfig.errorScope);
|
|
2120
2726
|
if (!matchesScope) {
|
|
@@ -2122,27 +2728,38 @@ class AIPromptRunner {
|
|
|
2122
2728
|
return { shouldRetry: false, shouldContinue: false, updatedCandidates };
|
|
2123
2729
|
}
|
|
2124
2730
|
}
|
|
2731
|
+
// Rate limit errors: check if we should retry the same model before failing over
|
|
2125
2732
|
if (errorInfo.errorType === 'RateLimit') {
|
|
2126
2733
|
const shouldRetry = await this.handleRateLimitRetry(errorInfo, candidate.model, candidate.vendorId, failoverAttempts, prompt, attemptIndex, updatedCandidates.length, failoverAttempt);
|
|
2127
2734
|
if (shouldRetry) {
|
|
2128
2735
|
return { shouldRetry: true, shouldContinue: false, updatedCandidates };
|
|
2129
2736
|
}
|
|
2130
2737
|
}
|
|
2738
|
+
// If this is the last candidate, we're done
|
|
2131
2739
|
if (isLastCandidate) {
|
|
2132
2740
|
this.logFailoverAttempt(prompt.ID, failoverAttempt, false);
|
|
2133
2741
|
return { shouldRetry: false, shouldContinue: false, updatedCandidates };
|
|
2134
2742
|
}
|
|
2743
|
+
// Log and signal to continue to next candidate
|
|
2135
2744
|
this.logFailoverAttempt(prompt.ID, failoverAttempt, true);
|
|
2136
2745
|
return { shouldRetry: false, shouldContinue: true, updatedCandidates };
|
|
2137
2746
|
}
|
|
2747
|
+
/**
|
|
2748
|
+
* Transitions to the next failover candidate.
|
|
2749
|
+
* Returns the next candidate info or null if no candidates are available.
|
|
2750
|
+
*/
|
|
2138
2751
|
async transitionToNextCandidate(currentModel, currentVendorId, failoverConfig, allCandidates, failoverAttempts, promptId, failoverAttempt, attemptNumber) {
|
|
2752
|
+
// Select next candidate using failover strategy
|
|
2139
2753
|
const nextCandidates = this.selectFailoverCandidates(currentModel, currentVendorId, failoverConfig.strategy, failoverConfig.modelStrategy, allCandidates, failoverAttempts);
|
|
2140
2754
|
if (nextCandidates.length === 0) {
|
|
2755
|
+
// No more candidates available
|
|
2141
2756
|
this.logFailoverAttempt(promptId, failoverAttempt, false);
|
|
2142
2757
|
return null;
|
|
2143
2758
|
}
|
|
2144
2759
|
const nextCandidate = nextCandidates[0];
|
|
2760
|
+
// Log the successful transition
|
|
2145
2761
|
this.logFailoverAttempt(promptId, failoverAttempt, true);
|
|
2762
|
+
// Apply delay before next attempt (if not the last attempt)
|
|
2146
2763
|
if (attemptNumber < failoverConfig.maxAttempts) {
|
|
2147
2764
|
const delay = this.calculateFailoverDelay(attemptNumber, failoverConfig.delaySeconds);
|
|
2148
2765
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
@@ -2155,6 +2772,9 @@ class AIPromptRunner {
|
|
|
2155
2772
|
supportsEffortLevel: nextCandidate.supportsEffortLevel || false
|
|
2156
2773
|
};
|
|
2157
2774
|
}
|
|
2775
|
+
/**
|
|
2776
|
+
* Provides a human-readable description of the validation decision
|
|
2777
|
+
*/
|
|
2158
2778
|
getValidationDecisionDescription(finalSuccess, totalAttempts, validationBehavior) {
|
|
2159
2779
|
if (finalSuccess) {
|
|
2160
2780
|
return totalAttempts === 1
|
|
@@ -2174,6 +2794,9 @@ class AIPromptRunner {
|
|
|
2174
2794
|
}
|
|
2175
2795
|
}
|
|
2176
2796
|
}
|
|
2797
|
+
/**
|
|
2798
|
+
* Generates a JSON schema from an example object for validation
|
|
2799
|
+
*/
|
|
2177
2800
|
generateSchemaFromExample(example) {
|
|
2178
2801
|
if (typeof example !== 'object' || example === null) {
|
|
2179
2802
|
return { type: 'object' };
|
|
@@ -2182,18 +2805,26 @@ class AIPromptRunner {
|
|
|
2182
2805
|
type: 'object',
|
|
2183
2806
|
properties: {},
|
|
2184
2807
|
required: [],
|
|
2185
|
-
additionalProperties: true,
|
|
2808
|
+
additionalProperties: true, // Allow additional properties for flexibility with examples
|
|
2186
2809
|
};
|
|
2810
|
+
// Check if this entire object appears to be a placeholder/example
|
|
2187
2811
|
const isPlaceholderObject = this.isObjectLikelyPlaceholder(example);
|
|
2188
2812
|
for (const [key, value] of Object.entries(example)) {
|
|
2813
|
+
// For placeholder objects, generate very permissive schemas
|
|
2189
2814
|
if (isPlaceholderObject) {
|
|
2815
|
+
// Don't define specific properties for placeholder objects
|
|
2816
|
+
// Just indicate it should be an object with any properties
|
|
2190
2817
|
schema.properties = {};
|
|
2191
2818
|
schema.required = [];
|
|
2192
2819
|
break;
|
|
2193
2820
|
}
|
|
2821
|
+
// Check if the key ends with '?' to indicate optional property (TypeScript style)
|
|
2194
2822
|
const isOptional = key.endsWith('?');
|
|
2195
2823
|
const cleanKey = isOptional ? key.slice(0, -1) : key;
|
|
2196
2824
|
schema.properties[cleanKey] = this.generateSchemaForValue(value);
|
|
2825
|
+
// Don't make fields required if:
|
|
2826
|
+
// 1. They're marked as optional with '?'
|
|
2827
|
+
// 2. They look like placeholder/example values
|
|
2197
2828
|
const isPlaceholder = this.isLikelyPlaceholder(cleanKey, value);
|
|
2198
2829
|
if (!isOptional && !isPlaceholder) {
|
|
2199
2830
|
schema.required.push(cleanKey);
|
|
@@ -2201,11 +2832,16 @@ class AIPromptRunner {
|
|
|
2201
2832
|
}
|
|
2202
2833
|
return schema;
|
|
2203
2834
|
}
|
|
2835
|
+
/**
|
|
2836
|
+
* Detects if a key/value pair looks like a placeholder or example value
|
|
2837
|
+
*/
|
|
2204
2838
|
isLikelyPlaceholder(key, value) {
|
|
2839
|
+
// Check if key contains common placeholder patterns
|
|
2205
2840
|
const placeholderKeyPatterns = /^(param|example|placeholder|sample|dummy|test)/i;
|
|
2206
2841
|
if (placeholderKeyPatterns.test(key)) {
|
|
2207
2842
|
return true;
|
|
2208
2843
|
}
|
|
2844
|
+
// Check if string value contains common placeholder text
|
|
2209
2845
|
if (typeof value === 'string') {
|
|
2210
2846
|
const placeholderValuePatterns = /(goes here|placeholder|example|sample value|value\d+|UUID|your .* here|insert .* here)/i;
|
|
2211
2847
|
if (placeholderValuePatterns.test(value)) {
|
|
@@ -2214,15 +2850,23 @@ class AIPromptRunner {
|
|
|
2214
2850
|
}
|
|
2215
2851
|
return false;
|
|
2216
2852
|
}
|
|
2853
|
+
/**
|
|
2854
|
+
* Detects if an entire object looks like it contains only placeholder/example data
|
|
2855
|
+
*/
|
|
2217
2856
|
isObjectLikelyPlaceholder(obj) {
|
|
2218
2857
|
if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
|
|
2219
2858
|
return false;
|
|
2220
2859
|
}
|
|
2221
2860
|
const entries = Object.entries(obj);
|
|
2861
|
+
// If object has placeholder-like keys (param1, param2, etc)
|
|
2222
2862
|
const hasPlaceholderKeys = entries.some(([key]) => /^(param\d+|key\d+|value\d+|example\d+|placeholder\d+)$/i.test(key));
|
|
2863
|
+
// If all values are simple placeholders
|
|
2223
2864
|
const allValuesArePlaceholders = entries.every(([key, value]) => this.isLikelyPlaceholder(key, value));
|
|
2224
2865
|
return hasPlaceholderKeys || allValuesArePlaceholders;
|
|
2225
2866
|
}
|
|
2867
|
+
/**
|
|
2868
|
+
* Generates schema for a specific value type
|
|
2869
|
+
*/
|
|
2226
2870
|
generateSchemaForValue(value) {
|
|
2227
2871
|
if (value === null) {
|
|
2228
2872
|
return { type: 'null' };
|
|
@@ -2240,7 +2884,7 @@ class AIPromptRunner {
|
|
|
2240
2884
|
return {
|
|
2241
2885
|
type: 'array',
|
|
2242
2886
|
items: this.generateSchemaForValue(value[0]),
|
|
2243
|
-
minItems: 0,
|
|
2887
|
+
minItems: 0, // Don't require minimum items for example arrays
|
|
2244
2888
|
};
|
|
2245
2889
|
}
|
|
2246
2890
|
else {
|
|
@@ -2251,9 +2895,19 @@ class AIPromptRunner {
|
|
|
2251
2895
|
return this.generateSchemaFromExample(value);
|
|
2252
2896
|
}
|
|
2253
2897
|
default:
|
|
2254
|
-
return { type: 'string' };
|
|
2255
|
-
}
|
|
2256
|
-
}
|
|
2898
|
+
return { type: 'string' }; // Fallback
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
/**
|
|
2902
|
+
* Enhanced parsing and validation with detailed error reporting and JSON repair capabilities.
|
|
2903
|
+
*
|
|
2904
|
+
* @param modelResult - The raw result from the AI model
|
|
2905
|
+
* @param prompt - The AI prompt entity containing configuration
|
|
2906
|
+
* @param skipValidation - Whether to skip validation
|
|
2907
|
+
* @param cleanValidationSyntax - Whether to clean validation syntax from results
|
|
2908
|
+
* @param params - Optional prompt parameters containing additional configuration like attemptJSONRepair
|
|
2909
|
+
* @returns Parsed result with optional validation results and errors
|
|
2910
|
+
*/
|
|
2257
2911
|
async parseAndValidateResultEnhanced(modelResult, prompt, skipValidation = false, cleanValidationSyntax = false, currentPromptRun, params) {
|
|
2258
2912
|
const validationErrors = [];
|
|
2259
2913
|
let rawOutput;
|
|
@@ -2265,6 +2919,7 @@ class AIPromptRunner {
|
|
|
2265
2919
|
if (!rawOutput) {
|
|
2266
2920
|
throw new Error('No output received from model');
|
|
2267
2921
|
}
|
|
2922
|
+
// Parse based on output type
|
|
2268
2923
|
let parsedResult = rawOutput;
|
|
2269
2924
|
try {
|
|
2270
2925
|
switch (prompt.OutputType) {
|
|
@@ -2288,24 +2943,27 @@ class AIPromptRunner {
|
|
|
2288
2943
|
}
|
|
2289
2944
|
}
|
|
2290
2945
|
catch (parseError) {
|
|
2291
|
-
|
|
2946
|
+
// Type parsing failed
|
|
2947
|
+
const validationResult = new ValidationResult();
|
|
2292
2948
|
validationResult.Success = false;
|
|
2293
|
-
const error = new
|
|
2949
|
+
const error = new ValidationErrorInfo('parseAndValidateResultEnhanced', `Invalid OutputExample JSON: ${parseError.message}`, rawOutput, ValidationErrorType.Failure);
|
|
2294
2950
|
validationErrors.push(error);
|
|
2295
2951
|
validationResult.Errors = validationErrors;
|
|
2296
2952
|
return { result: rawOutput, validationResult, validationErrors };
|
|
2297
2953
|
}
|
|
2954
|
+
// Perform JSON schema validation for object types
|
|
2298
2955
|
if (!skipValidation && prompt.OutputExample && prompt.OutputType === 'object' && parsedResult) {
|
|
2299
2956
|
try {
|
|
2300
2957
|
const schemaValidationErrors = await this.validateAgainstSchema(parsedResult, prompt.OutputExample, prompt.ID);
|
|
2301
2958
|
validationErrors.push(...schemaValidationErrors);
|
|
2302
2959
|
}
|
|
2303
2960
|
catch (schemaError) {
|
|
2304
|
-
const error = new
|
|
2961
|
+
const error = new ValidationErrorInfo('schema', `Schema validation failed: ${schemaError.message}`, undefined, ValidationErrorType.Failure);
|
|
2305
2962
|
validationErrors.push(error);
|
|
2306
2963
|
}
|
|
2307
2964
|
}
|
|
2308
|
-
|
|
2965
|
+
// Create validation result
|
|
2966
|
+
const validationResult = new ValidationResult();
|
|
2309
2967
|
validationResult.Success = validationErrors.length === 0;
|
|
2310
2968
|
validationResult.Errors = validationErrors;
|
|
2311
2969
|
return { result: parsedResult, validationResult, validationErrors };
|
|
@@ -2320,10 +2978,11 @@ class AIPromptRunner {
|
|
|
2320
2978
|
},
|
|
2321
2979
|
maxErrorLength: params?.maxErrorLength
|
|
2322
2980
|
});
|
|
2323
|
-
|
|
2981
|
+
// Handle validation behavior
|
|
2982
|
+
const validationResult = new ValidationResult();
|
|
2324
2983
|
validationResult.Success = false;
|
|
2325
2984
|
validationResult.Errors = validationErrors.length > 0 ? validationErrors : [
|
|
2326
|
-
new
|
|
2985
|
+
new ValidationErrorInfo('general', error.message, undefined, ValidationErrorType.Failure)
|
|
2327
2986
|
];
|
|
2328
2987
|
switch (prompt.ValidationBehavior) {
|
|
2329
2988
|
case 'Strict':
|
|
@@ -2342,26 +3001,51 @@ class AIPromptRunner {
|
|
|
2342
3001
|
return { result: modelResult.data?.choices?.[0]?.message?.content, validationResult, validationErrors: validationResult.Errors };
|
|
2343
3002
|
case 'None':
|
|
2344
3003
|
default:
|
|
3004
|
+
// For None, we still return the validation result but mark as successful
|
|
2345
3005
|
validationResult.Success = true;
|
|
2346
3006
|
return { result: modelResult.data?.choices?.[0]?.message?.content, validationResult, validationErrors: [] };
|
|
2347
3007
|
}
|
|
2348
3008
|
}
|
|
2349
3009
|
}
|
|
3010
|
+
/**
|
|
3011
|
+
* Parses a string output value.
|
|
3012
|
+
*
|
|
3013
|
+
* @param rawOutput - The raw output from the model
|
|
3014
|
+
* @returns The parsed string value
|
|
3015
|
+
*/
|
|
2350
3016
|
parseStringOutput(rawOutput) {
|
|
2351
3017
|
return rawOutput.toString();
|
|
2352
3018
|
}
|
|
3019
|
+
/**
|
|
3020
|
+
* Parses a number output value with validation.
|
|
3021
|
+
*
|
|
3022
|
+
* @param rawOutput - The raw output from the model
|
|
3023
|
+
* @param skipValidation - Whether to skip validation
|
|
3024
|
+
* @param validationErrors - Array to collect validation errors
|
|
3025
|
+
* @returns The parsed number value
|
|
3026
|
+
* @throws Error if the value cannot be parsed as a number and validation is enabled
|
|
3027
|
+
*/
|
|
2353
3028
|
parseNumberOutput(rawOutput, skipValidation, validationErrors) {
|
|
2354
3029
|
const numberResult = parseFloat(rawOutput);
|
|
2355
3030
|
if (isNaN(numberResult)) {
|
|
2356
3031
|
if (!skipValidation) {
|
|
2357
|
-
const error = new
|
|
3032
|
+
const error = new ValidationErrorInfo('output', `Expected number output but got: ${rawOutput}`, rawOutput, ValidationErrorType.Failure);
|
|
2358
3033
|
validationErrors.push(error);
|
|
2359
3034
|
throw new Error(error.Message);
|
|
2360
3035
|
}
|
|
2361
|
-
return numberResult;
|
|
3036
|
+
return numberResult; // Will be NaN if skipValidation is true
|
|
2362
3037
|
}
|
|
2363
3038
|
return numberResult;
|
|
2364
3039
|
}
|
|
3040
|
+
/**
|
|
3041
|
+
* Parses a boolean output value with flexible input handling.
|
|
3042
|
+
*
|
|
3043
|
+
* @param rawOutput - The raw output from the model
|
|
3044
|
+
* @param skipValidation - Whether to skip validation
|
|
3045
|
+
* @param validationErrors - Array to collect validation errors
|
|
3046
|
+
* @returns The parsed boolean value
|
|
3047
|
+
* @throws Error if the value cannot be parsed as a boolean and validation is enabled
|
|
3048
|
+
*/
|
|
2365
3049
|
parseBooleanOutput(rawOutput, skipValidation, validationErrors) {
|
|
2366
3050
|
const lowerOutput = rawOutput.toLowerCase().trim();
|
|
2367
3051
|
if (['true', 'yes', '1'].includes(lowerOutput)) {
|
|
@@ -2371,46 +3055,82 @@ class AIPromptRunner {
|
|
|
2371
3055
|
return false;
|
|
2372
3056
|
}
|
|
2373
3057
|
else if (!skipValidation) {
|
|
2374
|
-
const error = new
|
|
3058
|
+
const error = new ValidationErrorInfo('output', `Expected boolean output but got: ${rawOutput}`, rawOutput, ValidationErrorType.Failure);
|
|
2375
3059
|
validationErrors.push(error);
|
|
2376
3060
|
throw new Error(error.Message);
|
|
2377
3061
|
}
|
|
2378
|
-
return false;
|
|
2379
|
-
}
|
|
3062
|
+
return false; // Default to false if skipValidation is true
|
|
3063
|
+
}
|
|
3064
|
+
/**
|
|
3065
|
+
* Parses a date output value with validation.
|
|
3066
|
+
*
|
|
3067
|
+
* @param rawOutput - The raw output from the model
|
|
3068
|
+
* @param skipValidation - Whether to skip validation
|
|
3069
|
+
* @param validationErrors - Array to collect validation errors
|
|
3070
|
+
* @returns The parsed Date value
|
|
3071
|
+
* @throws Error if the value cannot be parsed as a date and validation is enabled
|
|
3072
|
+
*/
|
|
2380
3073
|
parseDateOutput(rawOutput, skipValidation, validationErrors) {
|
|
2381
3074
|
const dateResult = new Date(rawOutput);
|
|
2382
3075
|
if (isNaN(dateResult.getTime()) && !skipValidation) {
|
|
2383
|
-
const error = new
|
|
3076
|
+
const error = new ValidationErrorInfo('output', `Expected date output but got: ${rawOutput}`, rawOutput, ValidationErrorType.Failure);
|
|
2384
3077
|
validationErrors.push(error);
|
|
2385
3078
|
throw new Error(error.Message);
|
|
2386
3079
|
}
|
|
2387
3080
|
return dateResult;
|
|
2388
3081
|
}
|
|
3082
|
+
/**
|
|
3083
|
+
* Parses an object (JSON) output value with optional repair capabilities.
|
|
3084
|
+
*
|
|
3085
|
+
* @param rawOutput - The raw output from the model
|
|
3086
|
+
* @param prompt - The AI prompt entity containing configuration
|
|
3087
|
+
* @param skipValidation - Whether to skip validation
|
|
3088
|
+
* @param cleanValidationSyntax - Whether to clean validation syntax
|
|
3089
|
+
* @param validationErrors - Array to collect validation errors
|
|
3090
|
+
* @param params - Optional prompt parameters containing attemptJSONRepair flag
|
|
3091
|
+
* @returns The parsed object value
|
|
3092
|
+
* @throws Error if the value cannot be parsed as JSON and validation is enabled
|
|
3093
|
+
*/
|
|
2389
3094
|
async parseObjectOutput(rawOutput, prompt, skipValidation, cleanValidationSyntax, validationErrors, currentPromptRun, params) {
|
|
2390
3095
|
let parsedResult;
|
|
2391
3096
|
try {
|
|
2392
|
-
|
|
3097
|
+
// First attempt: Use CleanJSON to handle common JSON issues
|
|
3098
|
+
parsedResult = JSON.parse(CleanJSON(rawOutput));
|
|
2393
3099
|
}
|
|
2394
3100
|
catch (jsonError) {
|
|
3101
|
+
// If attemptJSONRepair is enabled and we're dealing with object output
|
|
2395
3102
|
if (params?.attemptJSONRepair && prompt.OutputType === 'object') {
|
|
2396
3103
|
parsedResult = await this.attemptJSONRepair(rawOutput, jsonError, params, currentPromptRun);
|
|
2397
3104
|
}
|
|
2398
3105
|
else {
|
|
3106
|
+
// Original error handling
|
|
2399
3107
|
if (!skipValidation) {
|
|
2400
|
-
const error = new
|
|
3108
|
+
const error = new ValidationErrorInfo('output', `Expected JSON object but got invalid JSON: ${rawOutput}`, rawOutput, ValidationErrorType.Failure);
|
|
2401
3109
|
validationErrors.push(error);
|
|
2402
3110
|
throw new Error(error.Message);
|
|
2403
3111
|
}
|
|
2404
|
-
return rawOutput;
|
|
3112
|
+
return rawOutput; // Return raw output if skipping validation
|
|
2405
3113
|
}
|
|
2406
3114
|
}
|
|
3115
|
+
// Clean validation syntax if needed
|
|
2407
3116
|
if (parsedResult && (cleanValidationSyntax || (!skipValidation && prompt.OutputExample))) {
|
|
2408
|
-
const validator = new
|
|
3117
|
+
const validator = new JSONValidator();
|
|
2409
3118
|
parsedResult = validator.cleanValidationSyntax(parsedResult);
|
|
2410
3119
|
}
|
|
2411
3120
|
return parsedResult;
|
|
2412
3121
|
}
|
|
3122
|
+
/**
|
|
3123
|
+
* Attempts to repair malformed JSON using a two-step process.
|
|
3124
|
+
*
|
|
3125
|
+
* @param rawOutput - The malformed JSON string
|
|
3126
|
+
* @param originalError - The original parsing error
|
|
3127
|
+
* @param params - Prompt parameters containing contextUser
|
|
3128
|
+
* @returns The repaired and parsed JSON object
|
|
3129
|
+
* @throws Error if JSON repair fails
|
|
3130
|
+
*/
|
|
2413
3131
|
async attemptJSONRepair(rawOutput, originalError, params, currentPromptRun) {
|
|
3132
|
+
// Step 0: First, see if the raw output has any { } [ ] characters at all
|
|
3133
|
+
// if not, we KNOW it is not JSON and we should not attempt to repair it
|
|
2414
3134
|
if (!rawOutput.includes('{') && !rawOutput.includes('[')) {
|
|
2415
3135
|
this.logError(new Error('Raw output does not contain any JSON-like characters'), {
|
|
2416
3136
|
category: 'JSONRepairSkipped',
|
|
@@ -2422,11 +3142,13 @@ class AIPromptRunner {
|
|
|
2422
3142
|
});
|
|
2423
3143
|
throw new Error(`JSON repair skipped: raw output does not contain JSON-like characters. Original error: ${originalError.message}`);
|
|
2424
3144
|
}
|
|
3145
|
+
// Step 1: Try JSON5 parsing
|
|
2425
3146
|
try {
|
|
2426
3147
|
this.logStatus(' 🔧 Attempting JSON repair with JSON5...', true, params);
|
|
3148
|
+
// first try to clean JSON in case we have it in a markdown block
|
|
2427
3149
|
let jsonToParse = rawOutput;
|
|
2428
3150
|
try {
|
|
2429
|
-
jsonToParse =
|
|
3151
|
+
jsonToParse = CleanJSON(rawOutput);
|
|
2430
3152
|
}
|
|
2431
3153
|
catch (cleanError) {
|
|
2432
3154
|
if (params.verbose) {
|
|
@@ -2447,14 +3169,17 @@ class AIPromptRunner {
|
|
|
2447
3169
|
return json5Result;
|
|
2448
3170
|
}
|
|
2449
3171
|
catch (json5Error) {
|
|
3172
|
+
// Step 2: Use AI to repair the JSON
|
|
2450
3173
|
if (params.verbose) {
|
|
2451
3174
|
this.logStatus(' 🤖 JSON5 failed, attempting AI-based JSON repair...', true, params);
|
|
2452
3175
|
}
|
|
2453
3176
|
try {
|
|
2454
|
-
|
|
3177
|
+
// Find the "Repair JSON" prompt in the "MJ: System" category
|
|
3178
|
+
const repairPrompt = AIEngine.Instance.Prompts.find(p => p.Name.trim().toLowerCase() === 'repair json' && p.Category.trim().toLowerCase() === 'mj: system');
|
|
2455
3179
|
if (!repairPrompt) {
|
|
2456
3180
|
throw new Error('Repair JSON prompt not found in MJ: System category');
|
|
2457
3181
|
}
|
|
3182
|
+
// Run the repair prompt
|
|
2458
3183
|
const repairResult = await this.ExecutePrompt({
|
|
2459
3184
|
parentPromptRunId: currentPromptRun.ID,
|
|
2460
3185
|
agentRunId: currentPromptRun.AgentRunID,
|
|
@@ -2464,19 +3189,23 @@ class AIPromptRunner {
|
|
|
2464
3189
|
ERROR_MESSAGE: originalError.message,
|
|
2465
3190
|
MALFORMED_JSON: rawOutput
|
|
2466
3191
|
},
|
|
2467
|
-
skipValidation: true
|
|
3192
|
+
skipValidation: true // don't want to validate as this would cause recursive infinity scenario if the JSON is invalid. Just one shot, fix or no fix
|
|
2468
3193
|
});
|
|
2469
3194
|
if (!repairResult.success || !repairResult.result) {
|
|
2470
3195
|
throw new Error('AI-based JSON repair failed' + (repairResult.errorMessage ? `: ${repairResult.errorMessage}` : ''));
|
|
2471
3196
|
}
|
|
3197
|
+
// if we get here we have the text result in the reapairResult.result so let's try to parse it
|
|
2472
3198
|
const repairedJSON = JSON.parse(repairResult.result);
|
|
3199
|
+
// make sure repairedJSON is not this object: { error: "not_json" } -- if it is that means the LLM said it isn't JSOn
|
|
2473
3200
|
if (repairedJSON && typeof repairedJSON === 'object' && Object.keys(repairedJSON).length === 1 && repairedJSON.error?.trim().toLowerCase() === 'not_json') {
|
|
2474
3201
|
throw new Error('AI-based JSON repair returned a non-JSON response indicating it could not repair the JSON');
|
|
2475
3202
|
}
|
|
3203
|
+
// if we get here, we successfully repaired the JSON!!!
|
|
2476
3204
|
this.logStatus(' ✅ AI successfully repaired the JSON', true, params);
|
|
2477
3205
|
return repairedJSON;
|
|
2478
3206
|
}
|
|
2479
3207
|
catch (aiRepairError) {
|
|
3208
|
+
// Both repair attempts failed
|
|
2480
3209
|
if (params.verbose) {
|
|
2481
3210
|
this.logError(aiRepairError, {
|
|
2482
3211
|
category: 'JSONRepairFailed',
|
|
@@ -2493,48 +3222,115 @@ class AIPromptRunner {
|
|
|
2493
3222
|
}
|
|
2494
3223
|
}
|
|
2495
3224
|
}
|
|
3225
|
+
/**
|
|
3226
|
+
* Validates parsed result against JSON schema derived from OutputExample
|
|
3227
|
+
*/
|
|
2496
3228
|
async validateAgainstSchema(parsedResult, outputExample, promptId) {
|
|
2497
3229
|
const validationErrors = [];
|
|
2498
3230
|
try {
|
|
3231
|
+
// Parse the output example
|
|
2499
3232
|
let exampleObject;
|
|
2500
3233
|
try {
|
|
2501
3234
|
exampleObject = JSON.parse(outputExample);
|
|
2502
3235
|
}
|
|
2503
3236
|
catch (parseError) {
|
|
2504
|
-
const error = new
|
|
3237
|
+
const error = new ValidationErrorInfo('outputExample', `Invalid OutputExample JSON: ${parseError.message}`, outputExample, ValidationErrorType.Failure);
|
|
2505
3238
|
validationErrors.push(error);
|
|
2506
3239
|
return validationErrors;
|
|
2507
3240
|
}
|
|
3241
|
+
// Use the JSONValidator to validate against the example
|
|
2508
3242
|
const validationResult = this._jsonValidator.validate(parsedResult, exampleObject);
|
|
2509
3243
|
validationErrors.push(...validationResult.Errors);
|
|
2510
3244
|
if (validationErrors.length !== 0) {
|
|
2511
|
-
|
|
3245
|
+
LogStatus(`⚠️ Validation found ${validationErrors.length} issues for prompt ${promptId}:`);
|
|
2512
3246
|
validationErrors.forEach((error, index) => {
|
|
2513
|
-
|
|
3247
|
+
LogStatus(` ${index + 1}. ${error.Source}: ${error.Message}`);
|
|
2514
3248
|
});
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
}
|
|
3249
|
+
LogStatus(` Note: Validation syntax in OutputExample:`);
|
|
3250
|
+
LogStatus(` - '?' = optional field (e.g., "reasoning?": "...")`);
|
|
3251
|
+
LogStatus(` - '*' = required but any content (e.g., "payload*": {})`);
|
|
3252
|
+
LogStatus(` - ':type' = type validation (e.g., "age:number": 25)`);
|
|
3253
|
+
LogStatus(` - ':[N+]' = array length (e.g., "items:[2+]": [])`);
|
|
3254
|
+
}
|
|
3255
|
+
/* FUTURE IMPLEMENTATION - Keep this commented for reference
|
|
3256
|
+
// Get or create cached validator for this prompt using static cache
|
|
3257
|
+
let validator = AIPromptRunner._schemaCache.get(promptId);
|
|
3258
|
+
|
|
3259
|
+
if (!validator) {
|
|
3260
|
+
// Parse the output example
|
|
3261
|
+
let exampleObject: unknown;
|
|
3262
|
+
try {
|
|
3263
|
+
exampleObject = JSON.parse(outputExample);
|
|
3264
|
+
} catch (parseError) {
|
|
3265
|
+
const error = new ValidationErrorInfo('outputExample', `Invalid OutputExample JSON: ${parseError.message}`, outputExample, ValidationErrorType.Failure);
|
|
3266
|
+
validationErrors.push(error);
|
|
3267
|
+
return validationErrors;
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3270
|
+
// Generate schema from example
|
|
3271
|
+
const schema = this.generateSchemaFromExample(exampleObject);
|
|
3272
|
+
|
|
3273
|
+
// Compile and cache the validator
|
|
3274
|
+
try {
|
|
3275
|
+
validator = this._ajv.compile(schema);
|
|
3276
|
+
AIPromptRunner._schemaCache.set(promptId, validator);
|
|
3277
|
+
const cacheStats = AIPromptRunner.getSchemaCacheStats();
|
|
3278
|
+
LogStatus(`📋 Compiled and cached JSON schema for prompt ${promptId} (global cache size: ${cacheStats.size})`);
|
|
3279
|
+
} catch (compileError) {
|
|
3280
|
+
const error = new ValidationErrorInfo('schema', `Failed to compile schema: ${compileError.message}`, schema, ValidationErrorType.Failure);
|
|
3281
|
+
validationErrors.push(error);
|
|
3282
|
+
return validationErrors;
|
|
3283
|
+
}
|
|
3284
|
+
}
|
|
3285
|
+
|
|
3286
|
+
// Validate the result
|
|
3287
|
+
const isValid = validator(parsedResult);
|
|
3288
|
+
|
|
3289
|
+
if (!isValid && validator.errors) {
|
|
3290
|
+
for (const ajvError of validator.errors) {
|
|
3291
|
+
const fieldPath = ajvError.instancePath || ajvError.schemaPath || 'root';
|
|
3292
|
+
const message = `${ajvError.instancePath || 'root'}: ${ajvError.message}`;
|
|
3293
|
+
const error = new ValidationErrorInfo(fieldPath, message, ajvError.data, ValidationErrorType.Failure);
|
|
3294
|
+
validationErrors.push(error);
|
|
3295
|
+
}
|
|
3296
|
+
}
|
|
3297
|
+
|
|
3298
|
+
if (validationErrors.length === 0) {
|
|
3299
|
+
//LogStatus(`✅ Schema validation passed for prompt ${promptId}`);
|
|
3300
|
+
} else {
|
|
3301
|
+
LogStatus(`⚠️ Schema validation found ${validationErrors.length} potential issues for prompt ${promptId}:`);
|
|
3302
|
+
validationErrors.forEach((error, index) => {
|
|
3303
|
+
LogStatus(` ${index + 1}. ${error.Source}: ${error.Message}`);
|
|
3304
|
+
});
|
|
3305
|
+
// Log additional context to help with debugging
|
|
3306
|
+
LogStatus(` Note: The schema was generated from OutputExample. Consider:`);
|
|
3307
|
+
LogStatus(` - Mark optional properties with '?' suffix (e.g., "subAgent?": {...})`)
|
|
3308
|
+
LogStatus(` - Example values like "param1", "value1" are treated as placeholders`);
|
|
3309
|
+
}
|
|
3310
|
+
*/
|
|
2521
3311
|
}
|
|
2522
3312
|
catch (error) {
|
|
2523
|
-
const validationError = new
|
|
3313
|
+
const validationError = new ValidationErrorInfo('validation', `Unexpected validation error: ${error.message}`, undefined, ValidationErrorType.Failure);
|
|
2524
3314
|
validationErrors.push(validationError);
|
|
2525
3315
|
}
|
|
2526
3316
|
return validationErrors;
|
|
2527
3317
|
}
|
|
3318
|
+
/**
|
|
3319
|
+
* Updates the AIPromptRun entity with execution results
|
|
3320
|
+
*/
|
|
2528
3321
|
async updatePromptRun(promptRun, prompt, modelResult, parsedResult, endTime, executionTimeMS, validationAttempts, cumulativeTokens) {
|
|
2529
3322
|
try {
|
|
2530
3323
|
promptRun.CompletedAt = endTime;
|
|
2531
3324
|
promptRun.ExecutionTimeMS = executionTimeMS;
|
|
3325
|
+
// Determine what to save as the result
|
|
2532
3326
|
let resultToSave;
|
|
2533
3327
|
const rawResult = modelResult.data?.choices?.[0]?.message?.content || '';
|
|
2534
3328
|
if (parsedResult.result === undefined ||
|
|
2535
3329
|
parsedResult.result === null ||
|
|
2536
3330
|
(typeof parsedResult.result === 'string' && parsedResult.result.trim().length === 0)) {
|
|
3331
|
+
// Use raw result as fallback when parsed result is undefined, null, or empty string
|
|
2537
3332
|
resultToSave = rawResult;
|
|
3333
|
+
// Also set error message when we have to fall back to raw result
|
|
2538
3334
|
if (!promptRun.ErrorMessage) {
|
|
2539
3335
|
const validationErrors = parsedResult.validationResult?.Errors;
|
|
2540
3336
|
if (validationErrors && validationErrors.length > 0) {
|
|
@@ -2552,25 +3348,31 @@ class AIPromptRunner {
|
|
|
2552
3348
|
resultToSave = JSON.stringify(parsedResult.result);
|
|
2553
3349
|
}
|
|
2554
3350
|
promptRun.Result = resultToSave;
|
|
3351
|
+
// Extract token usage and cost - use cumulative if retries occurred
|
|
2555
3352
|
if (cumulativeTokens && validationAttempts && validationAttempts.length > 1) {
|
|
3353
|
+
// Multiple attempts occurred, use cumulative totals
|
|
2556
3354
|
promptRun.TokensPrompt = cumulativeTokens.promptTokens;
|
|
2557
3355
|
promptRun.TokensCompletion = cumulativeTokens.completionTokens;
|
|
2558
3356
|
promptRun.TokensUsed = cumulativeTokens.promptTokens + cumulativeTokens.completionTokens;
|
|
2559
3357
|
promptRun.Cost = cumulativeTokens.totalCost;
|
|
3358
|
+
// Cost currency from the last model result
|
|
2560
3359
|
if (modelResult.data?.usage?.costCurrency !== undefined) {
|
|
2561
3360
|
promptRun.CostCurrency = modelResult.data.usage.costCurrency;
|
|
2562
3361
|
}
|
|
2563
3362
|
}
|
|
2564
3363
|
else if (modelResult.data?.usage) {
|
|
3364
|
+
// Single attempt, use standard token tracking
|
|
2565
3365
|
promptRun.TokensUsed = modelResult.data.usage.totalTokens;
|
|
2566
3366
|
promptRun.TokensPrompt = modelResult.data.usage.promptTokens;
|
|
2567
3367
|
promptRun.TokensCompletion = modelResult.data.usage.completionTokens;
|
|
3368
|
+
// Save cost information if available
|
|
2568
3369
|
if (modelResult.data.usage.cost !== undefined) {
|
|
2569
3370
|
promptRun.Cost = modelResult.data.usage.cost;
|
|
2570
3371
|
}
|
|
2571
3372
|
if (modelResult.data.usage.costCurrency !== undefined) {
|
|
2572
3373
|
promptRun.CostCurrency = modelResult.data.usage.costCurrency;
|
|
2573
3374
|
}
|
|
3375
|
+
// Save timing information if available
|
|
2574
3376
|
if (modelResult.data.usage.queueTime !== undefined) {
|
|
2575
3377
|
promptRun.QueueTime = modelResult.data.usage.queueTime;
|
|
2576
3378
|
}
|
|
@@ -2581,14 +3383,18 @@ class AIPromptRunner {
|
|
|
2581
3383
|
promptRun.CompletionTime = modelResult.data.usage.completionTime;
|
|
2582
3384
|
}
|
|
2583
3385
|
}
|
|
3386
|
+
// Save model-specific response details if available
|
|
2584
3387
|
if (modelResult.modelSpecificResponseDetails) {
|
|
2585
3388
|
promptRun.ModelSpecificResponseDetails = JSON.stringify(modelResult.modelSpecificResponseDetails);
|
|
2586
3389
|
}
|
|
3390
|
+
// Populate retry tracking columns
|
|
2587
3391
|
if (validationAttempts && validationAttempts.length > 0) {
|
|
3392
|
+
// Update retry tracking columns
|
|
2588
3393
|
promptRun.ValidationAttemptCount = validationAttempts.length;
|
|
2589
3394
|
promptRun.SuccessfulValidationCount = validationAttempts.filter(a => a.success).length;
|
|
2590
3395
|
promptRun.FinalValidationPassed = parsedResult.validationResult?.Success === true;
|
|
2591
3396
|
promptRun.LastAttemptAt = endTime;
|
|
3397
|
+
// Calculate total retry duration (excluding first attempt)
|
|
2592
3398
|
if (validationAttempts.length > 1) {
|
|
2593
3399
|
const firstAttemptTime = validationAttempts[0].timestamp;
|
|
2594
3400
|
const lastAttemptTime = validationAttempts[validationAttempts.length - 1].timestamp;
|
|
@@ -2597,11 +3403,13 @@ class AIPromptRunner {
|
|
|
2597
3403
|
else {
|
|
2598
3404
|
promptRun.TotalRetryDurationMS = 0;
|
|
2599
3405
|
}
|
|
3406
|
+
// Get final validation error if any
|
|
2600
3407
|
const finalAttempt = validationAttempts[validationAttempts.length - 1];
|
|
2601
3408
|
if (!finalAttempt.success && finalAttempt.errorMessage) {
|
|
2602
|
-
promptRun.FinalValidationError = finalAttempt.errorMessage.substring(0, 500);
|
|
3409
|
+
promptRun.FinalValidationError = finalAttempt.errorMessage.substring(0, 500); // Truncate to fit column
|
|
2603
3410
|
promptRun.ValidationErrorCount = finalAttempt.validationErrors?.length || 0;
|
|
2604
3411
|
}
|
|
3412
|
+
// Find most common validation error
|
|
2605
3413
|
if (validationAttempts.some(a => !a.success)) {
|
|
2606
3414
|
const errorCounts = new Map();
|
|
2607
3415
|
validationAttempts.forEach(attempt => {
|
|
@@ -2612,9 +3420,10 @@ class AIPromptRunner {
|
|
|
2612
3420
|
});
|
|
2613
3421
|
if (errorCounts.size > 0) {
|
|
2614
3422
|
const [commonError] = [...errorCounts.entries()].sort((a, b) => b[1] - a[1])[0];
|
|
2615
|
-
promptRun.CommonValidationError = commonError.substring(0, 255);
|
|
3423
|
+
promptRun.CommonValidationError = commonError.substring(0, 255); // Truncate to fit column
|
|
2616
3424
|
}
|
|
2617
3425
|
}
|
|
3426
|
+
// Store detailed attempts in JSON columns
|
|
2618
3427
|
promptRun.ValidationAttempts = JSON.stringify(validationAttempts.map(a => ({
|
|
2619
3428
|
attemptNumber: a.attemptNumber,
|
|
2620
3429
|
success: a.success,
|
|
@@ -2646,14 +3455,18 @@ class AIPromptRunner {
|
|
|
2646
3455
|
});
|
|
2647
3456
|
}
|
|
2648
3457
|
else {
|
|
2649
|
-
|
|
3458
|
+
// No validation attempts (possibly skipped validation)
|
|
3459
|
+
promptRun.ValidationAttemptCount = 1; // At least one attempt was made
|
|
2650
3460
|
promptRun.SuccessfulValidationCount = parsedResult.validationResult?.Success !== false ? 1 : 0;
|
|
2651
3461
|
promptRun.FinalValidationPassed = parsedResult.validationResult?.Success !== false;
|
|
2652
3462
|
promptRun.LastAttemptAt = endTime;
|
|
2653
3463
|
promptRun.TotalRetryDurationMS = 0;
|
|
2654
3464
|
}
|
|
3465
|
+
// Set Success flag based on validation result
|
|
2655
3466
|
promptRun.Success = modelResult.success && (parsedResult.validationResult?.Success !== false);
|
|
3467
|
+
// Set final Status based on success
|
|
2656
3468
|
promptRun.Status = promptRun.Success ? 'Completed' : 'Failed';
|
|
3469
|
+
// Set ErrorDetails if failed
|
|
2657
3470
|
if (!promptRun.Success) {
|
|
2658
3471
|
if (!modelResult.success && modelResult.errorMessage) {
|
|
2659
3472
|
promptRun.ErrorDetails = modelResult.errorMessage;
|
|
@@ -2662,6 +3475,9 @@ class AIPromptRunner {
|
|
|
2662
3475
|
promptRun.ErrorDetails = `Validation failed: ${parsedResult.validationResult.Errors?.map(e => e.Message).join(', ')}`;
|
|
2663
3476
|
}
|
|
2664
3477
|
}
|
|
3478
|
+
// Note: Failover tracking fields are now updated directly in executeModelWithFailover
|
|
3479
|
+
// The promptRun entity already has the failover information set
|
|
3480
|
+
// With template composition, we only execute once so rollup equals regular fields
|
|
2665
3481
|
promptRun.TokensPromptRollup = promptRun.TokensPrompt;
|
|
2666
3482
|
promptRun.TokensCompletionRollup = promptRun.TokensCompletion;
|
|
2667
3483
|
promptRun.TokensUsedRollup = promptRun.TokensUsed;
|
|
@@ -2670,6 +3486,7 @@ class AIPromptRunner {
|
|
|
2670
3486
|
}
|
|
2671
3487
|
const saveResult = await promptRun.Save();
|
|
2672
3488
|
if (!saveResult) {
|
|
3489
|
+
// Safely extract error message using CompleteMessage getter
|
|
2673
3490
|
let errorMsg = 'Unknown error';
|
|
2674
3491
|
try {
|
|
2675
3492
|
if (promptRun.LatestResult?.CompleteMessage) {
|
|
@@ -2699,6 +3516,27 @@ class AIPromptRunner {
|
|
|
2699
3516
|
});
|
|
2700
3517
|
}
|
|
2701
3518
|
}
|
|
3519
|
+
// ==================== CONTEXT LENGTH METHODS ====================
|
|
3520
|
+
/**
|
|
3521
|
+
* Estimates the number of tokens in a rendered prompt and conversation messages.
|
|
3522
|
+
* This is a rough estimation based on character count and typical token ratios.
|
|
3523
|
+
*
|
|
3524
|
+
* @param renderedPrompt - The rendered prompt text
|
|
3525
|
+
* @param conversationMessages - Optional conversation messages
|
|
3526
|
+
* @returns Estimated token count
|
|
3527
|
+
*/
|
|
3528
|
+
// ==================== FAILOVER METHODS ====================
|
|
3529
|
+
/**
|
|
3530
|
+
* Retrieves failover configuration from the prompt entity.
|
|
3531
|
+
*
|
|
3532
|
+
* @param prompt - The AI prompt entity containing failover settings
|
|
3533
|
+
* @returns FailoverConfiguration object with strategy and settings
|
|
3534
|
+
*
|
|
3535
|
+
* @remarks
|
|
3536
|
+
* This method extracts failover configuration from the prompt entity and provides
|
|
3537
|
+
* default values when configuration is not specified. Override this method to
|
|
3538
|
+
* implement custom failover configuration logic.
|
|
3539
|
+
*/
|
|
2702
3540
|
getFailoverConfiguration(prompt) {
|
|
2703
3541
|
return {
|
|
2704
3542
|
strategy: prompt.FailoverStrategy || 'None',
|
|
@@ -2708,14 +3546,31 @@ class AIPromptRunner {
|
|
|
2708
3546
|
errorScope: prompt.FailoverErrorScope || 'All'
|
|
2709
3547
|
};
|
|
2710
3548
|
}
|
|
3549
|
+
/**
|
|
3550
|
+
* Determines whether a failover attempt should be made based on the error and configuration.
|
|
3551
|
+
*
|
|
3552
|
+
* @param error - The error that occurred during execution
|
|
3553
|
+
* @param config - The failover configuration
|
|
3554
|
+
* @param attemptNumber - The current attempt number (1-based)
|
|
3555
|
+
* @returns True if failover should be attempted, false otherwise
|
|
3556
|
+
*
|
|
3557
|
+
* @remarks
|
|
3558
|
+
* This method uses the ErrorAnalyzer to classify errors and determine if they are
|
|
3559
|
+
* eligible for failover based on the configured error scope. Override this method
|
|
3560
|
+
* to implement custom failover decision logic.
|
|
3561
|
+
*/
|
|
2711
3562
|
shouldAttemptFailover(error, config, attemptNumber) {
|
|
3563
|
+
// Don't failover if strategy is None or we've exceeded max attempts
|
|
2712
3564
|
if (config.strategy === 'None' || attemptNumber > config.maxAttempts) {
|
|
2713
3565
|
return false;
|
|
2714
3566
|
}
|
|
2715
|
-
|
|
3567
|
+
// Analyze the error to determine if it's eligible for failover
|
|
3568
|
+
const errorAnalysis = ErrorAnalyzer.analyzeError(error);
|
|
3569
|
+
// Check if error analysis allows failover
|
|
2716
3570
|
if (!errorAnalysis.canFailover) {
|
|
2717
3571
|
return false;
|
|
2718
3572
|
}
|
|
3573
|
+
// Check error scope configuration
|
|
2719
3574
|
switch (config.errorScope) {
|
|
2720
3575
|
case 'NetworkOnly':
|
|
2721
3576
|
return errorAnalysis.errorType === 'NetworkError';
|
|
@@ -2729,6 +3584,13 @@ class AIPromptRunner {
|
|
|
2729
3584
|
return true;
|
|
2730
3585
|
}
|
|
2731
3586
|
}
|
|
3587
|
+
/**
|
|
3588
|
+
* Checks if an error type matches the configured error scope
|
|
3589
|
+
*
|
|
3590
|
+
* @param errorType - The error type from ErrorAnalyzer
|
|
3591
|
+
* @param scope - The configured error scope
|
|
3592
|
+
* @returns True if the error matches the scope
|
|
3593
|
+
*/
|
|
2732
3594
|
errorMatchesScope(errorType, scope) {
|
|
2733
3595
|
switch (scope) {
|
|
2734
3596
|
case 'NetworkOnly':
|
|
@@ -2742,31 +3604,74 @@ class AIPromptRunner {
|
|
|
2742
3604
|
return true;
|
|
2743
3605
|
}
|
|
2744
3606
|
}
|
|
3607
|
+
/**
|
|
3608
|
+
* Calculates the delay before the next failover attempt.
|
|
3609
|
+
*
|
|
3610
|
+
* @param attemptNumber - The current attempt number (1-based)
|
|
3611
|
+
* @param baseDelaySeconds - The base delay in seconds from configuration
|
|
3612
|
+
* @param previousError - The error from the previous attempt
|
|
3613
|
+
* @returns Delay in milliseconds before the next attempt
|
|
3614
|
+
*
|
|
3615
|
+
* @remarks
|
|
3616
|
+
* Implements exponential backoff with jitter by default. The delay increases
|
|
3617
|
+
* exponentially with each attempt and includes random jitter to prevent
|
|
3618
|
+
* thundering herd problems. Override this method to implement custom delay logic.
|
|
3619
|
+
*/
|
|
2745
3620
|
calculateFailoverDelay(attemptNumber, baseDelaySeconds) {
|
|
3621
|
+
// Exponential backoff: delay = base * 2^(attempt-1)
|
|
2746
3622
|
const exponentialDelay = baseDelaySeconds * Math.pow(2, attemptNumber - 1);
|
|
3623
|
+
// Add jitter (0-25% of delay) to prevent thundering herd
|
|
2747
3624
|
const jitter = exponentialDelay * 0.25 * Math.random();
|
|
3625
|
+
// Cap at 30 seconds to prevent excessive delays
|
|
2748
3626
|
const totalDelay = Math.min(exponentialDelay + jitter, 30);
|
|
2749
|
-
return totalDelay * 1000;
|
|
2750
|
-
}
|
|
3627
|
+
return totalDelay * 1000; // Convert to milliseconds
|
|
3628
|
+
}
|
|
3629
|
+
/**
|
|
3630
|
+
* Selects candidate models for failover based on the strategy and current failure.
|
|
3631
|
+
*
|
|
3632
|
+
* @param currentModel - The model that just failed
|
|
3633
|
+
* @param currentVendorId - The vendor ID that just failed
|
|
3634
|
+
* @param strategy - The failover strategy to use
|
|
3635
|
+
* @param modelStrategy - The model selection preference
|
|
3636
|
+
* @param allCandidates - All available model-vendor candidates
|
|
3637
|
+
* @param attemptHistory - History of previous failover attempts
|
|
3638
|
+
* @returns Array of candidates sorted by priority (highest first)
|
|
3639
|
+
*
|
|
3640
|
+
* @remarks
|
|
3641
|
+
* This method implements different strategies for selecting failover candidates:
|
|
3642
|
+
* - SameModelDifferentVendor: Try the same model with different vendors
|
|
3643
|
+
* - NextBestModel: Try different models in order of preference
|
|
3644
|
+
* - PowerRank: Use the global power ranking of models
|
|
3645
|
+
*
|
|
3646
|
+
* Override this method to implement custom candidate selection logic.
|
|
3647
|
+
*/
|
|
2751
3648
|
selectFailoverCandidates(currentModel, currentVendorId, strategy, modelStrategy, allCandidates, attemptHistory) {
|
|
3649
|
+
// Filter out candidates that have already failed
|
|
3650
|
+
// Note: Authentication errors are already filtered from allCandidates upstream,
|
|
3651
|
+
// so we only need to filter out specific model/vendor pairs that have failed
|
|
2752
3652
|
const failedPairs = new Set(attemptHistory.map(a => `${a.modelId}:${a.vendorId || 'default'}`));
|
|
2753
3653
|
const availableCandidates = allCandidates.filter(c => {
|
|
2754
3654
|
const key = `${c.model.ID}:${c.vendorId || 'default'}`;
|
|
2755
3655
|
return !failedPairs.has(key);
|
|
2756
3656
|
});
|
|
3657
|
+
// Check if we have context length exceeded errors in the attempt history
|
|
2757
3658
|
const hasContextLengthError = attemptHistory.some(a => a.errorType === 'ContextLengthExceeded' ||
|
|
2758
|
-
|
|
3659
|
+
ErrorAnalyzer.analyzeError(a.error).errorType === 'ContextLengthExceeded');
|
|
3660
|
+
// Apply strategy-specific filtering and sorting
|
|
2759
3661
|
let candidates;
|
|
2760
3662
|
switch (strategy) {
|
|
2761
3663
|
case 'SameModelDifferentVendor':
|
|
3664
|
+
// Only consider same model with different vendors
|
|
2762
3665
|
candidates = availableCandidates.filter(c => c.model.ID === currentModel.ID && c.vendorId !== currentVendorId);
|
|
2763
3666
|
break;
|
|
2764
3667
|
case 'NextBestModel':
|
|
3668
|
+
// Consider all models, apply model strategy preference
|
|
2765
3669
|
candidates = availableCandidates;
|
|
2766
3670
|
if (modelStrategy === 'RequireSameModel') {
|
|
2767
3671
|
candidates = candidates.filter(c => c.model.ID === currentModel.ID);
|
|
2768
3672
|
}
|
|
2769
3673
|
else if (modelStrategy === 'PreferSameModel') {
|
|
3674
|
+
// Sort to put same model first
|
|
2770
3675
|
candidates.sort((a, b) => {
|
|
2771
3676
|
const aSameModel = a.model.ID === currentModel.ID ? 1 : 0;
|
|
2772
3677
|
const bSameModel = b.model.ID === currentModel.ID ? 1 : 0;
|
|
@@ -2774,6 +3679,7 @@ class AIPromptRunner {
|
|
|
2774
3679
|
});
|
|
2775
3680
|
}
|
|
2776
3681
|
else if (modelStrategy === 'PreferDifferentModel') {
|
|
3682
|
+
// Sort to put different models first
|
|
2777
3683
|
candidates.sort((a, b) => {
|
|
2778
3684
|
const aDiffModel = a.model.ID !== currentModel.ID ? 1 : 0;
|
|
2779
3685
|
const bDiffModel = b.model.ID !== currentModel.ID ? 1 : 0;
|
|
@@ -2782,21 +3688,25 @@ class AIPromptRunner {
|
|
|
2782
3688
|
}
|
|
2783
3689
|
break;
|
|
2784
3690
|
case 'PowerRank':
|
|
3691
|
+
// Use all candidates, they're already sorted by power rank
|
|
2785
3692
|
candidates = availableCandidates;
|
|
2786
3693
|
break;
|
|
2787
3694
|
default:
|
|
2788
3695
|
candidates = [];
|
|
2789
3696
|
}
|
|
3697
|
+
// If we have context length errors, prioritize models with larger context windows
|
|
2790
3698
|
if (hasContextLengthError) {
|
|
2791
3699
|
const currentMaxTokens = currentModel.ModelVendors?.length > 0 ?
|
|
2792
3700
|
Math.max(...currentModel.ModelVendors.map(mv => mv.MaxInputTokens || 0)) : 0;
|
|
3701
|
+
// Filter out models with same or smaller context windows
|
|
2793
3702
|
candidates = candidates.filter(c => {
|
|
2794
3703
|
const candidateMaxTokens = c.model.ModelVendors?.length > 0 ?
|
|
2795
3704
|
Math.max(...c.model.ModelVendors.map(mv => mv.MaxInputTokens || 0)) : 0;
|
|
2796
3705
|
return candidateMaxTokens > currentMaxTokens;
|
|
2797
3706
|
});
|
|
3707
|
+
// If no larger models exist, this is a fatal error - return empty to stop retrying
|
|
2798
3708
|
if (candidates.length === 0) {
|
|
2799
|
-
|
|
3709
|
+
LogStatusEx({
|
|
2800
3710
|
message: `❌ Context length exceeded and no models with larger context windows available. Current model: ${currentModel.Name} (${currentMaxTokens} max tokens). This is a fatal error.`,
|
|
2801
3711
|
category: 'AI',
|
|
2802
3712
|
additionalArgs: [{
|
|
@@ -2806,22 +3716,27 @@ class AIPromptRunner {
|
|
|
2806
3716
|
reason: 'No models with larger context windows available for failover'
|
|
2807
3717
|
}]
|
|
2808
3718
|
});
|
|
3719
|
+
// Return empty array - caller will see no candidates and stop retrying
|
|
2809
3720
|
return [];
|
|
2810
3721
|
}
|
|
3722
|
+
// Sort by priority first (existing algorithm), then by context window size as tiebreaker
|
|
2811
3723
|
candidates.sort((a, b) => {
|
|
3724
|
+
// Primary sort: priority (higher is better) - maintains existing algorithm
|
|
2812
3725
|
if (a.priority !== b.priority) {
|
|
2813
3726
|
return b.priority - a.priority;
|
|
2814
3727
|
}
|
|
3728
|
+
// Secondary sort: context window size (largest first) - only as tiebreaker
|
|
2815
3729
|
const aMaxTokens = a.model.ModelVendors?.length > 0 ?
|
|
2816
3730
|
Math.max(...a.model.ModelVendors.map((mv) => mv.MaxInputTokens || 0)) : 0;
|
|
2817
3731
|
const bMaxTokens = b.model.ModelVendors?.length > 0 ?
|
|
2818
3732
|
Math.max(...b.model.ModelVendors.map((mv) => mv.MaxInputTokens || 0)) : 0;
|
|
2819
3733
|
return bMaxTokens - aMaxTokens;
|
|
2820
3734
|
});
|
|
3735
|
+
// Log context-aware failover selection
|
|
2821
3736
|
const bestCandidate = candidates[0];
|
|
2822
3737
|
const bestCandidateMaxTokens = bestCandidate.model.ModelVendors?.length > 0 ?
|
|
2823
3738
|
Math.max(...bestCandidate.model.ModelVendors.map((mv) => mv.MaxInputTokens || 0)) : 0;
|
|
2824
|
-
|
|
3739
|
+
LogStatusEx({
|
|
2825
3740
|
message: `🔄 Context-aware failover: Selected model ${bestCandidate.model.Name} with ${bestCandidateMaxTokens} max input tokens (vs ${currentMaxTokens} for failed model)`,
|
|
2826
3741
|
category: 'AI',
|
|
2827
3742
|
additionalArgs: [{
|
|
@@ -2834,10 +3749,23 @@ class AIPromptRunner {
|
|
|
2834
3749
|
});
|
|
2835
3750
|
}
|
|
2836
3751
|
else {
|
|
3752
|
+
// Final sort by priority (higher is better) for non-context-length errors
|
|
2837
3753
|
candidates.sort((a, b) => b.priority - a.priority);
|
|
2838
3754
|
}
|
|
2839
3755
|
return candidates;
|
|
2840
3756
|
}
|
|
3757
|
+
/**
|
|
3758
|
+
* Logs a failover attempt for tracking and debugging.
|
|
3759
|
+
*
|
|
3760
|
+
* @param promptId - The ID of the prompt being executed
|
|
3761
|
+
* @param attempt - The failover attempt details
|
|
3762
|
+
* @param willRetry - Whether another attempt will be made
|
|
3763
|
+
*
|
|
3764
|
+
* @remarks
|
|
3765
|
+
* This method logs detailed information about each failover attempt to help with
|
|
3766
|
+
* debugging and monitoring. Override this method to implement custom logging or
|
|
3767
|
+
* integrate with external monitoring systems.
|
|
3768
|
+
*/
|
|
2841
3769
|
logFailoverAttempt(promptId, attempt, willRetry) {
|
|
2842
3770
|
const message = `Failover attempt ${attempt.attemptNumber} for prompt ${promptId}`;
|
|
2843
3771
|
const metadata = {
|
|
@@ -2851,14 +3779,14 @@ class AIPromptRunner {
|
|
|
2851
3779
|
error: attempt.error.message
|
|
2852
3780
|
};
|
|
2853
3781
|
if (willRetry) {
|
|
2854
|
-
|
|
3782
|
+
LogStatusEx({
|
|
2855
3783
|
message: `⚡ ${message}`,
|
|
2856
3784
|
category: 'AI',
|
|
2857
3785
|
additionalArgs: [metadata]
|
|
2858
3786
|
});
|
|
2859
3787
|
}
|
|
2860
3788
|
else {
|
|
2861
|
-
|
|
3789
|
+
LogErrorEx({
|
|
2862
3790
|
message: message,
|
|
2863
3791
|
error: attempt.error,
|
|
2864
3792
|
category: 'AI',
|
|
@@ -2868,8 +3796,4 @@ class AIPromptRunner {
|
|
|
2868
3796
|
}
|
|
2869
3797
|
}
|
|
2870
3798
|
}
|
|
2871
|
-
exports.AIPromptRunner = AIPromptRunner;
|
|
2872
|
-
function LoadAIPromptRunner() {
|
|
2873
|
-
}
|
|
2874
|
-
exports.LoadAIPromptRunner = LoadAIPromptRunner;
|
|
2875
3799
|
//# sourceMappingURL=AIPromptRunner.js.map
|