@memberjunction/ai-prompts 3.3.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AIPromptRunner.d.ts +541 -2
- package/dist/AIPromptRunner.d.ts.map +1 -1
- package/dist/AIPromptRunner.js +1217 -260
- package/dist/AIPromptRunner.js.map +1 -1
- package/dist/ExecutionPlanner.d.ts +116 -1
- package/dist/ExecutionPlanner.d.ts.map +1 -1
- package/dist/ExecutionPlanner.js +171 -40
- package/dist/ExecutionPlanner.js.map +1 -1
- package/dist/ParallelExecution.d.ts +178 -1
- package/dist/ParallelExecution.d.ts.map +1 -1
- package/dist/ParallelExecution.js +1 -2
- package/dist/ParallelExecutionCoordinator.d.ts +182 -3
- package/dist/ParallelExecutionCoordinator.d.ts.map +1 -1
- package/dist/ParallelExecutionCoordinator.js +304 -79
- package/dist/ParallelExecutionCoordinator.js.map +1 -1
- package/dist/__tests__/AIPromptRunner.failover.test.d.ts +23 -0
- package/dist/__tests__/AIPromptRunner.failover.test.d.ts.map +1 -0
- package/dist/__tests__/AIPromptRunner.failover.test.js +471 -0
- package/dist/__tests__/AIPromptRunner.failover.test.js.map +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -19
- package/dist/index.js.map +1 -1
- package/package.json +24 -18
package/dist/AIPromptRunner.js
CHANGED
|
@@ -1,85 +1,77 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.LoadAIPromptRunner = exports.AIPromptRunner = void 0;
|
|
27
|
-
const ai_1 = require("@memberjunction/ai");
|
|
28
|
-
const ai_core_plus_1 = require("@memberjunction/ai-core-plus");
|
|
29
|
-
const core_1 = require("@memberjunction/core");
|
|
30
|
-
const global_1 = require("@memberjunction/global");
|
|
31
|
-
const credentials_1 = require("@memberjunction/credentials");
|
|
32
|
-
const templates_1 = require("@memberjunction/templates");
|
|
33
|
-
const ExecutionPlanner_1 = require("./ExecutionPlanner");
|
|
34
|
-
const ParallelExecutionCoordinator_1 = require("./ParallelExecutionCoordinator");
|
|
35
|
-
const aiengine_1 = require("@memberjunction/aiengine");
|
|
36
|
-
const ai_core_plus_2 = require("@memberjunction/ai-core-plus");
|
|
37
|
-
const JSON5 = __importStar(require("json5"));
|
|
38
|
-
class AIPromptRunner {
|
|
1
|
+
import { BaseLLM, ChatParams, ChatMessageRole, GetAIAPIKey, ErrorAnalyzer } from '@memberjunction/ai';
|
|
2
|
+
import { AIModelSelectionInfo } from '@memberjunction/ai-core-plus';
|
|
3
|
+
import { LogErrorEx, LogStatus, LogStatusEx, IsVerboseLoggingEnabled, Metadata } from '@memberjunction/core';
|
|
4
|
+
import { CleanJSON, MJGlobal, JSONValidator, ValidationResult, ValidationErrorInfo, ValidationErrorType } from '@memberjunction/global';
|
|
5
|
+
import { CredentialEngine } from '@memberjunction/credentials';
|
|
6
|
+
import { TemplateEngineServer } from '@memberjunction/templates';
|
|
7
|
+
import { ExecutionPlanner } from './ExecutionPlanner.js';
|
|
8
|
+
import { ParallelExecutionCoordinator } from './ParallelExecutionCoordinator.js';
|
|
9
|
+
import { AIEngine } from '@memberjunction/aiengine';
|
|
10
|
+
import { SystemPlaceholderManager } from '@memberjunction/ai-core-plus';
|
|
11
|
+
import * as JSON5 from 'json5';
|
|
12
|
+
export class AIPromptRunner {
|
|
39
13
|
constructor() {
|
|
40
|
-
this._metadata = new
|
|
41
|
-
this._templateEngine =
|
|
42
|
-
this._executionPlanner = new
|
|
43
|
-
this._parallelCoordinator = new
|
|
44
|
-
this._jsonValidator = new
|
|
45
|
-
}
|
|
14
|
+
this._metadata = new Metadata();
|
|
15
|
+
this._templateEngine = TemplateEngineServer.Instance;
|
|
16
|
+
this._executionPlanner = new ExecutionPlanner();
|
|
17
|
+
this._parallelCoordinator = new ParallelExecutionCoordinator();
|
|
18
|
+
this._jsonValidator = new JSONValidator();
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Performs robust validation of an API key
|
|
22
|
+
* @returns true if the API key is valid (not null, undefined, or empty/whitespace)
|
|
23
|
+
*/
|
|
46
24
|
isValidAPIKey(apiKey) {
|
|
47
25
|
if (apiKey === undefined || apiKey === null) {
|
|
48
26
|
return false;
|
|
49
27
|
}
|
|
28
|
+
// Check if it's just whitespace
|
|
50
29
|
const trimmed = apiKey.trim();
|
|
51
30
|
return trimmed.length > 0;
|
|
52
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* Internal logging helper that wraps LogStatusEx with verbose control
|
|
34
|
+
* @param message The message to log
|
|
35
|
+
* @param verboseOnly Whether this is a verbose-only message
|
|
36
|
+
* @param params Optional prompt parameters for custom verbose check
|
|
37
|
+
*/
|
|
53
38
|
logStatus(message, verboseOnly = false, params) {
|
|
54
39
|
if (verboseOnly) {
|
|
55
|
-
|
|
40
|
+
LogStatusEx({
|
|
56
41
|
message,
|
|
57
42
|
verboseOnly: true,
|
|
58
|
-
isVerboseEnabled: () => params?.verbose === true ||
|
|
43
|
+
isVerboseEnabled: () => params?.verbose === true || IsVerboseLoggingEnabled()
|
|
59
44
|
});
|
|
60
45
|
}
|
|
61
46
|
else {
|
|
62
|
-
|
|
47
|
+
LogStatus(message);
|
|
63
48
|
}
|
|
64
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* Helper method for enhanced error logging with metadata
|
|
52
|
+
*/
|
|
65
53
|
logError(error, options) {
|
|
66
54
|
let errorMessage = error instanceof Error ? error.message : error;
|
|
67
55
|
const errorObj = error instanceof Error ? error : undefined;
|
|
56
|
+
// Truncate extremely long error messages (like Groq's failed_generation JSON dumps)
|
|
57
|
+
// Only truncate if maxErrorLength is explicitly set
|
|
68
58
|
if (options?.maxErrorLength !== undefined && errorMessage.length > options.maxErrorLength) {
|
|
69
59
|
errorMessage = errorMessage.substring(0, options.maxErrorLength) + '... [truncated]';
|
|
70
60
|
}
|
|
71
61
|
const metadata = {
|
|
72
62
|
...options?.metadata
|
|
73
63
|
};
|
|
64
|
+
// Add prompt information if available
|
|
74
65
|
if (options?.prompt) {
|
|
75
66
|
metadata.promptId = options.prompt.ID;
|
|
76
67
|
metadata.promptName = options.prompt.Name;
|
|
77
68
|
}
|
|
69
|
+
// Add model information if available
|
|
78
70
|
if (options?.model) {
|
|
79
71
|
metadata.modelId = options.model.ID;
|
|
80
72
|
metadata.modelName = options.model.Name;
|
|
81
73
|
}
|
|
82
|
-
|
|
74
|
+
LogErrorEx({
|
|
83
75
|
message: errorMessage,
|
|
84
76
|
error: errorObj,
|
|
85
77
|
category: options?.category || 'AIPromptRunner',
|
|
@@ -87,46 +79,81 @@ class AIPromptRunner {
|
|
|
87
79
|
metadata: Object.keys(metadata).length > 0 ? metadata : undefined
|
|
88
80
|
});
|
|
89
81
|
}
|
|
82
|
+
/**
|
|
83
|
+
* Checks if a model vendor is configured as an inference provider
|
|
84
|
+
* @param modelVendor The model vendor to check
|
|
85
|
+
* @returns true if the vendor is an inference provider
|
|
86
|
+
*/
|
|
90
87
|
isInferenceProvider(modelVendor) {
|
|
91
|
-
|
|
88
|
+
// Find the inference provider type from cached vendor type definitions
|
|
89
|
+
const inferenceProviderType = AIEngine.Instance.VendorTypeDefinitions.find(vt => vt.Name === 'Inference Provider');
|
|
92
90
|
if (!inferenceProviderType) {
|
|
93
|
-
|
|
91
|
+
// Fallback to checking if it's not a model developer (should rarely happen)
|
|
92
|
+
const modelDeveloperType = AIEngine.Instance.VendorTypeDefinitions.find(vt => vt.Name === 'Model Developer');
|
|
94
93
|
return modelVendor.TypeID !== modelDeveloperType?.ID;
|
|
95
94
|
}
|
|
96
95
|
return modelVendor.TypeID === inferenceProviderType.ID;
|
|
97
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* Resolves credentials for AI model execution using a hierarchical resolution system.
|
|
99
|
+
*
|
|
100
|
+
* Resolution priority (highest to lowest):
|
|
101
|
+
* 1. Per-request override: params.credentialId
|
|
102
|
+
* 2. Prompt-Model specific: AIPromptModel.CredentialID
|
|
103
|
+
* 3. Model-Vendor specific: AIModelVendor.CredentialID
|
|
104
|
+
* 4. Vendor default: AIVendor.CredentialID
|
|
105
|
+
* 5. Legacy: params.apiKeys[] array
|
|
106
|
+
* 6. Legacy: AI_VENDOR_API_KEY__<DRIVER> environment variables
|
|
107
|
+
*
|
|
108
|
+
* IMPORTANT: When ANY credential ID is found (priorities 1-4), the system uses
|
|
109
|
+
* the Credentials path and ignores legacy methods (priorities 5-6).
|
|
110
|
+
*
|
|
111
|
+
* @param driverClass - The driver class name (e.g., 'OpenAILLM')
|
|
112
|
+
* @param promptId - The prompt ID for looking up AIPromptModel credentials
|
|
113
|
+
* @param modelId - The model ID for looking up AIPromptModel and AIModelVendor credentials
|
|
114
|
+
* @param vendorId - The vendor ID for looking up AIModelVendor and AIVendor credentials
|
|
115
|
+
* @param params - The prompt execution parameters containing contextUser and optional credentialId
|
|
116
|
+
* @returns The API key/configuration string to pass to the LLM constructor
|
|
117
|
+
*/
|
|
98
118
|
async resolveCredentialForExecution(driverClass, promptId, modelId, vendorId, params) {
|
|
99
|
-
const verbose = params.verbose === true ||
|
|
119
|
+
const verbose = params.verbose === true || IsVerboseLoggingEnabled();
|
|
120
|
+
// Priority 1: Per-request override - no failover, explicit choice
|
|
100
121
|
if (params.credentialId) {
|
|
101
122
|
return await this.resolveCredentialById(params.credentialId, 'per-request override', params, verbose);
|
|
102
123
|
}
|
|
103
|
-
|
|
124
|
+
// Ensure CredentialEngine is configured for binding lookups
|
|
125
|
+
await CredentialEngine.Instance.Config(false, params.contextUser);
|
|
126
|
+
// Priority 2: PromptModel bindings (most specific) - with failover
|
|
104
127
|
if (promptId && modelId) {
|
|
105
|
-
const promptModel =
|
|
128
|
+
const promptModel = AIEngine.Instance.PromptModels.find(pm => pm.PromptID === promptId && pm.ModelID === modelId);
|
|
106
129
|
if (promptModel) {
|
|
107
|
-
const bindings =
|
|
130
|
+
const bindings = AIEngine.Instance.GetCredentialBindingsForTarget('PromptModel', promptModel.ID);
|
|
108
131
|
const result = await this.tryCredentialBindingsWithFailover(bindings, 'AICredentialBinding(PromptModel)', params, verbose);
|
|
109
132
|
if (result)
|
|
110
133
|
return result;
|
|
111
134
|
}
|
|
112
135
|
}
|
|
136
|
+
// Priority 3: ModelVendor bindings - with failover
|
|
113
137
|
if (modelId && vendorId) {
|
|
114
|
-
const modelVendor =
|
|
138
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === modelId && mv.VendorID === vendorId && mv.Status === 'Active');
|
|
115
139
|
if (modelVendor) {
|
|
116
|
-
const bindings =
|
|
140
|
+
const bindings = AIEngine.Instance.GetCredentialBindingsForTarget('ModelVendor', modelVendor.ID);
|
|
117
141
|
const result = await this.tryCredentialBindingsWithFailover(bindings, 'AICredentialBinding(ModelVendor)', params, verbose);
|
|
118
142
|
if (result)
|
|
119
143
|
return result;
|
|
120
144
|
}
|
|
121
145
|
}
|
|
146
|
+
// Priority 4: Vendor bindings - with failover
|
|
122
147
|
if (vendorId) {
|
|
123
|
-
const bindings =
|
|
148
|
+
const bindings = AIEngine.Instance.GetCredentialBindingsForTarget('Vendor', vendorId);
|
|
124
149
|
const result = await this.tryCredentialBindingsWithFailover(bindings, 'AICredentialBinding(Vendor)', params, verbose);
|
|
125
150
|
if (result)
|
|
126
151
|
return result;
|
|
127
152
|
}
|
|
153
|
+
// Priority 5: Type-based default credential
|
|
154
|
+
// If the vendor declares a CredentialTypeID, try to find a default credential of that type
|
|
128
155
|
if (vendorId) {
|
|
129
|
-
const vendor =
|
|
156
|
+
const vendor = AIEngine.Instance.Vendors.find(v => v.ID === vendorId);
|
|
130
157
|
if (vendor?.CredentialTypeID) {
|
|
131
158
|
const defaultCredential = this.findDefaultCredentialByType(vendor.CredentialTypeID);
|
|
132
159
|
if (defaultCredential) {
|
|
@@ -136,31 +163,42 @@ class AIPromptRunner {
|
|
|
136
163
|
}
|
|
137
164
|
}
|
|
138
165
|
}
|
|
166
|
+
// No credential bindings found - fall back to legacy methods
|
|
139
167
|
if (verbose) {
|
|
140
168
|
this.logStatus(` Using legacy API key resolution for driver ${driverClass}`, true, params);
|
|
141
169
|
}
|
|
142
|
-
|
|
170
|
+
// Priority 6 & 7: Legacy apiKeys array and environment variables
|
|
171
|
+
return GetAIAPIKey(driverClass, params.apiKeys, verbose);
|
|
143
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* Attempts to resolve credentials from bindings with priority-based failover.
|
|
175
|
+
* Tries each binding in priority order until one succeeds.
|
|
176
|
+
*/
|
|
144
177
|
async tryCredentialBindingsWithFailover(bindings, source, params, verbose) {
|
|
145
178
|
if (bindings.length === 0)
|
|
146
179
|
return null;
|
|
147
180
|
for (let i = 0; i < bindings.length; i++) {
|
|
148
181
|
const binding = bindings[i];
|
|
149
|
-
const credential =
|
|
182
|
+
const credential = CredentialEngine.Instance.getCredentialById(binding.CredentialID);
|
|
150
183
|
if (!credential) {
|
|
151
184
|
if (verbose) {
|
|
152
185
|
this.logStatus(` ⚠️ Credential ${binding.CredentialID} not found (priority ${binding.Priority}), trying next...`, true, params);
|
|
153
186
|
}
|
|
154
187
|
continue;
|
|
155
188
|
}
|
|
156
|
-
const result = await this.tryResolveCredential(credential, `${source} priority ${binding.Priority}`, params, verbose, i < bindings.length - 1
|
|
189
|
+
const result = await this.tryResolveCredential(credential, `${source} priority ${binding.Priority}`, params, verbose, i < bindings.length - 1 // hasMoreBindings
|
|
190
|
+
);
|
|
157
191
|
if (result)
|
|
158
192
|
return result;
|
|
159
193
|
}
|
|
160
194
|
return null;
|
|
161
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* Attempts to resolve a single credential, returning null on failure for failover support.
|
|
198
|
+
*/
|
|
162
199
|
async tryResolveCredential(credential, source, params, verbose, hasMoreBindings = false) {
|
|
163
200
|
try {
|
|
201
|
+
// Check if credential is active and not expired
|
|
164
202
|
if (!credential.IsActive) {
|
|
165
203
|
if (verbose) {
|
|
166
204
|
this.logStatus(` ⚠️ Credential "${credential.Name}" is inactive, trying next...`, true, params);
|
|
@@ -173,7 +211,8 @@ class AIPromptRunner {
|
|
|
173
211
|
}
|
|
174
212
|
return null;
|
|
175
213
|
}
|
|
176
|
-
|
|
214
|
+
// Resolve the credential values
|
|
215
|
+
const resolved = await CredentialEngine.Instance.getCredential(credential.Name, {
|
|
177
216
|
credentialId: credential.ID,
|
|
178
217
|
contextUser: params.contextUser,
|
|
179
218
|
subsystem: 'AIPromptRunner'
|
|
@@ -185,12 +224,14 @@ class AIPromptRunner {
|
|
|
185
224
|
}
|
|
186
225
|
catch (error) {
|
|
187
226
|
if (hasMoreBindings) {
|
|
227
|
+
// More bindings to try - log warning and continue
|
|
188
228
|
if (verbose) {
|
|
189
229
|
this.logStatus(` ⚠️ Failed to resolve credential "${credential.Name}" from ${source}: ${error instanceof Error ? error.message : String(error)}, trying next...`, true, params);
|
|
190
230
|
}
|
|
191
231
|
return null;
|
|
192
232
|
}
|
|
193
233
|
else {
|
|
234
|
+
// No more bindings - log error but still return null for legacy fallback
|
|
194
235
|
this.logError(error instanceof Error ? error : new Error(String(error)), {
|
|
195
236
|
category: 'CredentialResolution',
|
|
196
237
|
severity: 'warning',
|
|
@@ -205,13 +246,17 @@ class AIPromptRunner {
|
|
|
205
246
|
}
|
|
206
247
|
}
|
|
207
248
|
}
|
|
249
|
+
/**
|
|
250
|
+
* Resolves a credential by its explicit ID (used for per-request override).
|
|
251
|
+
* This does not support failover since it's an explicit choice.
|
|
252
|
+
*/
|
|
208
253
|
async resolveCredentialById(credentialId, source, params, verbose) {
|
|
209
|
-
await
|
|
210
|
-
const credential =
|
|
254
|
+
await CredentialEngine.Instance.Config(false, params.contextUser);
|
|
255
|
+
const credential = CredentialEngine.Instance.getCredentialById(credentialId);
|
|
211
256
|
if (!credential) {
|
|
212
257
|
throw new Error(`Credential with ID ${credentialId} not found`);
|
|
213
258
|
}
|
|
214
|
-
const resolved = await
|
|
259
|
+
const resolved = await CredentialEngine.Instance.getCredential(credential.Name, {
|
|
215
260
|
credentialId,
|
|
216
261
|
contextUser: params.contextUser,
|
|
217
262
|
subsystem: 'AIPromptRunner'
|
|
@@ -221,36 +266,66 @@ class AIPromptRunner {
|
|
|
221
266
|
}
|
|
222
267
|
return JSON.stringify(resolved.values);
|
|
223
268
|
}
|
|
269
|
+
/**
|
|
270
|
+
* Finds a default credential matching a specific credential type.
|
|
271
|
+
*/
|
|
224
272
|
findDefaultCredentialByType(credentialTypeId) {
|
|
225
|
-
const credentials =
|
|
273
|
+
const credentials = CredentialEngine.Instance.Credentials;
|
|
226
274
|
return credentials.find(c => c.CredentialTypeID === credentialTypeId &&
|
|
227
275
|
c.IsDefault === true &&
|
|
228
276
|
c.IsActive === true &&
|
|
229
277
|
(!c.ExpiresAt || new Date(c.ExpiresAt) > new Date())) || null;
|
|
230
278
|
}
|
|
279
|
+
/**
|
|
280
|
+
* Checks if credentials are available for a given model-vendor combination.
|
|
281
|
+
* This is a pre-flight check used during model selection to determine which
|
|
282
|
+
* candidates have valid authentication configured.
|
|
283
|
+
*
|
|
284
|
+
* Checks the credential hierarchy:
|
|
285
|
+
* 1. Per-request override: params.credentialId
|
|
286
|
+
* 2. PromptModel bindings: AICredentialBinding WHERE BindingType='PromptModel'
|
|
287
|
+
* 3. ModelVendor bindings: AICredentialBinding WHERE BindingType='ModelVendor'
|
|
288
|
+
* 4. Vendor bindings: AICredentialBinding WHERE BindingType='Vendor'
|
|
289
|
+
* 5. Type-based default: Credential.IsDefault=1 matching AIVendor.CredentialTypeID
|
|
290
|
+
* 6. Legacy: params.apiKeys[] array
|
|
291
|
+
* 7. Legacy: AI_VENDOR_API_KEY__<DRIVER> environment variables
|
|
292
|
+
*
|
|
293
|
+
* @param driverClass - The driver class name (e.g., 'OpenAILLM')
|
|
294
|
+
* @param promptId - The prompt ID for looking up AIPromptModel bindings
|
|
295
|
+
* @param modelId - The model ID for looking up AIPromptModel and AIModelVendor bindings
|
|
296
|
+
* @param vendorId - The vendor ID for looking up AIModelVendor and AIVendor bindings
|
|
297
|
+
* @param params - The prompt execution parameters
|
|
298
|
+
* @returns true if credentials are available, false otherwise
|
|
299
|
+
*/
|
|
231
300
|
hasCredentialsAvailable(driverClass, promptId, modelId, vendorId, params) {
|
|
301
|
+
// Priority 1: Per-request override
|
|
232
302
|
if (params?.credentialId) {
|
|
303
|
+
// Assume valid if credential ID is provided - will be validated at execution time
|
|
233
304
|
return true;
|
|
234
305
|
}
|
|
306
|
+
// Priority 2: PromptModel bindings
|
|
235
307
|
if (promptId && modelId) {
|
|
236
|
-
const promptModel =
|
|
237
|
-
if (promptModel &&
|
|
308
|
+
const promptModel = AIEngine.Instance.PromptModels.find(pm => pm.PromptID === promptId && pm.ModelID === modelId);
|
|
309
|
+
if (promptModel && AIEngine.Instance.HasCredentialBindings('PromptModel', promptModel.ID)) {
|
|
238
310
|
return true;
|
|
239
311
|
}
|
|
240
312
|
}
|
|
313
|
+
// Priority 3: ModelVendor bindings
|
|
241
314
|
if (modelId && vendorId) {
|
|
242
|
-
const modelVendor =
|
|
243
|
-
if (modelVendor &&
|
|
315
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === modelId && mv.VendorID === vendorId && mv.Status === 'Active');
|
|
316
|
+
if (modelVendor && AIEngine.Instance.HasCredentialBindings('ModelVendor', modelVendor.ID)) {
|
|
244
317
|
return true;
|
|
245
318
|
}
|
|
246
319
|
}
|
|
320
|
+
// Priority 4: Vendor bindings
|
|
247
321
|
if (vendorId) {
|
|
248
|
-
if (
|
|
322
|
+
if (AIEngine.Instance.HasCredentialBindings('Vendor', vendorId)) {
|
|
249
323
|
return true;
|
|
250
324
|
}
|
|
251
325
|
}
|
|
326
|
+
// Priority 5: Type-based default credential
|
|
252
327
|
if (vendorId) {
|
|
253
|
-
const vendor =
|
|
328
|
+
const vendor = AIEngine.Instance.Vendors.find(v => v.ID === vendorId);
|
|
254
329
|
if (vendor?.CredentialTypeID) {
|
|
255
330
|
const defaultCredential = this.findDefaultCredentialByType(vendor.CredentialTypeID);
|
|
256
331
|
if (defaultCredential) {
|
|
@@ -258,12 +333,40 @@ class AIPromptRunner {
|
|
|
258
333
|
}
|
|
259
334
|
}
|
|
260
335
|
}
|
|
261
|
-
|
|
336
|
+
// Priority 6 & 7: Legacy methods - check if API key is available
|
|
337
|
+
const apiKey = GetAIAPIKey(driverClass, params?.apiKeys, params?.verbose);
|
|
262
338
|
return this.isValidAPIKey(apiKey);
|
|
263
339
|
}
|
|
340
|
+
/**
|
|
341
|
+
* Executes an AI prompt with full support for templates, model selection, and validation.
|
|
342
|
+
*
|
|
343
|
+
* @param params Parameters for prompt execution
|
|
344
|
+
* @returns Promise<AIPromptRunResult<T>> The execution result with tracking information
|
|
345
|
+
*
|
|
346
|
+
* @example
|
|
347
|
+
* ```typescript
|
|
348
|
+
* // Execute with specific result type
|
|
349
|
+
* interface AnalysisResult {
|
|
350
|
+
* sentiment: string;
|
|
351
|
+
* score: number;
|
|
352
|
+
* keywords: string[];
|
|
353
|
+
* }
|
|
354
|
+
*
|
|
355
|
+
* const result = await promptRunner.ExecutePrompt<AnalysisResult>({
|
|
356
|
+
* prompt: sentimentPrompt,
|
|
357
|
+
* data: { text: "Customer feedback text" }
|
|
358
|
+
* });
|
|
359
|
+
*
|
|
360
|
+
* if (result.success && result.result) {
|
|
361
|
+
* // result.result is typed as AnalysisResult
|
|
362
|
+
* console.log(`Sentiment: ${result.result.sentiment}, Score: ${result.result.score}`);
|
|
363
|
+
* }
|
|
364
|
+
* ```
|
|
365
|
+
*/
|
|
264
366
|
async ExecutePrompt(params) {
|
|
265
367
|
const startTime = new Date();
|
|
266
368
|
const promptRun = null;
|
|
369
|
+
// Check for cancellation at the start
|
|
267
370
|
if (params.cancellationToken?.aborted) {
|
|
268
371
|
const result = {
|
|
269
372
|
success: false,
|
|
@@ -279,6 +382,7 @@ class AIPromptRunner {
|
|
|
279
382
|
return result;
|
|
280
383
|
}
|
|
281
384
|
try {
|
|
385
|
+
// Use the prompt entity directly from params
|
|
282
386
|
const prompt = params.prompt;
|
|
283
387
|
if (!prompt) {
|
|
284
388
|
throw new Error(`Prompt entity is required`);
|
|
@@ -287,42 +391,60 @@ class AIPromptRunner {
|
|
|
287
391
|
throw new Error(`Prompt ${prompt.Name} is not active (Status: ${prompt.Status})`);
|
|
288
392
|
}
|
|
289
393
|
let renderedPromptText = '';
|
|
394
|
+
// For hierarchical prompts, we need to create the parent prompt run first to get its ID
|
|
290
395
|
let parentPromptRun;
|
|
291
396
|
let selectedModel;
|
|
292
397
|
let childTemplateRenderingResult;
|
|
293
398
|
let modelSelectionInfo;
|
|
399
|
+
// Handle different prompt execution modes
|
|
294
400
|
if (params.childPrompts && params.childPrompts.length > 0) {
|
|
401
|
+
// Hierarchical template composition mode - render child templates first, then compose
|
|
402
|
+
//this.logStatus(`🌳 Composing prompt with ${params.childPrompts.length} child templates in hierarchical mode`, true, params);
|
|
403
|
+
// Determine which prompt to use for model selection
|
|
295
404
|
let modelSelectionPrompt = prompt;
|
|
296
405
|
if (params.modelSelectionPrompt) {
|
|
297
406
|
modelSelectionPrompt = params.modelSelectionPrompt;
|
|
407
|
+
//this.logStatus(`🎯 Using prompt "${modelSelectionPrompt.Name}" for model selection instead of parent prompt`, true, params);
|
|
298
408
|
}
|
|
409
|
+
// Select model using the appropriate prompt
|
|
299
410
|
const modelResult = await this.selectModel(modelSelectionPrompt, params.override?.modelId, params.contextUser, params.configurationId, params.override?.vendorId, params);
|
|
300
411
|
selectedModel = modelResult.model;
|
|
301
412
|
modelSelectionInfo = modelResult.selectionInfo;
|
|
302
413
|
if (!selectedModel) {
|
|
303
414
|
throw new Error(`No suitable model found for prompt ${modelSelectionPrompt.Name}`);
|
|
304
415
|
}
|
|
416
|
+
// Check if we have a system prompt override
|
|
305
417
|
if (params.systemPromptOverride) {
|
|
418
|
+
// Use the override instead of rendering child templates and parent template
|
|
306
419
|
renderedPromptText = params.systemPromptOverride;
|
|
307
420
|
this.logStatus(` Using system prompt override for prompt "${prompt.Name}" (bypassing hierarchical template rendering)`, true, params);
|
|
308
421
|
}
|
|
309
422
|
else {
|
|
423
|
+
// Render all child prompt templates recursively
|
|
310
424
|
childTemplateRenderingResult = await this.renderChildPromptTemplates(params.childPrompts, params, params.cancellationToken);
|
|
425
|
+
// Render the parent prompt with child templates embedded
|
|
311
426
|
renderedPromptText = await this.renderPromptWithChildTemplates(prompt, params, childTemplateRenderingResult.renderedTemplates);
|
|
312
427
|
}
|
|
428
|
+
// Create parent prompt run for the final composed prompt execution
|
|
313
429
|
parentPromptRun = await this.createPromptRun(prompt, selectedModel, params, renderedPromptText, startTime, params.override?.vendorId, modelSelectionInfo);
|
|
314
430
|
}
|
|
315
431
|
else if (prompt.TemplateID && (!params.conversationMessages || params.templateMessageRole !== 'none')) {
|
|
432
|
+
// Check if we have a system prompt override
|
|
316
433
|
if (params.systemPromptOverride) {
|
|
434
|
+
// Use the override instead of rendering the template
|
|
317
435
|
renderedPromptText = params.systemPromptOverride;
|
|
318
436
|
this.logStatus(` Using system prompt override for prompt "${prompt.Name}" (bypassing template rendering)`, true, params);
|
|
319
437
|
}
|
|
320
438
|
else {
|
|
439
|
+
// Regular template rendering mode
|
|
440
|
+
// Initialize template engine
|
|
321
441
|
await this._templateEngine.Config(false, params.contextUser);
|
|
442
|
+
// Load the template for the prompt
|
|
322
443
|
const template = await this.loadTemplate(prompt.TemplateID, params.contextUser);
|
|
323
444
|
if (!template) {
|
|
324
445
|
throw new Error(`Template with ID ${prompt.TemplateID} not found for prompt ${prompt.Name}`);
|
|
325
446
|
}
|
|
447
|
+
// Render the template with full params context
|
|
326
448
|
const renderedPrompt = await this.renderPromptTemplate(template, params);
|
|
327
449
|
if (!renderedPrompt.Success) {
|
|
328
450
|
throw new Error(`Failed to render template for prompt ${prompt.Name}: ${renderedPrompt.Message}`);
|
|
@@ -330,9 +452,11 @@ class AIPromptRunner {
|
|
|
330
452
|
renderedPromptText = renderedPrompt.Output;
|
|
331
453
|
}
|
|
332
454
|
}
|
|
455
|
+
// Check for cancellation after template rendering
|
|
333
456
|
if (params.cancellationToken?.aborted) {
|
|
334
457
|
throw new Error('Prompt execution was cancelled during template rendering');
|
|
335
458
|
}
|
|
459
|
+
// If no model was selected yet (no template case), select one now
|
|
336
460
|
if (!selectedModel) {
|
|
337
461
|
let modelSelectionPrompt = prompt;
|
|
338
462
|
if (params.modelSelectionPrompt) {
|
|
@@ -346,14 +470,20 @@ class AIPromptRunner {
|
|
|
346
470
|
throw new Error(`No suitable model found for prompt ${modelSelectionPrompt.Name}`);
|
|
347
471
|
}
|
|
348
472
|
}
|
|
473
|
+
// Check if we need parallel execution based on ParallelizationMode
|
|
349
474
|
const shouldUseParallelExecution = prompt.ParallelizationMode && prompt.ParallelizationMode !== 'None';
|
|
350
475
|
let result;
|
|
351
476
|
if (shouldUseParallelExecution) {
|
|
477
|
+
// Use parallel execution path
|
|
352
478
|
result = await this.executePromptInParallel(prompt, renderedPromptText, params, startTime, parentPromptRun, selectedModel, modelSelectionInfo);
|
|
353
479
|
}
|
|
354
480
|
else {
|
|
481
|
+
// Use traditional single execution path
|
|
355
482
|
result = await this.executeSinglePrompt(prompt, renderedPromptText, params, startTime, parentPromptRun, selectedModel, modelSelectionInfo);
|
|
356
483
|
}
|
|
484
|
+
// Note: With template composition, we only execute once so no rollup calculations needed
|
|
485
|
+
// The final composed prompt is executed as a single operation
|
|
486
|
+
// Model selection info is now included in the result from both execution methods
|
|
357
487
|
return result;
|
|
358
488
|
}
|
|
359
489
|
catch (error) {
|
|
@@ -367,10 +497,12 @@ class AIPromptRunner {
|
|
|
367
497
|
});
|
|
368
498
|
const endTime = new Date();
|
|
369
499
|
const executionTimeMS = endTime.getTime() - startTime.getTime();
|
|
500
|
+
// Update prompt run with error if it was created
|
|
370
501
|
if (promptRun) {
|
|
371
502
|
promptRun.CompletedAt = endTime;
|
|
372
503
|
promptRun.ExecutionTimeMS = executionTimeMS;
|
|
373
504
|
promptRun.Result = `ERROR: ${error.message}`;
|
|
505
|
+
// Set Status and Cancelled based on error type
|
|
374
506
|
if (error.message.includes('cancelled')) {
|
|
375
507
|
promptRun.Status = 'Cancelled';
|
|
376
508
|
promptRun.Cancelled = true;
|
|
@@ -404,10 +536,21 @@ class AIPromptRunner {
|
|
|
404
536
|
return errorResult;
|
|
405
537
|
}
|
|
406
538
|
}
|
|
539
|
+
/**
|
|
540
|
+
* Executes a single prompt (non-parallel) using traditional model selection.
|
|
541
|
+
*
|
|
542
|
+
* @param prompt - The AI prompt to execute
|
|
543
|
+
* @param renderedPromptText - The rendered prompt text
|
|
544
|
+
* @param params - Original execution parameters
|
|
545
|
+
* @param startTime - Execution start time
|
|
546
|
+
* @returns Promise<AIPromptRunResult<T>> - The execution result
|
|
547
|
+
*/
|
|
407
548
|
async executeSinglePrompt(prompt, renderedPromptText, params, startTime, existingPromptRun, existingModel, existingModelSelectionInfo) {
|
|
549
|
+
// Check for cancellation before model selection
|
|
408
550
|
if (params.cancellationToken?.aborted) {
|
|
409
551
|
throw new Error('Prompt execution was cancelled before model selection');
|
|
410
552
|
}
|
|
553
|
+
// Use existing model if provided (hierarchical case) or select one
|
|
411
554
|
let selectedModel = existingModel;
|
|
412
555
|
let modelSelectionInfo = existingModelSelectionInfo;
|
|
413
556
|
let vendorDriverClass;
|
|
@@ -416,18 +559,21 @@ class AIPromptRunner {
|
|
|
416
559
|
let modelEffortLevel;
|
|
417
560
|
let allCandidates = [];
|
|
418
561
|
if (modelSelectionInfo) {
|
|
562
|
+
// we received model selection info, need to lookup vendor driver class and api name from there
|
|
419
563
|
const vendorID = modelSelectionInfo.vendorSelected?.ID;
|
|
420
564
|
const modelID = modelSelectionInfo.modelSelected.ID;
|
|
421
|
-
const modelVendor =
|
|
565
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find(mv => mv.VendorID === vendorID &&
|
|
422
566
|
mv.ModelID === modelID);
|
|
423
567
|
if (modelVendor) {
|
|
424
568
|
vendorDriverClass = modelVendor.DriverClass;
|
|
425
569
|
vendorApiName = modelVendor.APIName;
|
|
426
570
|
vendorSupportsEffortLevel = modelVendor.SupportsEffortLevel;
|
|
427
571
|
}
|
|
572
|
+
// Extract valid candidates from selection info for retry logic
|
|
428
573
|
allCandidates = this.buildCandidatesFromSelectionInfo(modelSelectionInfo);
|
|
429
574
|
}
|
|
430
575
|
if (!selectedModel) {
|
|
576
|
+
// Determine which prompt to use for model selection
|
|
431
577
|
let modelSelectionPrompt = prompt;
|
|
432
578
|
if (params.modelSelectionPrompt) {
|
|
433
579
|
modelSelectionPrompt = params.modelSelectionPrompt;
|
|
@@ -445,34 +591,46 @@ class AIPromptRunner {
|
|
|
445
591
|
throw new Error(`No suitable model found for prompt ${modelSelectionPrompt.Name}`);
|
|
446
592
|
}
|
|
447
593
|
}
|
|
594
|
+
// Check for cancellation after model selection
|
|
448
595
|
if (params.cancellationToken?.aborted) {
|
|
449
596
|
throw new Error('Prompt execution was cancelled after model selection');
|
|
450
597
|
}
|
|
598
|
+
// Use existing prompt run if provided (hierarchical case) or create new one
|
|
451
599
|
const promptRun = existingPromptRun || await this.createPromptRun(prompt, selectedModel, params, renderedPromptText, startTime, params.override?.vendorId, modelSelectionInfo);
|
|
600
|
+
// Check for cancellation before model execution
|
|
452
601
|
if (params.cancellationToken?.aborted) {
|
|
453
602
|
throw new Error('Prompt execution was cancelled before model execution');
|
|
454
603
|
}
|
|
455
|
-
|
|
604
|
+
// Execute with retry logic for validation failures
|
|
605
|
+
const { modelResult, parsedResult, validationAttempts, cumulativeTokens } = await this.executeWithValidationRetries(selectedModel, renderedPromptText, prompt, params, promptRun, allCandidates, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel // Pass model-specific effort level
|
|
606
|
+
);
|
|
607
|
+
// Calculate execution metrics
|
|
456
608
|
const endTime = new Date();
|
|
457
609
|
const executionTimeMS = endTime.getTime() - startTime.getTime();
|
|
610
|
+
// Update the prompt run with results including validation attempts and cumulative tokens
|
|
458
611
|
await this.updatePromptRun(promptRun, prompt, modelResult, parsedResult, endTime, executionTimeMS, validationAttempts, cumulativeTokens);
|
|
459
612
|
const chatResult = modelResult;
|
|
460
613
|
const usage = chatResult.data?.usage;
|
|
614
|
+
// CRITICAL: Populate errorMessage field when execution fails
|
|
615
|
+
// This ensures errors are properly propagated to BaseAgent and visible in AgentRunStep logs
|
|
461
616
|
let errorMessage;
|
|
462
617
|
if (!chatResult.success) {
|
|
618
|
+
// Model execution failed
|
|
463
619
|
errorMessage = chatResult.errorMessage;
|
|
464
620
|
}
|
|
465
621
|
else if (parsedResult.validationResult?.Success === false) {
|
|
622
|
+
// Validation failed (Warn or Strict mode)
|
|
466
623
|
errorMessage = `Validation failed: ${parsedResult.validationResult.Errors?.map(e => e.Message).join('; ')}`;
|
|
467
624
|
}
|
|
468
625
|
return {
|
|
469
626
|
success: chatResult.success,
|
|
470
627
|
rawResult: chatResult.data?.choices?.[0]?.message?.content,
|
|
471
628
|
result: parsedResult?.result ? parsedResult.result : parsedResult,
|
|
472
|
-
errorMessage,
|
|
629
|
+
errorMessage, // Include error message for proper error propagation
|
|
473
630
|
chatResult,
|
|
474
631
|
promptRun,
|
|
475
632
|
executionTimeMS,
|
|
633
|
+
// Use cumulative tokens if retries occurred, otherwise use single attempt tokens
|
|
476
634
|
promptTokens: cumulativeTokens.promptTokens || usage?.promptTokens,
|
|
477
635
|
completionTokens: cumulativeTokens.completionTokens || usage?.completionTokens,
|
|
478
636
|
tokensUsed: (cumulativeTokens.promptTokens + cumulativeTokens.completionTokens) || ((usage?.promptTokens || 0) + (usage?.completionTokens || 0)),
|
|
@@ -481,21 +639,35 @@ class AIPromptRunner {
|
|
|
481
639
|
validationResult: parsedResult.validationResult,
|
|
482
640
|
validationAttempts,
|
|
483
641
|
combinedTokensUsed: (cumulativeTokens.promptTokens + cumulativeTokens.completionTokens) || ((usage?.promptTokens || 0) + (usage?.completionTokens || 0)),
|
|
484
|
-
modelSelectionInfo
|
|
642
|
+
modelSelectionInfo // Include model selection info if available
|
|
485
643
|
};
|
|
486
644
|
}
|
|
645
|
+
/**
|
|
646
|
+
* Executes a prompt using parallel execution with multiple models/tasks.
|
|
647
|
+
*
|
|
648
|
+
* @param prompt - The AI prompt to execute
|
|
649
|
+
* @param renderedPromptText - The rendered prompt text
|
|
650
|
+
* @param params - Original execution parameters
|
|
651
|
+
* @param startTime - Execution start time
|
|
652
|
+
* @returns Promise<AIPromptRunResult<T>> - The aggregated execution result
|
|
653
|
+
*/
|
|
487
654
|
async executePromptInParallel(prompt, renderedPromptText, params, startTime, existingPromptRun, existingModel, existingModelSelectionInfo) {
|
|
655
|
+
// Check for cancellation before starting parallel execution
|
|
488
656
|
if (params.cancellationToken?.aborted) {
|
|
489
657
|
throw new Error('Parallel execution was cancelled before starting');
|
|
490
658
|
}
|
|
491
|
-
|
|
659
|
+
// Load AI Engine to get models and prompt models
|
|
660
|
+
await AIEngine.Instance.Config(false, params.contextUser);
|
|
492
661
|
let executionTasks;
|
|
662
|
+
// If a model is already selected (from hierarchical template composition),
|
|
663
|
+
// create a single task with that model instead of using the planner
|
|
493
664
|
if (existingModel) {
|
|
665
|
+
// Create a single execution task with the pre-selected model
|
|
494
666
|
executionTasks = [{
|
|
495
667
|
taskId: 'pre-selected',
|
|
496
668
|
model: existingModel,
|
|
497
|
-
vendorDriverClass: undefined,
|
|
498
|
-
vendorApiName: existingModel.Vendor,
|
|
669
|
+
vendorDriverClass: undefined, // Would need to look up vendor entity for this
|
|
670
|
+
vendorApiName: existingModel.Vendor, // Vendor is already the name string
|
|
499
671
|
messages: params.conversationMessages || [],
|
|
500
672
|
promptText: renderedPromptText,
|
|
501
673
|
templateMessageRole: params.templateMessageRole || 'system',
|
|
@@ -504,31 +676,39 @@ class AIPromptRunner {
|
|
|
504
676
|
this.logStatus(` Using pre-selected model "${existingModel.Name}" for parallel execution`, true, params);
|
|
505
677
|
}
|
|
506
678
|
else {
|
|
679
|
+
// Normal parallel execution path - let the planner decide
|
|
680
|
+
// Determine which prompt to use for model selection
|
|
507
681
|
let modelSelectionPrompt = prompt;
|
|
508
682
|
if (params.modelSelectionPrompt) {
|
|
509
683
|
modelSelectionPrompt = params.modelSelectionPrompt;
|
|
510
684
|
this.logStatus(` Using prompt "${modelSelectionPrompt.Name}" for model selection in parallel execution`, true, params);
|
|
511
685
|
}
|
|
512
|
-
|
|
686
|
+
// Get prompt-specific model associations using the model selection prompt
|
|
687
|
+
const promptModels = AIEngine.Instance.PromptModels.filter((pm) => pm.PromptID === modelSelectionPrompt.ID &&
|
|
513
688
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
514
689
|
(!params.configurationId || !pm.ConfigurationID || pm.ConfigurationID === params.configurationId));
|
|
515
|
-
|
|
690
|
+
// Create execution plan using the modelSelectionPrompt for model configurations
|
|
691
|
+
executionTasks = this._executionPlanner.createExecutionPlan(modelSelectionPrompt, promptModels, AIEngine.Instance.Models, renderedPromptText, params.contextUser, params.configurationId, params.conversationMessages, params.templateMessageRole || 'system');
|
|
516
692
|
}
|
|
517
693
|
if (executionTasks.length === 0) {
|
|
518
694
|
throw new Error(`No execution tasks created for parallel execution of prompt ${prompt.Name}`);
|
|
519
695
|
}
|
|
696
|
+
// Check for cancellation before executing tasks
|
|
520
697
|
if (params.cancellationToken?.aborted) {
|
|
521
698
|
throw new Error('Parallel execution was cancelled before task execution');
|
|
522
699
|
}
|
|
700
|
+
// Execute tasks in parallel
|
|
523
701
|
const parallelResult = await this._parallelCoordinator.executeTasksInParallel(params, executionTasks, undefined, undefined, params.cancellationToken, undefined, params.agentRunId);
|
|
524
702
|
if (!parallelResult.success) {
|
|
525
703
|
throw new Error(`Parallel execution failed: ${parallelResult.errors.join(', ')}`);
|
|
526
704
|
}
|
|
705
|
+
// Select best result if multiple successful results
|
|
527
706
|
const successfulResults = parallelResult.taskResults.filter((r) => r.success);
|
|
528
707
|
if (successfulResults.length === 0) {
|
|
529
708
|
throw new Error(`No successful results from parallel execution`);
|
|
530
709
|
}
|
|
531
|
-
let selectedResult = successfulResults[0];
|
|
710
|
+
let selectedResult = successfulResults[0]; // Default to first
|
|
711
|
+
// Use result selector if configured
|
|
532
712
|
if (successfulResults.length > 1 && prompt.ResultSelectorPromptID) {
|
|
533
713
|
const selectionConfig = {
|
|
534
714
|
method: 'PromptSelector',
|
|
@@ -539,6 +719,7 @@ class AIPromptRunner {
|
|
|
539
719
|
selectedResult = aiSelectedResult;
|
|
540
720
|
}
|
|
541
721
|
}
|
|
722
|
+
// Calculate total tokens and costs from all parallel executions
|
|
542
723
|
let totalPromptTokens = 0;
|
|
543
724
|
let totalCompletionTokens = 0;
|
|
544
725
|
let totalCost = 0;
|
|
@@ -554,12 +735,16 @@ class AIPromptRunner {
|
|
|
554
735
|
}
|
|
555
736
|
}
|
|
556
737
|
}
|
|
738
|
+
// Use existing prompt run if provided (hierarchical case) or create new one
|
|
739
|
+
// Use the model selection info if provided (from hierarchical execution)
|
|
557
740
|
const consolidatedPromptRun = existingPromptRun || await this.createPromptRun(prompt, selectedResult.task.model, params, renderedPromptText, startTime, params.override?.vendorId, existingModelSelectionInfo);
|
|
741
|
+
// Update with parallel execution metadata
|
|
558
742
|
const endTime = new Date();
|
|
559
743
|
consolidatedPromptRun.CompletedAt = endTime;
|
|
560
744
|
consolidatedPromptRun.ExecutionTimeMS = parallelResult.totalExecutionTimeMS;
|
|
561
745
|
consolidatedPromptRun.Result = selectedResult.rawResult || '';
|
|
562
746
|
consolidatedPromptRun.TokensUsed = parallelResult.totalTokensUsed;
|
|
747
|
+
// Extract token and cost info from selected result
|
|
563
748
|
const selectedResultUsage = selectedResult.modelResult?.data?.usage;
|
|
564
749
|
if (selectedResultUsage) {
|
|
565
750
|
consolidatedPromptRun.TokensPrompt = selectedResultUsage.promptTokens;
|
|
@@ -571,6 +756,7 @@ class AIPromptRunner {
|
|
|
571
756
|
consolidatedPromptRun.CostCurrency = selectedResultUsage.costCurrency;
|
|
572
757
|
}
|
|
573
758
|
}
|
|
759
|
+
// Add parallel execution metadata to Messages field
|
|
574
760
|
const parallelMetadata = {
|
|
575
761
|
parallelizationMode: prompt.ParallelizationMode,
|
|
576
762
|
totalTasks: executionTasks.length,
|
|
@@ -587,14 +773,16 @@ class AIPromptRunner {
|
|
|
587
773
|
messages: params.conversationMessages || [],
|
|
588
774
|
});
|
|
589
775
|
}
|
|
776
|
+
// For parallel execution, set rollup fields to match totals (no child execution to roll up)
|
|
590
777
|
consolidatedPromptRun.TokensPromptRollup = totalPromptTokens;
|
|
591
778
|
consolidatedPromptRun.TokensCompletionRollup = totalCompletionTokens;
|
|
592
779
|
consolidatedPromptRun.TokensUsedRollup = totalPromptTokens + totalCompletionTokens;
|
|
593
780
|
if (hasCost) {
|
|
594
781
|
consolidatedPromptRun.TotalCost = totalCost;
|
|
595
782
|
}
|
|
783
|
+
// Set Status and WasSelectedResult for parallel execution
|
|
596
784
|
consolidatedPromptRun.Status = parallelResult.successCount > 0 ? 'Completed' : 'Failed';
|
|
597
|
-
consolidatedPromptRun.WasSelectedResult = true;
|
|
785
|
+
consolidatedPromptRun.WasSelectedResult = true; // This is the consolidated result chosen by judge
|
|
598
786
|
const saveResult = await consolidatedPromptRun.Save();
|
|
599
787
|
if (!saveResult) {
|
|
600
788
|
this.logError(`Failed to save consolidated AIPromptRun: ${consolidatedPromptRun.LatestResult?.CompleteMessage || 'Unknown error'}`, {
|
|
@@ -607,7 +795,9 @@ class AIPromptRunner {
|
|
|
607
795
|
maxErrorLength: params.maxErrorLength
|
|
608
796
|
});
|
|
609
797
|
}
|
|
798
|
+
// Create additional results from all other successful results (excluding the best one)
|
|
610
799
|
const additionalResults = [];
|
|
800
|
+
// Sort successful results by ranking (if available) or keep original order
|
|
611
801
|
const sortedResults = successfulResults.sort((a, b) => {
|
|
612
802
|
if (a.ranking && b.ranking) {
|
|
613
803
|
return a.ranking - b.ranking;
|
|
@@ -616,6 +806,7 @@ class AIPromptRunner {
|
|
|
616
806
|
});
|
|
617
807
|
for (const result of sortedResults) {
|
|
618
808
|
if (result.task.taskId !== selectedResult.task.taskId) {
|
|
809
|
+
// Parse and validate this result
|
|
619
810
|
const { result: parsedResultData, validationResult } = await this.parseAndValidateResultEnhanced(result.modelResult, prompt, params.skipValidation, params.cleanValidationSyntax, consolidatedPromptRun, params);
|
|
620
811
|
const parsedResult = { result: parsedResultData, validationResult };
|
|
621
812
|
const resultUsage = result.modelResult?.data?.usage;
|
|
@@ -636,13 +827,14 @@ class AIPromptRunner {
|
|
|
636
827
|
modelInfo: {
|
|
637
828
|
modelId: result.task.model.ID,
|
|
638
829
|
modelName: result.task.model.Name,
|
|
639
|
-
vendorId: undefined,
|
|
830
|
+
vendorId: undefined, // VendorID not directly available on AIModel
|
|
640
831
|
vendorName: result.task.model.Vendor,
|
|
641
832
|
},
|
|
642
833
|
combinedTokensUsed: (resultUsage?.promptTokens || 0) + (resultUsage?.completionTokens || 0)
|
|
643
834
|
});
|
|
644
835
|
}
|
|
645
836
|
}
|
|
837
|
+
// Parse and validate the selected result
|
|
646
838
|
const { result: selectedResultData, validationResult: selectedValidationResult } = await this.parseAndValidateResultEnhanced(selectedResult.modelResult, prompt, params.skipValidation, params.cleanValidationSyntax, consolidatedPromptRun, params);
|
|
647
839
|
const selectedParsedResult = { result: selectedResultData, validationResult: selectedValidationResult };
|
|
648
840
|
const selectedUsage = selectedResult.modelResult?.data?.usage;
|
|
@@ -658,6 +850,7 @@ class AIPromptRunner {
|
|
|
658
850
|
tokensUsed: (selectedUsage?.promptTokens || 0) + (selectedUsage?.completionTokens || 0),
|
|
659
851
|
cost: selectedUsage?.cost,
|
|
660
852
|
costCurrency: selectedUsage?.costCurrency,
|
|
853
|
+
// Combined totals for parallel execution
|
|
661
854
|
combinedPromptTokens: totalPromptTokens,
|
|
662
855
|
combinedCompletionTokens: totalCompletionTokens,
|
|
663
856
|
combinedTokensUsed: totalPromptTokens + totalCompletionTokens,
|
|
@@ -669,15 +862,19 @@ class AIPromptRunner {
|
|
|
669
862
|
modelInfo: {
|
|
670
863
|
modelId: selectedResult.task.model.ID,
|
|
671
864
|
modelName: selectedResult.task.model.Name,
|
|
672
|
-
vendorId: existingModelSelectionInfo?.vendorSelected?.ID,
|
|
865
|
+
vendorId: existingModelSelectionInfo?.vendorSelected?.ID, // VendorID not directly available on AIModel
|
|
673
866
|
vendorName: selectedResult.task.model.Vendor,
|
|
674
867
|
},
|
|
675
868
|
judgeMetadata: selectedResult.judgeMetadata,
|
|
676
|
-
modelSelectionInfo: existingModelSelectionInfo,
|
|
869
|
+
modelSelectionInfo: existingModelSelectionInfo, // Include model selection info if provided
|
|
677
870
|
};
|
|
678
871
|
}
|
|
872
|
+
/**
|
|
873
|
+
* Loads a template entity by ID
|
|
874
|
+
*/
|
|
679
875
|
async loadTemplate(templateId, _contextUser) {
|
|
680
876
|
try {
|
|
877
|
+
// Use the template engine to find the template
|
|
681
878
|
const template = this._templateEngine.Templates.find((t) => t.ID === templateId);
|
|
682
879
|
return template || null;
|
|
683
880
|
}
|
|
@@ -692,41 +889,61 @@ class AIPromptRunner {
|
|
|
692
889
|
return null;
|
|
693
890
|
}
|
|
694
891
|
}
|
|
892
|
+
/**
|
|
893
|
+
* Renders child prompt templates in a depth-first manner, composing them into a final template.
|
|
894
|
+
*
|
|
895
|
+
* @param childPrompts - Array of child prompts to render templates for
|
|
896
|
+
* @param params - Original execution parameters for context
|
|
897
|
+
* @param cancellationToken - Cancellation token for aborting rendering
|
|
898
|
+
* @returns Promise with rendered templates map
|
|
899
|
+
*/
|
|
695
900
|
async renderChildPromptTemplates(childPrompts, params, cancellationToken) {
|
|
696
901
|
if (!childPrompts || childPrompts.length === 0) {
|
|
697
902
|
return {
|
|
698
903
|
renderedTemplates: {}
|
|
699
904
|
};
|
|
700
905
|
}
|
|
906
|
+
// Check for cancellation
|
|
701
907
|
if (cancellationToken?.aborted) {
|
|
702
908
|
throw new Error('Child prompt execution was cancelled');
|
|
703
909
|
}
|
|
910
|
+
//this.logStatus(`🔄 Rendering ${childPrompts.length} child prompt templates in parallel`, true, params);
|
|
911
|
+
// Render all child prompt templates in parallel at this level
|
|
704
912
|
const childRenderingPromises = childPrompts.map(async (childParam) => {
|
|
705
913
|
try {
|
|
914
|
+
// Check for cancellation before each child rendering
|
|
706
915
|
if (cancellationToken?.aborted) {
|
|
707
916
|
throw new Error('Child prompt template rendering was cancelled');
|
|
708
917
|
}
|
|
918
|
+
// First, recursively render any grandchild prompt templates
|
|
709
919
|
let childData = { ...childParam.childPrompt.data };
|
|
710
920
|
if (childParam.childPrompt.childPrompts && childParam.childPrompt.childPrompts.length > 0) {
|
|
711
921
|
const grandchildResults = await this.renderChildPromptTemplates(childParam.childPrompt.childPrompts, params, cancellationToken);
|
|
922
|
+
// Merge grandchild rendered templates into the child's data context
|
|
712
923
|
childData = { ...childData, ...grandchildResults.renderedTemplates };
|
|
713
924
|
}
|
|
925
|
+
// Render the child prompt template with merged data
|
|
926
|
+
//this.logStatus(` 🔹 Rendering child prompt template: ${childParam.childPrompt.prompt.Name} -> ${childParam.parentPlaceholder}`, true, params);
|
|
714
927
|
const childPrompt = childParam.childPrompt.prompt;
|
|
715
928
|
let renderedChildTemplate = '';
|
|
716
929
|
if (childPrompt.TemplateID) {
|
|
930
|
+
// Initialize template engine if not already done
|
|
717
931
|
await this._templateEngine.Config(false, params.contextUser);
|
|
932
|
+
// Load the template for the child prompt
|
|
718
933
|
const template = await this.loadTemplate(childPrompt.TemplateID, params.contextUser);
|
|
719
934
|
if (!template) {
|
|
720
935
|
throw new Error(`Template with ID ${childPrompt.TemplateID} not found for child prompt ${childPrompt.Name}`);
|
|
721
936
|
}
|
|
937
|
+
// Merge child data with original params context
|
|
722
938
|
const mergedChildData = {
|
|
723
|
-
...params.data,
|
|
724
|
-
...childData,
|
|
725
|
-
...childParam.childPrompt.templateData
|
|
939
|
+
...params.data, // Original context
|
|
940
|
+
...childData, // Child-specific data with grandchildren
|
|
941
|
+
...childParam.childPrompt.templateData // Child template data
|
|
726
942
|
};
|
|
943
|
+
// Render the child template
|
|
727
944
|
const childRenderResult = await this.renderPromptTemplate(template, {
|
|
728
|
-
...params,
|
|
729
|
-
prompt: childPrompt,
|
|
945
|
+
...params, // spread original params
|
|
946
|
+
prompt: childPrompt, // THEN, override the prompt for child so we get child related OUTPUT_EXAMPLE and anything else along those lines
|
|
730
947
|
data: mergedChildData,
|
|
731
948
|
templateData: childParam.childPrompt.templateData
|
|
732
949
|
});
|
|
@@ -736,8 +953,10 @@ class AIPromptRunner {
|
|
|
736
953
|
renderedChildTemplate = childRenderResult.Output;
|
|
737
954
|
}
|
|
738
955
|
else {
|
|
956
|
+
// If no template, use empty string (child might be using conversation messages)
|
|
739
957
|
renderedChildTemplate = '';
|
|
740
958
|
}
|
|
959
|
+
// Return the placeholder name and rendered template
|
|
741
960
|
return {
|
|
742
961
|
placeholder: childParam.parentPlaceholder,
|
|
743
962
|
renderedTemplate: renderedChildTemplate,
|
|
@@ -752,6 +971,7 @@ class AIPromptRunner {
|
|
|
752
971
|
},
|
|
753
972
|
maxErrorLength: params.maxErrorLength
|
|
754
973
|
});
|
|
974
|
+
// Return error result but allow other children to continue
|
|
755
975
|
return {
|
|
756
976
|
placeholder: childParam.parentPlaceholder,
|
|
757
977
|
renderedTemplate: `ERROR: ${error.message}`,
|
|
@@ -759,7 +979,9 @@ class AIPromptRunner {
|
|
|
759
979
|
};
|
|
760
980
|
}
|
|
761
981
|
});
|
|
982
|
+
// Wait for all child template rendering to complete
|
|
762
983
|
const childResults = await Promise.all(childRenderingPromises);
|
|
984
|
+
// Check if any critical errors occurred
|
|
763
985
|
const failedChildren = childResults.filter(r => !r.success);
|
|
764
986
|
if (failedChildren.length > 0) {
|
|
765
987
|
this.logError(`${failedChildren.length} out of ${childResults.length} child prompt templates failed to render`, {
|
|
@@ -772,37 +994,57 @@ class AIPromptRunner {
|
|
|
772
994
|
},
|
|
773
995
|
maxErrorLength: params.maxErrorLength
|
|
774
996
|
});
|
|
997
|
+
// any child render failure means we must throw an error
|
|
775
998
|
throw new Error(`Failed to render ${failedChildren.length} child prompt templates: ${failedChildren.map(fc => fc.placeholder).join(', ')}`);
|
|
776
999
|
}
|
|
1000
|
+
// Build rendered templates map
|
|
777
1001
|
const renderedTemplatesMap = {};
|
|
778
1002
|
for (const childResult of childResults) {
|
|
779
1003
|
renderedTemplatesMap[childResult.placeholder] = childResult.renderedTemplate;
|
|
780
1004
|
}
|
|
1005
|
+
//this.logStatus(`✅ Completed rendering of ${childResults.length} child prompt templates`, true, params);
|
|
781
1006
|
return {
|
|
782
1007
|
renderedTemplates: renderedTemplatesMap
|
|
783
1008
|
};
|
|
784
1009
|
}
|
|
1010
|
+
/**
|
|
1011
|
+
* Renders a prompt template with child prompt templates merged into the data context.
|
|
1012
|
+
*
|
|
1013
|
+
* @param prompt - The AI prompt to render
|
|
1014
|
+
* @param params - Original execution parameters
|
|
1015
|
+
* @param childTemplates - Map of placeholder names to rendered child prompt templates
|
|
1016
|
+
* @returns Promise<string> - The rendered prompt text with child templates embedded
|
|
1017
|
+
*/
|
|
785
1018
|
async renderPromptWithChildTemplates(prompt, params, childTemplates) {
|
|
786
1019
|
if (!prompt.TemplateID) {
|
|
1020
|
+
// If no template, return empty string (will be handled by conversation messages)
|
|
787
1021
|
return '';
|
|
788
1022
|
}
|
|
789
1023
|
try {
|
|
1024
|
+
// Initialize template engine
|
|
790
1025
|
await this._templateEngine.Config(false, params.contextUser);
|
|
1026
|
+
// Load the template for the prompt
|
|
791
1027
|
const template = await this.loadTemplate(prompt.TemplateID, params.contextUser);
|
|
792
1028
|
if (!template) {
|
|
793
1029
|
throw new Error(`Template with ID ${prompt.TemplateID} not found for prompt ${prompt.Name}`);
|
|
794
1030
|
}
|
|
795
|
-
|
|
1031
|
+
// Resolve system placeholders with full prompt context
|
|
1032
|
+
const systemPlaceholders = await SystemPlaceholderManager.resolveAllPlaceholders(params);
|
|
1033
|
+
// Merge all data sources with proper priority order
|
|
796
1034
|
const mergedData = {
|
|
797
|
-
...systemPlaceholders,
|
|
798
|
-
...params.data,
|
|
799
|
-
...childTemplates,
|
|
800
|
-
...params.templateData
|
|
1035
|
+
...systemPlaceholders, // System placeholders (lowest priority)
|
|
1036
|
+
...params.data, // Original data context
|
|
1037
|
+
...childTemplates, // Child prompt templates with placeholder names as keys
|
|
1038
|
+
...params.templateData // Additional template data (highest priority)
|
|
801
1039
|
};
|
|
802
1040
|
this.logStatus(` 🔧 ${prompt.Name} [Rendering Prompt Template]`, true, params);
|
|
1041
|
+
// Log placeholder replacement for debugging
|
|
803
1042
|
for (const [placeholder, template] of Object.entries(childTemplates)) {
|
|
804
1043
|
const truncatedTemplate = template.length > 100 ? template.substring(0, 100) + '...' : template;
|
|
1044
|
+
//this.logStatus(` 📝 ${placeholder} -> ${truncatedTemplate}`, true, params);
|
|
805
1045
|
}
|
|
1046
|
+
// Render the template with the full params context
|
|
1047
|
+
// We already have system placeholders resolved, so we'll render directly
|
|
806
1048
|
const renderedPrompt = await this._templateEngine.RenderTemplate(template, template.GetHighestPriorityContent(), mergedData);
|
|
807
1049
|
if (!renderedPrompt.Success) {
|
|
808
1050
|
throw new Error(`Failed to render template for prompt ${prompt.Name}: ${renderedPrompt.Message}`);
|
|
@@ -822,11 +1064,19 @@ class AIPromptRunner {
|
|
|
822
1064
|
throw error;
|
|
823
1065
|
}
|
|
824
1066
|
}
|
|
1067
|
+
/**
|
|
1068
|
+
* Selects the appropriate AI model based on prompt configuration and parameters.
|
|
1069
|
+
* Uses the unified buildModelVendorCandidates method to create an ordered list of candidates,
|
|
1070
|
+
* then selects the first one with an available API key.
|
|
1071
|
+
*/
|
|
825
1072
|
async selectModel(prompt, explicitModelId, contextUser, configurationId, vendorId, params) {
|
|
1073
|
+
// Declare variables outside try block for catch block access
|
|
826
1074
|
let configurationName;
|
|
827
1075
|
let configuration;
|
|
828
1076
|
try {
|
|
829
|
-
|
|
1077
|
+
// Load AI Engine to access cached models and prompt models
|
|
1078
|
+
await AIEngine.Instance.Config(false, contextUser);
|
|
1079
|
+
// Determine selection strategy
|
|
830
1080
|
let selectionStrategy = 'Default';
|
|
831
1081
|
if (explicitModelId) {
|
|
832
1082
|
selectionStrategy = 'Specific';
|
|
@@ -837,11 +1087,14 @@ class AIPromptRunner {
|
|
|
837
1087
|
else if (prompt.SelectionStrategy === 'ByPower' || prompt.MinPowerRank != null) {
|
|
838
1088
|
selectionStrategy = 'ByPower';
|
|
839
1089
|
}
|
|
1090
|
+
// Get configuration info if provided
|
|
840
1091
|
if (configurationId) {
|
|
841
|
-
configuration =
|
|
1092
|
+
configuration = AIEngine.Instance.Configurations.find(c => c.ID === configurationId);
|
|
842
1093
|
configurationName = configuration?.Name;
|
|
843
1094
|
}
|
|
1095
|
+
// Build unified list of model-vendor candidates
|
|
844
1096
|
const candidates = this.buildModelVendorCandidates(prompt, explicitModelId, configurationId, vendorId, params.verbose);
|
|
1097
|
+
// Track all models considered for selection info
|
|
845
1098
|
const modelsConsidered = [];
|
|
846
1099
|
if (candidates.length === 0) {
|
|
847
1100
|
this.logError(`No suitable model candidates found for prompt ${prompt.Name}`, {
|
|
@@ -859,16 +1112,25 @@ class AIPromptRunner {
|
|
|
859
1112
|
selectionInfo: this.createSelectionInfo({
|
|
860
1113
|
aiConfiguration: configuration,
|
|
861
1114
|
modelsConsidered: [],
|
|
862
|
-
modelSelected: undefined,
|
|
1115
|
+
modelSelected: undefined, // Type requirement, but null model means no selection
|
|
863
1116
|
selectionReason: 'No suitable model candidates found',
|
|
864
1117
|
fallbackUsed: false,
|
|
865
1118
|
selectionStrategy
|
|
866
1119
|
})
|
|
867
1120
|
};
|
|
868
1121
|
}
|
|
1122
|
+
// this.logStatus(`🔍 Found ${candidates.length} model-vendor candidates for prompt ${prompt.Name}`, true, params);
|
|
1123
|
+
// if (candidates.length <= 5) {
|
|
1124
|
+
// candidates.forEach((c, i) => {
|
|
1125
|
+
// this.logStatus(` ${i + 1}. ${c.model.Name} via ${c.vendorName || 'default'} (${c.driverClass}) - Priority: ${c.priority}${c.isPreferredVendor ? ' [PREFERRED]' : ''}`, true, params);
|
|
1126
|
+
// });
|
|
1127
|
+
// }
|
|
1128
|
+
// Select the first candidate with available credentials and track all attempts
|
|
869
1129
|
const { selected, consideredModels } = await this.selectModelWithAPIKeyTracked(candidates, prompt.ID, params);
|
|
1130
|
+
// Merge considered models into our tracking
|
|
870
1131
|
modelsConsidered.push(...consideredModels);
|
|
871
1132
|
if (!selected) {
|
|
1133
|
+
// No models with API keys found
|
|
872
1134
|
return {
|
|
873
1135
|
model: null,
|
|
874
1136
|
vendorDriverClass: undefined,
|
|
@@ -879,13 +1141,14 @@ class AIPromptRunner {
|
|
|
879
1141
|
selectionInfo: this.createSelectionInfo({
|
|
880
1142
|
aiConfiguration: configuration,
|
|
881
1143
|
modelsConsidered,
|
|
882
|
-
modelSelected: undefined,
|
|
1144
|
+
modelSelected: undefined, // Type requirement, but null model means no selection
|
|
883
1145
|
selectionReason: 'No API keys found for any model-vendor combination',
|
|
884
1146
|
fallbackUsed: false,
|
|
885
1147
|
selectionStrategy
|
|
886
1148
|
})
|
|
887
1149
|
};
|
|
888
1150
|
}
|
|
1151
|
+
// Determine selection reason
|
|
889
1152
|
let selectionReason = `Selected ${selected.model.Name} via ${selected.vendorName || 'default vendor'}`;
|
|
890
1153
|
if (selected.source === 'explicit') {
|
|
891
1154
|
selectionReason = `Explicitly requested model ${selected.model.Name}`;
|
|
@@ -902,17 +1165,19 @@ class AIPromptRunner {
|
|
|
902
1165
|
if (selected.isPreferredVendor) {
|
|
903
1166
|
selectionReason += ' using preferred vendor';
|
|
904
1167
|
}
|
|
1168
|
+
// Check if fallback was used (not the first candidate)
|
|
905
1169
|
const fallbackUsed = candidates.indexOf(selected) > 0;
|
|
1170
|
+
// Get selected vendor entity
|
|
906
1171
|
let selectedVendor;
|
|
907
1172
|
if (selected.vendorId) {
|
|
908
|
-
selectedVendor =
|
|
1173
|
+
selectedVendor = AIEngine.Instance.Vendors.find(v => v.ID === selected.vendorId);
|
|
909
1174
|
}
|
|
910
1175
|
return {
|
|
911
1176
|
model: selected.model,
|
|
912
1177
|
vendorDriverClass: selected.driverClass,
|
|
913
1178
|
vendorApiName: selected.apiName,
|
|
914
1179
|
vendorSupportsEffortLevel: selected.supportsEffortLevel,
|
|
915
|
-
modelEffortLevel: selected.effortLevel,
|
|
1180
|
+
modelEffortLevel: selected.effortLevel, // Pass through model-specific effort level
|
|
916
1181
|
allCandidates: candidates,
|
|
917
1182
|
selectionInfo: this.createSelectionInfo({
|
|
918
1183
|
aiConfiguration: configuration,
|
|
@@ -941,7 +1206,7 @@ class AIPromptRunner {
|
|
|
941
1206
|
selectionInfo: this.createSelectionInfo({
|
|
942
1207
|
aiConfiguration: configuration,
|
|
943
1208
|
modelsConsidered: [],
|
|
944
|
-
modelSelected: undefined,
|
|
1209
|
+
modelSelected: undefined, // Type requirement, but null model means no selection
|
|
945
1210
|
selectionReason: `Error during model selection: ${error.message}`,
|
|
946
1211
|
fallbackUsed: false,
|
|
947
1212
|
selectionStrategy: 'Default'
|
|
@@ -949,20 +1214,43 @@ class AIPromptRunner {
|
|
|
949
1214
|
};
|
|
950
1215
|
}
|
|
951
1216
|
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Builds a unified, ordered list of model-vendor candidates based on all selection criteria.
|
|
1219
|
+
* Uses a 3-phase approach to properly handle SelectionStrategy='Specific' with AIPromptModel priorities.
|
|
1220
|
+
*
|
|
1221
|
+
* Phase 1: Handle explicit model ID (highest priority)
|
|
1222
|
+
* Phase 2: Check if SelectionStrategy='Specific' with AIPromptModel entries - use ONLY those with AIPromptModel priorities
|
|
1223
|
+
* Phase 3: Use general selection strategy (fallback) - blended priorities from legacy behavior
|
|
1224
|
+
*
|
|
1225
|
+
* @param prompt - The AI prompt with selection criteria
|
|
1226
|
+
* @param explicitModelId - Explicitly specified model ID (highest priority)
|
|
1227
|
+
* @param configurationId - Configuration ID for filtering
|
|
1228
|
+
* @param preferredVendorId - Preferred vendor ID
|
|
1229
|
+
* @returns Ordered array of model-vendor candidates (highest priority first)
|
|
1230
|
+
*/
|
|
952
1231
|
buildModelVendorCandidates(prompt, explicitModelId, configurationId, preferredVendorId, verbose) {
|
|
1232
|
+
// PHASE 1: Handle explicit model ID (highest priority)
|
|
953
1233
|
if (explicitModelId) {
|
|
954
1234
|
return this.buildCandidatesForExplicitModel(explicitModelId, prompt, preferredVendorId);
|
|
955
1235
|
}
|
|
1236
|
+
// PHASE 2: SelectionStrategy='Specific' - Use explicit AIPromptModel configuration
|
|
956
1237
|
if (prompt.SelectionStrategy === 'Specific') {
|
|
957
1238
|
return this.buildCandidatesForSpecificStrategy(prompt, configurationId, verbose);
|
|
958
1239
|
}
|
|
1240
|
+
// PHASE 3: Build candidates with configuration-aware fallback hierarchy
|
|
1241
|
+
// (SelectionStrategy='Default' or 'ByPower')
|
|
959
1242
|
return this.buildCandidatesForGeneralSelection(prompt, configurationId, preferredVendorId, verbose);
|
|
960
1243
|
}
|
|
1244
|
+
/**
|
|
1245
|
+
* PHASE 1: Build candidates for explicitly specified model ID.
|
|
1246
|
+
* Returns candidates for the single model if it's active and compatible.
|
|
1247
|
+
*/
|
|
961
1248
|
buildCandidatesForExplicitModel(explicitModelId, prompt, preferredVendorId) {
|
|
962
|
-
const model =
|
|
1249
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === explicitModelId);
|
|
963
1250
|
if (!model || !model.IsActive) {
|
|
964
1251
|
return [];
|
|
965
1252
|
}
|
|
1253
|
+
// Check model type compatibility
|
|
966
1254
|
if (prompt.AIModelTypeID && model.AIModelTypeID !== prompt.AIModelTypeID) {
|
|
967
1255
|
return [];
|
|
968
1256
|
}
|
|
@@ -970,85 +1258,130 @@ class AIPromptRunner {
|
|
|
970
1258
|
candidates.sort((a, b) => b.priority - a.priority);
|
|
971
1259
|
return candidates;
|
|
972
1260
|
}
|
|
1261
|
+
/**
|
|
1262
|
+
* PHASE 2: Build candidates for 'Specific' selection strategy.
|
|
1263
|
+
* Uses AIPromptModel configuration with clean ranking:
|
|
1264
|
+
* 1. Config-matching models first (by priority DESC)
|
|
1265
|
+
* 2. Then universal (null config) models (by priority DESC)
|
|
1266
|
+
*/
|
|
973
1267
|
buildCandidatesForSpecificStrategy(prompt, configurationId, verbose) {
|
|
974
|
-
|
|
1268
|
+
// Get all active AIPromptModel records for this prompt
|
|
1269
|
+
const allPromptModels = AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID && (pm.Status === 'Active' || pm.Status === 'Preview'));
|
|
1270
|
+
// Filter by configuration matching rules
|
|
975
1271
|
const promptModels = this.filterPromptModelsByConfiguration(allPromptModels, configurationId);
|
|
1272
|
+
// Sort: config-specific before universal, then by priority DESC within each group
|
|
976
1273
|
const sortedPromptModels = this.sortPromptModelsForSpecificStrategy(promptModels, configurationId);
|
|
1274
|
+
// Build candidates maintaining order
|
|
977
1275
|
const candidates = this.buildCandidatesFromPromptModels(sortedPromptModels);
|
|
1276
|
+
// Strategy='Specific' requires explicit configuration
|
|
978
1277
|
if (candidates.length === 0) {
|
|
979
1278
|
const configInfo = configurationId ? ` with configuration "${configurationId}"` : '';
|
|
980
1279
|
throw new Error(`SelectionStrategy is 'Specific' but no valid AIPromptModel candidates found for prompt "${prompt.Name}"${configInfo}. ` +
|
|
981
1280
|
`Please configure AIPromptModel records for this prompt.`);
|
|
982
1281
|
}
|
|
983
1282
|
if (verbose) {
|
|
984
|
-
|
|
1283
|
+
LogStatus(`Using SelectionStrategy='Specific' with ${sortedPromptModels.length} AIPromptModel entries, generated ${candidates.length} candidates`);
|
|
985
1284
|
}
|
|
986
1285
|
return candidates;
|
|
987
1286
|
}
|
|
1287
|
+
/**
|
|
1288
|
+
* PHASE 3: Build candidates for general selection strategies ('Default' or 'ByPower').
|
|
1289
|
+
* Uses configuration-aware fallback hierarchy with legacy blended priority calculation.
|
|
1290
|
+
*/
|
|
988
1291
|
buildCandidatesForGeneralSelection(prompt, configurationId, preferredVendorId, verbose) {
|
|
989
1292
|
const preferredVendorName = preferredVendorId ?
|
|
990
|
-
|
|
1293
|
+
AIEngine.Instance.Vendors.find(v => v.ID === preferredVendorId)?.Name : undefined;
|
|
1294
|
+
// Get prompt models for configuration
|
|
991
1295
|
const promptModels = this.getPromptModelsForConfiguration(prompt, configurationId);
|
|
992
1296
|
const candidates = [];
|
|
993
1297
|
if (promptModels.length > 0) {
|
|
1298
|
+
// Use prompt-specific models with blended priorities
|
|
994
1299
|
this.addPromptSpecificCandidates(candidates, promptModels, preferredVendorId);
|
|
1300
|
+
// Add configuration fallback candidates if needed
|
|
995
1301
|
if (configurationId) {
|
|
996
1302
|
this.addConfigurationFallbackCandidates(candidates, prompt, configurationId, preferredVendorId, verbose);
|
|
997
1303
|
}
|
|
998
1304
|
}
|
|
999
1305
|
else {
|
|
1306
|
+
// No prompt-specific models, use selection strategy
|
|
1000
1307
|
this.addStrategyBasedCandidates(candidates, prompt, preferredVendorName);
|
|
1001
1308
|
}
|
|
1309
|
+
// Sort all candidates by priority (highest first)
|
|
1002
1310
|
candidates.sort((a, b) => b.priority - a.priority);
|
|
1003
1311
|
return candidates;
|
|
1004
1312
|
}
|
|
1313
|
+
/**
|
|
1314
|
+
* Helper: Filter prompt models by configuration matching rules.
|
|
1315
|
+
* Supports configuration inheritance - includes models from the entire inheritance chain.
|
|
1316
|
+
*/
|
|
1005
1317
|
filterPromptModelsByConfiguration(allPromptModels, configurationId) {
|
|
1006
1318
|
if (configurationId) {
|
|
1007
|
-
|
|
1319
|
+
// Get the configuration inheritance chain
|
|
1320
|
+
const chain = AIEngine.Instance.GetConfigurationChain(configurationId);
|
|
1008
1321
|
const chainIds = new Set(chain.map(c => c.ID));
|
|
1322
|
+
// Include models matching any config in the chain, plus null-config (universal fallback)
|
|
1009
1323
|
return allPromptModels.filter(pm => (pm.ConfigurationID && chainIds.has(pm.ConfigurationID)) ||
|
|
1010
1324
|
pm.ConfigurationID === null);
|
|
1011
1325
|
}
|
|
1012
1326
|
else {
|
|
1327
|
+
// No config specified - only include null-config models
|
|
1013
1328
|
return allPromptModels.filter(pm => pm.ConfigurationID === null);
|
|
1014
1329
|
}
|
|
1015
1330
|
}
|
|
1331
|
+
/**
|
|
1332
|
+
* Helper: Sort prompt models for 'Specific' strategy.
|
|
1333
|
+
* Respects configuration inheritance chain - child configs first, then parents, then null-config.
|
|
1334
|
+
* Within each config level, sorts by priority DESC.
|
|
1335
|
+
*/
|
|
1016
1336
|
sortPromptModelsForSpecificStrategy(promptModels, configurationId) {
|
|
1017
1337
|
if (!configurationId) {
|
|
1338
|
+
// No config specified - just sort by priority
|
|
1018
1339
|
return promptModels.sort((a, b) => (b.Priority || 0) - (a.Priority || 0));
|
|
1019
1340
|
}
|
|
1020
|
-
|
|
1341
|
+
// Get the configuration inheritance chain and create position map
|
|
1342
|
+
const chain = AIEngine.Instance.GetConfigurationChain(configurationId);
|
|
1021
1343
|
const chainOrder = new Map(chain.map((c, index) => [c.ID, index]));
|
|
1022
1344
|
return promptModels.sort((a, b) => {
|
|
1345
|
+
// Primary: Chain position (lower index = higher priority, null config = last)
|
|
1023
1346
|
const aChainPos = a.ConfigurationID ? (chainOrder.get(a.ConfigurationID) ?? 999) : 1000;
|
|
1024
1347
|
const bChainPos = b.ConfigurationID ? (chainOrder.get(b.ConfigurationID) ?? 999) : 1000;
|
|
1025
1348
|
if (aChainPos !== bChainPos) {
|
|
1026
|
-
return aChainPos - bChainPos;
|
|
1349
|
+
return aChainPos - bChainPos; // Lower chain position first (child before parent)
|
|
1027
1350
|
}
|
|
1351
|
+
// Secondary: Higher priority first within same config level
|
|
1028
1352
|
return (b.Priority || 0) - (a.Priority || 0);
|
|
1029
1353
|
});
|
|
1030
1354
|
}
|
|
1355
|
+
/**
|
|
1356
|
+
* Helper: Build candidates from sorted AIPromptModel records.
|
|
1357
|
+
* Expands VendorID=null to all vendors for that model.
|
|
1358
|
+
*/
|
|
1031
1359
|
buildCandidatesFromPromptModels(promptModels) {
|
|
1032
1360
|
const candidates = [];
|
|
1033
1361
|
for (const pm of promptModels) {
|
|
1034
|
-
const model =
|
|
1362
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === pm.ModelID);
|
|
1035
1363
|
if (!model || !model.IsActive)
|
|
1036
1364
|
continue;
|
|
1037
1365
|
if (pm.VendorID) {
|
|
1366
|
+
// Specific vendor specified - create single candidate
|
|
1038
1367
|
const candidate = this.createCandidateForSpecificVendor(model, pm);
|
|
1039
1368
|
if (candidate) {
|
|
1040
1369
|
candidates.push(candidate);
|
|
1041
1370
|
}
|
|
1042
1371
|
}
|
|
1043
1372
|
else {
|
|
1373
|
+
// No vendor specified - create candidates for all vendors
|
|
1044
1374
|
const vendorCandidates = this.createCandidatesForAllVendors(model);
|
|
1045
1375
|
candidates.push(...vendorCandidates);
|
|
1046
1376
|
}
|
|
1047
1377
|
}
|
|
1048
1378
|
return candidates;
|
|
1049
1379
|
}
|
|
1380
|
+
/**
|
|
1381
|
+
* Helper: Create candidate for specific vendor from AIPromptModel.
|
|
1382
|
+
*/
|
|
1050
1383
|
createCandidateForSpecificVendor(model, promptModel) {
|
|
1051
|
-
const modelVendor =
|
|
1384
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === promptModel.ModelID &&
|
|
1052
1385
|
mv.VendorID === promptModel.VendorID &&
|
|
1053
1386
|
mv.Status === 'Active' &&
|
|
1054
1387
|
this.isInferenceProvider(mv));
|
|
@@ -1061,14 +1394,17 @@ class AIPromptRunner {
|
|
|
1061
1394
|
driverClass: modelVendor.DriverClass || model.DriverClass,
|
|
1062
1395
|
apiName: modelVendor.APIName || model.APIName,
|
|
1063
1396
|
supportsEffortLevel: modelVendor.SupportsEffortLevel ?? model.SupportsEffortLevel ?? false,
|
|
1064
|
-
effortLevel: promptModel.EffortLevel ?? undefined,
|
|
1397
|
+
effortLevel: promptModel.EffortLevel ?? undefined, // Model-specific effort level override
|
|
1065
1398
|
isPreferredVendor: false,
|
|
1066
|
-
priority: 0,
|
|
1399
|
+
priority: 0, // Order is determined by promptModels sort
|
|
1067
1400
|
source: 'prompt-model'
|
|
1068
1401
|
};
|
|
1069
1402
|
}
|
|
1403
|
+
/**
|
|
1404
|
+
* Helper: Create candidates for all vendors of a model, sorted by vendor priority.
|
|
1405
|
+
*/
|
|
1070
1406
|
createCandidatesForAllVendors(model) {
|
|
1071
|
-
const vendors =
|
|
1407
|
+
const vendors = AIEngine.Instance.ModelVendors
|
|
1072
1408
|
.filter(mv => mv.ModelID === model.ID &&
|
|
1073
1409
|
mv.Status === 'Active' &&
|
|
1074
1410
|
this.isInferenceProvider(mv))
|
|
@@ -1083,10 +1419,11 @@ class AIPromptRunner {
|
|
|
1083
1419
|
apiName: vendor.APIName || model.APIName,
|
|
1084
1420
|
supportsEffortLevel: vendor.SupportsEffortLevel ?? model.SupportsEffortLevel ?? false,
|
|
1085
1421
|
isPreferredVendor: false,
|
|
1086
|
-
priority: 0,
|
|
1422
|
+
priority: 0, // Order is determined by promptModels sort
|
|
1087
1423
|
source: 'prompt-model'
|
|
1088
1424
|
});
|
|
1089
1425
|
}
|
|
1426
|
+
// If no vendors found, use model defaults
|
|
1090
1427
|
if (candidates.length === 0 && model.DriverClass) {
|
|
1091
1428
|
candidates.push({
|
|
1092
1429
|
model,
|
|
@@ -1100,94 +1437,128 @@ class AIPromptRunner {
|
|
|
1100
1437
|
}
|
|
1101
1438
|
return candidates;
|
|
1102
1439
|
}
|
|
1440
|
+
/**
|
|
1441
|
+
* Helper: Get prompt models for configuration with inheritance chain fallback.
|
|
1442
|
+
* Walks the configuration inheritance chain looking for prompt models.
|
|
1443
|
+
* Returns models from the first config in the chain that has any, or falls back to null-config.
|
|
1444
|
+
*/
|
|
1103
1445
|
getPromptModelsForConfiguration(prompt, configurationId) {
|
|
1104
1446
|
if (configurationId) {
|
|
1105
|
-
|
|
1447
|
+
// Get the configuration inheritance chain (child -> parent -> grandparent -> ...)
|
|
1448
|
+
const chain = AIEngine.Instance.GetConfigurationChain(configurationId);
|
|
1449
|
+
// Walk the chain looking for prompt models
|
|
1106
1450
|
for (const config of chain) {
|
|
1107
|
-
const promptModels =
|
|
1451
|
+
const promptModels = AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID &&
|
|
1108
1452
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
1109
1453
|
pm.ConfigurationID === config.ID);
|
|
1110
1454
|
if (promptModels.length > 0) {
|
|
1111
1455
|
return promptModels;
|
|
1112
1456
|
}
|
|
1113
1457
|
}
|
|
1114
|
-
|
|
1458
|
+
// No match in chain, fall back to NULL config models
|
|
1459
|
+
LogStatus(`No models found in configuration chain for "${configurationId}", falling back to default models`);
|
|
1115
1460
|
}
|
|
1116
|
-
|
|
1461
|
+
// Return null-config (universal) models
|
|
1462
|
+
return AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID &&
|
|
1117
1463
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
1118
1464
|
!pm.ConfigurationID);
|
|
1119
1465
|
}
|
|
1466
|
+
/**
|
|
1467
|
+
* Helper: Add prompt-specific candidates with blended priorities (legacy behavior).
|
|
1468
|
+
*/
|
|
1120
1469
|
addPromptSpecificCandidates(candidates, promptModels, preferredVendorId) {
|
|
1121
1470
|
for (const pm of promptModels) {
|
|
1122
|
-
const model =
|
|
1471
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === pm.ModelID);
|
|
1123
1472
|
if (model && model.IsActive) {
|
|
1124
1473
|
const modelCandidates = this.createCandidatesForModel(model, 5000, 'prompt-model', preferredVendorId, pm.Priority);
|
|
1125
1474
|
candidates.push(...modelCandidates);
|
|
1126
1475
|
}
|
|
1127
1476
|
}
|
|
1128
1477
|
}
|
|
1478
|
+
/**
|
|
1479
|
+
* Helper: Add configuration fallback candidates from the inheritance chain.
|
|
1480
|
+
* Adds models from parent configs (with decreasing priority) and null-config models as final fallback.
|
|
1481
|
+
*/
|
|
1129
1482
|
addConfigurationFallbackCandidates(candidates, prompt, configurationId, preferredVendorId, verbose) {
|
|
1130
|
-
const chain =
|
|
1483
|
+
const chain = AIEngine.Instance.GetConfigurationChain(configurationId);
|
|
1484
|
+
// Add models from parent configs (skip index 0 which is the direct config, already handled)
|
|
1131
1485
|
for (let i = 1; i < chain.length; i++) {
|
|
1132
1486
|
const parentConfig = chain[i];
|
|
1133
|
-
const parentModels =
|
|
1487
|
+
const parentModels = AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID &&
|
|
1134
1488
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
1135
1489
|
pm.ConfigurationID === parentConfig.ID);
|
|
1136
1490
|
if (parentModels.length > 0 && verbose) {
|
|
1137
|
-
|
|
1491
|
+
LogStatus(`Adding ${parentModels.length} models from parent config "${parentConfig.Name}" as fallback`);
|
|
1138
1492
|
}
|
|
1139
1493
|
for (const pm of parentModels) {
|
|
1140
|
-
const model =
|
|
1494
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === pm.ModelID);
|
|
1141
1495
|
if (model && model.IsActive) {
|
|
1496
|
+
// Decrease base priority for each level up the chain (3000, 2500, 2000, etc.)
|
|
1142
1497
|
const basePriority = 3000 - (i * 500);
|
|
1143
1498
|
const modelCandidates = this.createCandidatesForModel(model, basePriority, 'prompt-model', preferredVendorId, pm.Priority);
|
|
1144
1499
|
candidates.push(...modelCandidates);
|
|
1145
1500
|
}
|
|
1146
1501
|
}
|
|
1147
1502
|
}
|
|
1148
|
-
|
|
1503
|
+
// Finally add NULL config models (universal fallback) with lowest priority
|
|
1504
|
+
const nullConfigModels = AIEngine.Instance.PromptModels.filter(pm => pm.PromptID === prompt.ID &&
|
|
1149
1505
|
(pm.Status === 'Active' || pm.Status === 'Preview') &&
|
|
1150
1506
|
!pm.ConfigurationID);
|
|
1151
1507
|
if (nullConfigModels.length > 0 && verbose) {
|
|
1152
|
-
|
|
1508
|
+
LogStatus(`Adding ${nullConfigModels.length} NULL configuration models as universal fallback`);
|
|
1153
1509
|
}
|
|
1154
1510
|
for (const pm of nullConfigModels) {
|
|
1155
|
-
const model =
|
|
1511
|
+
const model = AIEngine.Instance.Models.find(m => m.ID === pm.ModelID);
|
|
1156
1512
|
if (model && model.IsActive) {
|
|
1157
|
-
const modelCandidates = this.createCandidatesForModel(model, 1000,
|
|
1513
|
+
const modelCandidates = this.createCandidatesForModel(model, 1000, // Lowest priority tier
|
|
1514
|
+
'prompt-model', preferredVendorId, pm.Priority);
|
|
1158
1515
|
candidates.push(...modelCandidates);
|
|
1159
1516
|
}
|
|
1160
1517
|
}
|
|
1161
1518
|
}
|
|
1519
|
+
/**
|
|
1520
|
+
* Helper: Add strategy-based candidates when no prompt models exist.
|
|
1521
|
+
*/
|
|
1162
1522
|
addStrategyBasedCandidates(candidates, prompt, preferredVendorName) {
|
|
1163
1523
|
let modelPool = this.getModelPoolForStrategy(prompt, preferredVendorName);
|
|
1164
1524
|
modelPool = this.sortModelPoolByStrategy(modelPool, prompt);
|
|
1525
|
+
// Create candidates for each model in the pool
|
|
1165
1526
|
modelPool.forEach((model, index) => {
|
|
1166
|
-
const basePriority = 1000 - index * 10;
|
|
1527
|
+
const basePriority = 1000 - index * 10; // Decrease priority by position
|
|
1167
1528
|
const source = prompt.SelectionStrategy === 'ByPower' ? 'power-rank' : 'model-type';
|
|
1168
1529
|
candidates.push(...this.createCandidatesForModel(model, basePriority, source));
|
|
1169
1530
|
});
|
|
1170
1531
|
}
|
|
1532
|
+
/**
|
|
1533
|
+
* Helper: Get model pool filtered for strategy.
|
|
1534
|
+
*/
|
|
1171
1535
|
getModelPoolForStrategy(prompt, preferredVendorName) {
|
|
1172
|
-
return
|
|
1536
|
+
return AIEngine.Instance.Models.filter(m => m.IsActive &&
|
|
1173
1537
|
(!prompt.AIModelTypeID || m.AIModelTypeID === prompt.AIModelTypeID) &&
|
|
1174
1538
|
(!preferredVendorName ||
|
|
1175
|
-
|
|
1539
|
+
AIEngine.Instance.ModelVendors.some(mv => mv.ModelID === m.ID &&
|
|
1176
1540
|
mv.Status === 'Active' &&
|
|
1177
1541
|
mv.Vendor === preferredVendorName &&
|
|
1178
1542
|
this.isInferenceProvider(mv))));
|
|
1179
1543
|
}
|
|
1544
|
+
/**
|
|
1545
|
+
* Helper: Sort model pool by selection strategy.
|
|
1546
|
+
*/
|
|
1180
1547
|
sortModelPoolByStrategy(modelPool, prompt) {
|
|
1181
1548
|
if (prompt.SelectionStrategy === 'ByPower') {
|
|
1182
1549
|
return this.sortByPowerPreference(modelPool, prompt.PowerPreference);
|
|
1183
1550
|
}
|
|
1184
1551
|
else {
|
|
1552
|
+
// Default strategy
|
|
1185
1553
|
const minPowerRank = prompt.MinPowerRank || 0;
|
|
1186
1554
|
return modelPool
|
|
1187
1555
|
.filter(m => m.PowerRank >= minPowerRank)
|
|
1188
1556
|
.sort((a, b) => b.PowerRank - a.PowerRank);
|
|
1189
1557
|
}
|
|
1190
1558
|
}
|
|
1559
|
+
/**
|
|
1560
|
+
* Helper: Sort models by power preference.
|
|
1561
|
+
*/
|
|
1191
1562
|
sortByPowerPreference(modelPool, powerPreference) {
|
|
1192
1563
|
const pool = [...modelPool];
|
|
1193
1564
|
switch (powerPreference) {
|
|
@@ -1202,11 +1573,16 @@ class AIPromptRunner {
|
|
|
1202
1573
|
return pool.sort((a, b) => b.PowerRank - a.PowerRank);
|
|
1203
1574
|
}
|
|
1204
1575
|
}
|
|
1576
|
+
/**
|
|
1577
|
+
* Helper: Create candidates for a model with AIModelVendor priorities (legacy behavior).
|
|
1578
|
+
*/
|
|
1205
1579
|
createCandidatesForModel(model, basePriority, source, preferredVendorId, promptModelPriority) {
|
|
1206
1580
|
const modelCandidates = [];
|
|
1207
|
-
|
|
1581
|
+
// Get all vendors for this model - filter for inference providers only
|
|
1582
|
+
const modelVendors = AIEngine.Instance.ModelVendors
|
|
1208
1583
|
.filter(mv => mv.ModelID === model.ID && mv.Status === 'Active' && this.isInferenceProvider(mv))
|
|
1209
1584
|
.sort((a, b) => b.Priority - a.Priority);
|
|
1585
|
+
// First, add preferred vendor if it exists
|
|
1210
1586
|
if (preferredVendorId) {
|
|
1211
1587
|
const preferredVendor = modelVendors.find(mv => mv.VendorID === preferredVendorId);
|
|
1212
1588
|
if (preferredVendor) {
|
|
@@ -1218,11 +1594,12 @@ class AIPromptRunner {
|
|
|
1218
1594
|
apiName: preferredVendor.APIName || model.APIName,
|
|
1219
1595
|
supportsEffortLevel: preferredVendor.SupportsEffortLevel ?? model.SupportsEffortLevel ?? false,
|
|
1220
1596
|
isPreferredVendor: true,
|
|
1221
|
-
priority: basePriority + 1000,
|
|
1597
|
+
priority: basePriority + 1000, // Boost priority for preferred vendor
|
|
1222
1598
|
source
|
|
1223
1599
|
});
|
|
1224
1600
|
}
|
|
1225
1601
|
}
|
|
1602
|
+
// Then add other vendors in priority order
|
|
1226
1603
|
for (const vendor of modelVendors) {
|
|
1227
1604
|
if (vendor.VendorID !== preferredVendorId) {
|
|
1228
1605
|
modelCandidates.push({
|
|
@@ -1238,6 +1615,7 @@ class AIPromptRunner {
|
|
|
1238
1615
|
});
|
|
1239
1616
|
}
|
|
1240
1617
|
}
|
|
1618
|
+
// If no vendors found, add model with its default driver
|
|
1241
1619
|
if (modelCandidates.length === 0 && model.DriverClass) {
|
|
1242
1620
|
modelCandidates.push({
|
|
1243
1621
|
model,
|
|
@@ -1249,21 +1627,34 @@ class AIPromptRunner {
|
|
|
1249
1627
|
source
|
|
1250
1628
|
});
|
|
1251
1629
|
}
|
|
1630
|
+
// Apply prompt model priority if provided (legacy blended approach)
|
|
1252
1631
|
if (promptModelPriority !== undefined) {
|
|
1253
1632
|
modelCandidates.forEach(c => c.priority += promptModelPriority * 10);
|
|
1254
1633
|
}
|
|
1255
1634
|
return modelCandidates;
|
|
1256
1635
|
}
|
|
1636
|
+
/**
|
|
1637
|
+
* Creates a properly typed AIModelSelectionInfo instance.
|
|
1638
|
+
* TypeScript requires instantiating the class to get the getValidCandidates() method.
|
|
1639
|
+
*/
|
|
1257
1640
|
createSelectionInfo(data) {
|
|
1258
|
-
const info = new
|
|
1641
|
+
const info = new AIModelSelectionInfo();
|
|
1259
1642
|
Object.assign(info, data);
|
|
1260
1643
|
return info;
|
|
1261
1644
|
}
|
|
1645
|
+
/**
|
|
1646
|
+
* Converts model selection info into ModelVendorCandidate array for retry logic.
|
|
1647
|
+
* Extracts only the valid candidates (those with available API keys) from the selection info.
|
|
1648
|
+
*
|
|
1649
|
+
* @param selectionInfo - Model selection information containing considered models
|
|
1650
|
+
* @returns Array of valid model-vendor candidates sorted by priority
|
|
1651
|
+
*/
|
|
1262
1652
|
buildCandidatesFromSelectionInfo(selectionInfo) {
|
|
1263
1653
|
const validModels = selectionInfo.extractValidCandidates();
|
|
1264
1654
|
return validModels.map(considered => {
|
|
1655
|
+
// Find matching model vendor for driver and API info
|
|
1265
1656
|
const modelVendor = considered.vendor
|
|
1266
|
-
?
|
|
1657
|
+
? AIEngine.Instance.ModelVendors.find(mv => mv.ModelID === considered.model.ID &&
|
|
1267
1658
|
mv.VendorID === considered.vendor.ID)
|
|
1268
1659
|
: undefined;
|
|
1269
1660
|
return {
|
|
@@ -1273,29 +1664,47 @@ class AIPromptRunner {
|
|
|
1273
1664
|
driverClass: modelVendor?.DriverClass || considered.model.DriverClass,
|
|
1274
1665
|
apiName: modelVendor?.APIName || considered.model.APIName,
|
|
1275
1666
|
supportsEffortLevel: modelVendor?.SupportsEffortLevel ?? considered.model.SupportsEffortLevel ?? false,
|
|
1276
|
-
isPreferredVendor: false,
|
|
1667
|
+
isPreferredVendor: false, // Can't determine from selection info alone
|
|
1277
1668
|
priority: considered.priority,
|
|
1278
1669
|
source: (selectionInfo.selectionStrategy === 'ByPower' ? 'power-rank' : 'model-type')
|
|
1279
1670
|
};
|
|
1280
|
-
}).sort((a, b) => b.priority - a.priority);
|
|
1281
|
-
}
|
|
1671
|
+
}).sort((a, b) => b.priority - a.priority); // Sort by priority descending
|
|
1672
|
+
}
|
|
1673
|
+
/**
|
|
1674
|
+
* Enhanced version of selectModelWithAPIKey that tracks all considered models
|
|
1675
|
+
* for model selection reporting. Uses the hierarchical credential resolution
|
|
1676
|
+
* system to check for available credentials.
|
|
1677
|
+
*
|
|
1678
|
+
* @param candidates - Ordered array of model-vendor candidates
|
|
1679
|
+
* @param promptId - The prompt ID for credential resolution
|
|
1680
|
+
* @param params - Optional prompt parameters for verbose logging and credential override
|
|
1681
|
+
* @returns Object containing selected candidate and all considered models
|
|
1682
|
+
*/
|
|
1282
1683
|
async selectModelWithAPIKeyTracked(candidates, promptId, params) {
|
|
1684
|
+
// Cache for credential availability checks
|
|
1685
|
+
// Key format: "driverClass:modelId:vendorId" to properly cache credential hierarchy
|
|
1283
1686
|
const credentialCache = new Map();
|
|
1284
1687
|
const consideredModels = [];
|
|
1688
|
+
// Check ALL candidates to build complete list of valid and invalid options
|
|
1285
1689
|
for (const candidate of candidates) {
|
|
1690
|
+
// Build cache key including model and vendor for proper credential resolution
|
|
1286
1691
|
const cacheKey = `${candidate.driverClass}:${candidate.model.ID}:${candidate.vendorId || 'default'}`;
|
|
1692
|
+
// Check cache first
|
|
1287
1693
|
let hasCredentials;
|
|
1288
1694
|
if (credentialCache.has(cacheKey)) {
|
|
1289
1695
|
hasCredentials = credentialCache.get(cacheKey);
|
|
1290
1696
|
}
|
|
1291
1697
|
else {
|
|
1698
|
+
// Check for credentials using hierarchical resolution
|
|
1292
1699
|
hasCredentials = this.hasCredentialsAvailable(candidate.driverClass, promptId, candidate.model.ID, candidate.vendorId, params);
|
|
1293
1700
|
credentialCache.set(cacheKey, hasCredentials);
|
|
1294
1701
|
}
|
|
1702
|
+
// Get vendor entity from AIEngine cache if vendorId is available
|
|
1295
1703
|
let vendorEntity;
|
|
1296
1704
|
if (candidate.vendorId) {
|
|
1297
|
-
vendorEntity =
|
|
1705
|
+
vendorEntity = AIEngine.Instance.Vendors.find(v => v.ID === candidate.vendorId);
|
|
1298
1706
|
}
|
|
1707
|
+
// Track this model as considered with availability status
|
|
1299
1708
|
consideredModels.push({
|
|
1300
1709
|
model: candidate.model,
|
|
1301
1710
|
vendor: vendorEntity,
|
|
@@ -1304,6 +1713,7 @@ class AIPromptRunner {
|
|
|
1304
1713
|
unavailableReason: hasCredentials ? undefined : `No credentials configured for driver ${candidate.driverClass}`
|
|
1305
1714
|
});
|
|
1306
1715
|
}
|
|
1716
|
+
// Select the first available candidate (highest priority with API key)
|
|
1307
1717
|
const selected = consideredModels.find(m => m.available);
|
|
1308
1718
|
const selectedCandidate = selected ? candidates.find(c => c.model.ID === selected.model.ID &&
|
|
1309
1719
|
c.vendorId === selected.vendor?.ID) : null;
|
|
@@ -1316,6 +1726,7 @@ class AIPromptRunner {
|
|
|
1316
1726
|
this.logStatus(` Found ${validCount} valid candidate(s) out of ${candidates.length} total`, true, params);
|
|
1317
1727
|
}
|
|
1318
1728
|
else {
|
|
1729
|
+
// Log what we tried
|
|
1319
1730
|
const triedSummary = candidates.slice(0, 5).map(c => `${c.model.Name}/${c.vendorName || 'default'}(${c.driverClass})`).join(', ');
|
|
1320
1731
|
this.logError(`No credentials found for any model-vendor combination. Tried: ${triedSummary}${candidates.length > 5 ? `... (${candidates.length} total)` : ''}`, {
|
|
1321
1732
|
category: 'CredentialValidation',
|
|
@@ -1329,18 +1740,24 @@ class AIPromptRunner {
|
|
|
1329
1740
|
}
|
|
1330
1741
|
return { selected: selectedCandidate, consideredModels };
|
|
1331
1742
|
}
|
|
1743
|
+
/**
|
|
1744
|
+
* Creates an AIPromptRun entity for execution tracking
|
|
1745
|
+
*/
|
|
1332
1746
|
async createPromptRun(prompt, model, params, systemPromptText, startTime, vendorId, modelSelectionInfo) {
|
|
1333
1747
|
const promptRun = await this._metadata.GetEntityObject('MJ: AI Prompt Runs', params.contextUser);
|
|
1334
1748
|
try {
|
|
1335
1749
|
promptRun.NewRecord();
|
|
1336
1750
|
promptRun.PromptID = prompt.ID;
|
|
1337
1751
|
promptRun.ModelID = model.ID;
|
|
1752
|
+
// Set initial status and tracking fields
|
|
1338
1753
|
promptRun.Status = 'Running';
|
|
1339
1754
|
promptRun.Cancelled = false;
|
|
1340
1755
|
promptRun.CacheHit = false;
|
|
1341
1756
|
promptRun.StreamingEnabled = false;
|
|
1342
1757
|
promptRun.WasSelectedResult = false;
|
|
1758
|
+
// Set model selection tracking fields
|
|
1343
1759
|
if (modelSelectionInfo) {
|
|
1760
|
+
// Convert the rich entity objects to simple IDs/names for database storage
|
|
1344
1761
|
const dbSelectionInfo = {
|
|
1345
1762
|
configurationId: modelSelectionInfo.aiConfiguration?.ID,
|
|
1346
1763
|
configurationName: modelSelectionInfo.aiConfiguration?.Name,
|
|
@@ -1361,28 +1778,35 @@ class AIPromptRunner {
|
|
|
1361
1778
|
};
|
|
1362
1779
|
promptRun.ModelSelection = JSON.stringify(dbSelectionInfo);
|
|
1363
1780
|
promptRun.SelectionStrategy = modelSelectionInfo.selectionStrategy || 'Default';
|
|
1781
|
+
// Set ModelPowerRank if available
|
|
1364
1782
|
if (model.PowerRank != null) {
|
|
1365
1783
|
promptRun.ModelPowerRank = model.PowerRank;
|
|
1366
1784
|
}
|
|
1367
1785
|
}
|
|
1786
|
+
// Set original model tracking for failover
|
|
1368
1787
|
promptRun.OriginalModelID = model.ID;
|
|
1369
1788
|
promptRun.OriginalRequestStartTime = startTime;
|
|
1789
|
+
// Initialize failover tracking fields
|
|
1370
1790
|
promptRun.FailoverAttempts = 0;
|
|
1371
1791
|
promptRun.FailoverErrors = null;
|
|
1372
1792
|
promptRun.FailoverDurations = null;
|
|
1373
1793
|
promptRun.TotalFailoverDuration = 0;
|
|
1794
|
+
// Check if model has pre-selected vendor info from selectModel
|
|
1374
1795
|
const modelWithVendor = model;
|
|
1375
1796
|
if (modelSelectionInfo) {
|
|
1376
1797
|
promptRun.VendorID = modelSelectionInfo.vendorSelected?.ID || vendorId || modelWithVendor._selectedVendorId;
|
|
1377
1798
|
}
|
|
1378
1799
|
else if (vendorId) {
|
|
1800
|
+
// Explicit vendor ID provided
|
|
1379
1801
|
promptRun.VendorID = vendorId;
|
|
1380
1802
|
}
|
|
1381
1803
|
else if (modelWithVendor._selectedVendorId) {
|
|
1804
|
+
// Use vendor selected during model selection (with API key verification)
|
|
1382
1805
|
promptRun.VendorID = modelWithVendor._selectedVendorId;
|
|
1383
1806
|
}
|
|
1384
1807
|
else {
|
|
1385
|
-
|
|
1808
|
+
// Fallback: grab the highest priority AI Model Vendor record for this model (inference providers only)
|
|
1809
|
+
const modelVendors = AIEngine.Instance.ModelVendors
|
|
1386
1810
|
.filter((mv) => mv.ModelID === model.ID && mv.Status === 'Active' && this.isInferenceProvider(mv))
|
|
1387
1811
|
.sort((a, b) => b.Priority - a.Priority);
|
|
1388
1812
|
if (modelVendors.length > 0) {
|
|
@@ -1391,24 +1815,32 @@ class AIPromptRunner {
|
|
|
1391
1815
|
}
|
|
1392
1816
|
promptRun.ConfigurationID = params.configurationId;
|
|
1393
1817
|
promptRun.RunAt = startTime;
|
|
1818
|
+
// Set AgentRunID if provided for agent-prompt execution tracking
|
|
1394
1819
|
if (params.agentRunId) {
|
|
1395
1820
|
promptRun.AgentRunID = params.agentRunId;
|
|
1396
1821
|
}
|
|
1822
|
+
// Resolve and save the effort level used (same precedence as ChatParams resolution)
|
|
1397
1823
|
if (params.effortLevel !== undefined && params.effortLevel !== null) {
|
|
1398
1824
|
promptRun.EffortLevel = params.effortLevel;
|
|
1399
1825
|
}
|
|
1400
1826
|
else if (prompt.EffortLevel !== undefined && prompt.EffortLevel !== null) {
|
|
1401
1827
|
promptRun.EffortLevel = prompt.EffortLevel;
|
|
1402
1828
|
}
|
|
1829
|
+
// If neither is set, EffortLevel remains null (provider default was used)
|
|
1830
|
+
// Set ParentID for hierarchical prompt execution tracking
|
|
1403
1831
|
if (params.parentPromptRunId) {
|
|
1404
1832
|
promptRun.ParentID = params.parentPromptRunId;
|
|
1405
1833
|
}
|
|
1834
|
+
// Set RerunFromPromptRunID if this is a rerun
|
|
1406
1835
|
if (params.rerunFromPromptRunID) {
|
|
1407
1836
|
promptRun.RerunFromPromptRunID = params.rerunFromPromptRunID;
|
|
1408
1837
|
}
|
|
1838
|
+
// Always save the response format from the prompt if it exists
|
|
1409
1839
|
if (prompt.ResponseFormat && prompt.ResponseFormat !== 'Any') {
|
|
1410
1840
|
promptRun.ResponseFormat = prompt.ResponseFormat;
|
|
1411
1841
|
}
|
|
1842
|
+
// Save the actual values that will be used (either from prompt defaults or additionalParameters)
|
|
1843
|
+
// First, apply defaults from prompt entity
|
|
1412
1844
|
if (prompt.Temperature != null)
|
|
1413
1845
|
promptRun.Temperature = prompt.Temperature;
|
|
1414
1846
|
if (prompt.TopP != null)
|
|
@@ -1429,6 +1861,7 @@ class AIPromptRunner {
|
|
|
1429
1861
|
promptRun.LogProbs = prompt.IncludeLogProbs;
|
|
1430
1862
|
if (prompt.TopLogProbs != null)
|
|
1431
1863
|
promptRun.TopLogProbs = prompt.TopLogProbs;
|
|
1864
|
+
// Then override with additionalParameters if provided
|
|
1432
1865
|
if (params.additionalParameters) {
|
|
1433
1866
|
if (params.additionalParameters.temperature !== undefined) {
|
|
1434
1867
|
promptRun.Temperature = params.additionalParameters.temperature;
|
|
@@ -1461,6 +1894,7 @@ class AIPromptRunner {
|
|
|
1461
1894
|
promptRun.TopLogProbs = params.additionalParameters.topLogProbs;
|
|
1462
1895
|
}
|
|
1463
1896
|
}
|
|
1897
|
+
// Store the input data/context as JSON in Messages field
|
|
1464
1898
|
if (params.data || params.templateData || systemPromptText) {
|
|
1465
1899
|
const messages = [];
|
|
1466
1900
|
if (systemPromptText) {
|
|
@@ -1476,13 +1910,14 @@ class AIPromptRunner {
|
|
|
1476
1910
|
messages: messages || [],
|
|
1477
1911
|
});
|
|
1478
1912
|
}
|
|
1913
|
+
// Populate new retry tracking columns with initial values
|
|
1479
1914
|
promptRun.ValidationBehavior = prompt.ValidationBehavior || 'Warn';
|
|
1480
1915
|
promptRun.RetryStrategy = prompt.RetryStrategy || 'Fixed';
|
|
1481
1916
|
promptRun.MaxRetriesConfigured = prompt.MaxRetries || 0;
|
|
1482
1917
|
promptRun.FirstAttemptAt = startTime;
|
|
1483
|
-
promptRun.ValidationAttemptCount = 0;
|
|
1918
|
+
promptRun.ValidationAttemptCount = 0; // Will be updated during execution
|
|
1484
1919
|
promptRun.SuccessfulValidationCount = 0;
|
|
1485
|
-
promptRun.FinalValidationPassed = false;
|
|
1920
|
+
promptRun.FinalValidationPassed = false; // Will be updated after execution
|
|
1486
1921
|
const saveResult = await promptRun.Save();
|
|
1487
1922
|
if (!saveResult) {
|
|
1488
1923
|
const error = `Failed to save AIPromptRun: ${promptRun.LatestResult?.CompleteMessage || 'Unknown error'}`;
|
|
@@ -1497,12 +1932,14 @@ class AIPromptRunner {
|
|
|
1497
1932
|
});
|
|
1498
1933
|
throw new Error(error);
|
|
1499
1934
|
}
|
|
1935
|
+
// Invoke callback if provided
|
|
1500
1936
|
if (params.onPromptRunCreated) {
|
|
1501
1937
|
try {
|
|
1502
1938
|
await params.onPromptRunCreated(promptRun.ID);
|
|
1503
1939
|
}
|
|
1504
1940
|
catch (callbackError) {
|
|
1505
|
-
|
|
1941
|
+
LogStatus(`Error in onPromptRunCreated callback: ${callbackError.message}`);
|
|
1942
|
+
// Don't fail the execution if callback fails
|
|
1506
1943
|
}
|
|
1507
1944
|
}
|
|
1508
1945
|
return promptRun;
|
|
@@ -1520,18 +1957,26 @@ class AIPromptRunner {
|
|
|
1520
1957
|
throw new Error(msg);
|
|
1521
1958
|
}
|
|
1522
1959
|
}
|
|
1960
|
+
/**
|
|
1961
|
+
* Renders the prompt template with provided data
|
|
1962
|
+
*/
|
|
1523
1963
|
async renderPromptTemplate(template, params) {
|
|
1524
1964
|
try {
|
|
1965
|
+
// Get the highest priority content for the template
|
|
1525
1966
|
const templateContent = template.GetHighestPriorityContent();
|
|
1526
1967
|
if (!templateContent) {
|
|
1527
1968
|
throw new Error(`No content found for template ${template.Name}`);
|
|
1528
1969
|
}
|
|
1529
|
-
|
|
1970
|
+
// Resolve system placeholders with full params context
|
|
1971
|
+
const systemPlaceholders = await SystemPlaceholderManager.resolveAllPlaceholders(params);
|
|
1972
|
+
// Merge data contexts with system placeholders having lowest priority
|
|
1530
1973
|
const mergedData = {
|
|
1531
|
-
...systemPlaceholders,
|
|
1532
|
-
...params.data,
|
|
1533
|
-
...params.templateData
|
|
1974
|
+
...systemPlaceholders, // System placeholders first (lowest priority)
|
|
1975
|
+
...params.data, // User data overrides system placeholders
|
|
1976
|
+
...params.templateData // Template data has highest priority
|
|
1534
1977
|
};
|
|
1978
|
+
//LogStatus(`🔧 Rendering template '${template.Name}' with ${Object.keys(systemPlaceholders).length} system placeholders`);
|
|
1979
|
+
// Render the template
|
|
1535
1980
|
return await this._templateEngine.RenderTemplate(template, templateContent, mergedData);
|
|
1536
1981
|
}
|
|
1537
1982
|
catch (error) {
|
|
@@ -1547,20 +1992,40 @@ class AIPromptRunner {
|
|
|
1547
1992
|
throw error;
|
|
1548
1993
|
}
|
|
1549
1994
|
}
|
|
1995
|
+
/**
|
|
1996
|
+
* Executes the AI model with failover support
|
|
1997
|
+
*
|
|
1998
|
+
* @remarks
|
|
1999
|
+
* This method wraps the core executeModel functionality with intelligent failover
|
|
2000
|
+
* capabilities. It will attempt to execute with different models/vendors according
|
|
2001
|
+
* to the configured failover strategy when errors occur.
|
|
2002
|
+
*
|
|
2003
|
+
* The method calls several smaller, focused helper methods:
|
|
2004
|
+
* - buildFailoverCandidates: Creates candidate models based on type restrictions
|
|
2005
|
+
* - createCandidatesFromModels: Converts models to vendor-specific candidates
|
|
2006
|
+
* - updatePromptRunWithFailoverSuccess: Records successful failover metadata
|
|
2007
|
+
* - updatePromptRunWithFailoverFailure: Records failed failover metadata
|
|
2008
|
+
* - createFailoverErrorResult: Creates standardized error response
|
|
2009
|
+
*/
|
|
1550
2010
|
async executeModelWithFailover(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole = 'system', cancellationToken, allCandidates, promptRun, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel) {
|
|
2011
|
+
// Get failover configuration (used for errorScope filtering)
|
|
1551
2012
|
const failoverConfig = this.getFailoverConfiguration(prompt);
|
|
2013
|
+
// If no candidates provided or failover disabled, execute normally with first model
|
|
1552
2014
|
if (!allCandidates || allCandidates.length === 0 || failoverConfig.strategy === 'None') {
|
|
1553
2015
|
return this.executeModel(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole, cancellationToken, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel);
|
|
1554
2016
|
}
|
|
2017
|
+
// Track failover attempts
|
|
1555
2018
|
const failoverAttempts = [];
|
|
1556
2019
|
let lastError = null;
|
|
2020
|
+
// Iterate through all candidates in priority order with instant failover
|
|
1557
2021
|
for (let i = 0; i < allCandidates.length; i++) {
|
|
1558
2022
|
const candidate = allCandidates[i];
|
|
1559
2023
|
const attemptStartTime = Date.now();
|
|
1560
2024
|
try {
|
|
2025
|
+
// Log the attempt if not the first one
|
|
1561
2026
|
if (i > 0) {
|
|
1562
2027
|
const vendorName = candidate.vendorName || 'default';
|
|
1563
|
-
|
|
2028
|
+
LogStatusEx({
|
|
1564
2029
|
message: `🔄 Trying candidate ${i + 1}/${allCandidates.length}: ${candidate.model.Name} via ${vendorName}`,
|
|
1565
2030
|
category: 'AI',
|
|
1566
2031
|
additionalArgs: [{
|
|
@@ -1573,79 +2038,95 @@ class AIPromptRunner {
|
|
|
1573
2038
|
}]
|
|
1574
2039
|
});
|
|
1575
2040
|
}
|
|
2041
|
+
// Execute the model with this candidate
|
|
1576
2042
|
const result = await this.executeModel(candidate.model, renderedPrompt, prompt, params, candidate.vendorId || null, conversationMessages, templateMessageRole, cancellationToken, candidate.driverClass, candidate.apiName, candidate.supportsEffortLevel, candidate.effortLevel);
|
|
2043
|
+
// CRITICAL FIX: Check if result failed but is retriable (network errors, rate limits, etc.)
|
|
2044
|
+
// Provider drivers (GeminiLLM, OpenAILLM, etc.) catch errors internally and return ChatResult{success: false}
|
|
2045
|
+
// instead of throwing, so we must check result.success here.
|
|
2046
|
+
if (!result.success && result.errorInfo?.canFailover) {
|
|
2047
|
+
lastError = result.exception || new Error(result.errorMessage || 'Model execution failed');
|
|
2048
|
+
// Use shared failover error handling logic
|
|
2049
|
+
const decision = await this.processFailoverError(lastError, result.errorInfo, candidate, attemptStartTime, i, allCandidates, failoverAttempts, prompt, failoverConfig);
|
|
2050
|
+
// Update candidates list (may have been filtered)
|
|
2051
|
+
allCandidates = decision.updatedCandidates;
|
|
2052
|
+
if (decision.shouldRetry) {
|
|
2053
|
+
i--; // Retry same model/vendor
|
|
2054
|
+
continue;
|
|
2055
|
+
}
|
|
2056
|
+
if (decision.shouldContinue) {
|
|
2057
|
+
continue; // Try next candidate
|
|
2058
|
+
}
|
|
2059
|
+
// Otherwise break (fatal error or last candidate)
|
|
2060
|
+
break;
|
|
2061
|
+
}
|
|
2062
|
+
// If we reach here, the result was successful
|
|
2063
|
+
// Update promptRun with failover information if we had prior failures
|
|
1577
2064
|
if (failoverAttempts.length > 0 && promptRun) {
|
|
1578
2065
|
this.updatePromptRunWithFailoverSuccess(promptRun, failoverAttempts, candidate.model, candidate.vendorId || null);
|
|
1579
2066
|
}
|
|
1580
2067
|
return result;
|
|
1581
2068
|
}
|
|
1582
2069
|
catch (error) {
|
|
1583
|
-
const attemptDuration = Date.now() - attemptStartTime;
|
|
1584
2070
|
lastError = error;
|
|
1585
|
-
|
|
1586
|
-
const
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
};
|
|
1595
|
-
failoverAttempts.push(failoverAttempt);
|
|
1596
|
-
if (errorAnalysis.errorType === 'Authentication' || errorAnalysis.errorType === 'VendorValidationError') {
|
|
1597
|
-
allCandidates = this.filterVendorCandidates(errorAnalysis.errorType, candidate.vendorId, allCandidates);
|
|
1598
|
-
}
|
|
1599
|
-
const isLastCandidate = i === allCandidates.length - 1;
|
|
1600
|
-
if (errorAnalysis.severity === 'Fatal') {
|
|
1601
|
-
const errorMessage = error?.message || error?.errorMessage || 'Unknown error';
|
|
1602
|
-
(0, core_1.LogErrorEx)(`Stopping failover: Fatal error (${errorAnalysis.errorType}): ${errorMessage}`);
|
|
1603
|
-
this.logFailoverAttempt(prompt.ID, failoverAttempt, false);
|
|
1604
|
-
break;
|
|
2071
|
+
// Analyze error to get error info
|
|
2072
|
+
const errorInfo = ErrorAnalyzer.analyzeError(lastError);
|
|
2073
|
+
// Use shared failover error handling logic
|
|
2074
|
+
const decision = await this.processFailoverError(lastError, errorInfo, candidate, attemptStartTime, i, allCandidates, failoverAttempts, prompt, failoverConfig);
|
|
2075
|
+
// Update candidates list (may have been filtered)
|
|
2076
|
+
allCandidates = decision.updatedCandidates;
|
|
2077
|
+
if (decision.shouldRetry) {
|
|
2078
|
+
i--; // Retry same model/vendor
|
|
2079
|
+
continue;
|
|
1605
2080
|
}
|
|
1606
|
-
if (
|
|
1607
|
-
|
|
1608
|
-
if (!matchesScope) {
|
|
1609
|
-
this.logFailoverAttempt(prompt.ID, failoverAttempt, false);
|
|
1610
|
-
break;
|
|
1611
|
-
}
|
|
2081
|
+
if (decision.shouldContinue) {
|
|
2082
|
+
continue; // Try next candidate
|
|
1612
2083
|
}
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
break;
|
|
1616
|
-
}
|
|
1617
|
-
this.logFailoverAttempt(prompt.ID, failoverAttempt, true);
|
|
2084
|
+
// Otherwise break (fatal error or last candidate)
|
|
2085
|
+
break;
|
|
1618
2086
|
}
|
|
1619
2087
|
}
|
|
2088
|
+
// All candidates failed
|
|
1620
2089
|
if (promptRun && failoverAttempts.length > 0) {
|
|
1621
2090
|
this.updatePromptRunWithFailoverFailure(promptRun, failoverAttempts);
|
|
1622
2091
|
}
|
|
1623
2092
|
return this.createFailoverErrorResult(lastError, failoverAttempts);
|
|
1624
2093
|
}
|
|
2094
|
+
/**
|
|
2095
|
+
* Builds failover candidates for a prompt based on available models and type restrictions
|
|
2096
|
+
*/
|
|
1625
2097
|
async buildFailoverCandidates(prompt) {
|
|
1626
|
-
const aiEngine =
|
|
2098
|
+
const aiEngine = AIEngine.Instance;
|
|
2099
|
+
// Get all models, filtered by type if specified
|
|
1627
2100
|
let allModels;
|
|
1628
2101
|
if (prompt.AIModelTypeID) {
|
|
2102
|
+
// Find the model type from the prompt
|
|
1629
2103
|
const modelType = aiEngine.ModelTypes.find(mt => mt.ID === prompt.AIModelTypeID);
|
|
1630
2104
|
if (!modelType) {
|
|
1631
2105
|
throw new Error(`Model type ${prompt.AIModelTypeID} not found`);
|
|
1632
2106
|
}
|
|
2107
|
+
// Get all models of this specific type
|
|
1633
2108
|
const targetTypeName = modelType.Name.trim().toLowerCase();
|
|
1634
2109
|
allModels = aiEngine.Models.filter(m => {
|
|
2110
|
+
// Guard against AIModelType being non-string (defensive coding for data issues)
|
|
1635
2111
|
const mType = typeof m.AIModelType === 'string' ? m.AIModelType.trim().toLowerCase() : '';
|
|
1636
2112
|
return mType === targetTypeName;
|
|
1637
2113
|
});
|
|
1638
2114
|
}
|
|
1639
2115
|
else {
|
|
2116
|
+
// No type restriction - get all models
|
|
1640
2117
|
allModels = aiEngine.Models;
|
|
1641
2118
|
}
|
|
1642
2119
|
return this.createCandidatesFromModels(allModels);
|
|
1643
2120
|
}
|
|
2121
|
+
/**
|
|
2122
|
+
* Creates model-vendor candidates from a list of models
|
|
2123
|
+
*/
|
|
1644
2124
|
createCandidatesFromModels(models) {
|
|
1645
2125
|
const candidates = [];
|
|
1646
2126
|
for (const model of models) {
|
|
1647
2127
|
const vendors = model.ModelVendors || [];
|
|
1648
2128
|
if (vendors.length === 0) {
|
|
2129
|
+
// Model without specific vendors
|
|
1649
2130
|
candidates.push({
|
|
1650
2131
|
model: model,
|
|
1651
2132
|
vendorId: undefined,
|
|
@@ -1659,6 +2140,7 @@ class AIPromptRunner {
|
|
|
1659
2140
|
});
|
|
1660
2141
|
}
|
|
1661
2142
|
else {
|
|
2143
|
+
// Add each vendor as a separate candidate
|
|
1662
2144
|
for (const vendor of vendors) {
|
|
1663
2145
|
candidates.push({
|
|
1664
2146
|
model: model,
|
|
@@ -1676,6 +2158,9 @@ class AIPromptRunner {
|
|
|
1676
2158
|
}
|
|
1677
2159
|
return candidates;
|
|
1678
2160
|
}
|
|
2161
|
+
/**
|
|
2162
|
+
* Updates prompt run with successful failover tracking data
|
|
2163
|
+
*/
|
|
1679
2164
|
updatePromptRunWithFailoverSuccess(promptRun, failoverAttempts, currentModel, currentVendorId) {
|
|
1680
2165
|
promptRun.FailoverAttempts = failoverAttempts.length;
|
|
1681
2166
|
promptRun.FailoverErrors = JSON.stringify(failoverAttempts.map(a => ({
|
|
@@ -1686,6 +2171,7 @@ class AIPromptRunner {
|
|
|
1686
2171
|
})));
|
|
1687
2172
|
promptRun.FailoverDurations = JSON.stringify(failoverAttempts.map(a => a.duration));
|
|
1688
2173
|
promptRun.TotalFailoverDuration = failoverAttempts.reduce((sum, a) => sum + a.duration, 0);
|
|
2174
|
+
// Update ModelID if we ended up using a different model
|
|
1689
2175
|
if (currentModel.ID !== promptRun.OriginalModelID) {
|
|
1690
2176
|
promptRun.ModelID = currentModel.ID;
|
|
1691
2177
|
}
|
|
@@ -1693,6 +2179,9 @@ class AIPromptRunner {
|
|
|
1693
2179
|
promptRun.VendorID = currentVendorId;
|
|
1694
2180
|
}
|
|
1695
2181
|
}
|
|
2182
|
+
/**
|
|
2183
|
+
* Updates prompt run with failover failure tracking data
|
|
2184
|
+
*/
|
|
1696
2185
|
updatePromptRunWithFailoverFailure(promptRun, failoverAttempts) {
|
|
1697
2186
|
promptRun.FailoverAttempts = failoverAttempts.length;
|
|
1698
2187
|
promptRun.FailoverErrors = JSON.stringify(failoverAttempts.map(a => ({
|
|
@@ -1704,14 +2193,20 @@ class AIPromptRunner {
|
|
|
1704
2193
|
promptRun.FailoverDurations = JSON.stringify(failoverAttempts.map(a => a.duration));
|
|
1705
2194
|
promptRun.TotalFailoverDuration = failoverAttempts.reduce((sum, a) => sum + a.duration, 0);
|
|
1706
2195
|
}
|
|
2196
|
+
/**
|
|
2197
|
+
* Creates an error result for failed failover attempts
|
|
2198
|
+
*/
|
|
1707
2199
|
createFailoverErrorResult(lastError, failoverAttempts) {
|
|
1708
2200
|
const startTime = new Date();
|
|
1709
2201
|
const endTime = new Date();
|
|
2202
|
+
// Check if this is a ContextLengthExceeded error - if so, mark as Fatal
|
|
1710
2203
|
const hasContextLengthError = failoverAttempts.some(a => a.errorType === 'ContextLengthExceeded' ||
|
|
1711
|
-
|
|
2204
|
+
ErrorAnalyzer.analyzeError(a.error).errorType === 'ContextLengthExceeded');
|
|
2205
|
+
// If ContextLengthExceeded and all failover attempts failed, this is fatal
|
|
1712
2206
|
let errorInfo;
|
|
1713
2207
|
if (lastError) {
|
|
1714
|
-
errorInfo =
|
|
2208
|
+
errorInfo = ErrorAnalyzer.analyzeError(lastError);
|
|
2209
|
+
// Override severity to Fatal if context length exceeded and no larger models exist
|
|
1715
2210
|
if (hasContextLengthError && errorInfo.errorType === 'ContextLengthExceeded') {
|
|
1716
2211
|
errorInfo.severity = 'Fatal';
|
|
1717
2212
|
}
|
|
@@ -1728,43 +2223,63 @@ class AIPromptRunner {
|
|
|
1728
2223
|
data: null
|
|
1729
2224
|
};
|
|
1730
2225
|
}
|
|
2226
|
+
/**
|
|
2227
|
+
* Executes the AI model with the rendered prompt
|
|
2228
|
+
*/
|
|
1731
2229
|
async executeModel(model, renderedPrompt, prompt, params, vendorId, conversationMessages, templateMessageRole = 'system', cancellationToken, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel) {
|
|
2230
|
+
// define these variables here to ensure they're available in the catch block
|
|
1732
2231
|
let driverClass;
|
|
1733
2232
|
let apiName;
|
|
1734
2233
|
let llm;
|
|
1735
2234
|
let chatParams;
|
|
1736
2235
|
try {
|
|
1737
|
-
|
|
2236
|
+
// Get verbose flag for logging
|
|
2237
|
+
const verbose = params.verbose === true || IsVerboseLoggingEnabled();
|
|
2238
|
+
// Determine if effort level is supported
|
|
1738
2239
|
let supportsEffortLevel = false;
|
|
2240
|
+
// Get vendor-specific configuration
|
|
2241
|
+
// Use passed vendor info if available, otherwise fall back to vendor lookup
|
|
1739
2242
|
if (vendorDriverClass && vendorApiName) {
|
|
2243
|
+
// Vendor info was provided by the caller (from model selection)
|
|
1740
2244
|
driverClass = vendorDriverClass;
|
|
1741
2245
|
apiName = vendorApiName;
|
|
2246
|
+
// Use provided vendorSupportsEffortLevel, or default to false
|
|
1742
2247
|
supportsEffortLevel = vendorSupportsEffortLevel ?? false;
|
|
1743
2248
|
}
|
|
1744
2249
|
else {
|
|
2250
|
+
// Fallback to model defaults or vendor lookup
|
|
1745
2251
|
driverClass = model.DriverClass;
|
|
1746
2252
|
apiName = model.APIName;
|
|
2253
|
+
// Start with model's SupportsEffortLevel setting
|
|
1747
2254
|
supportsEffortLevel = model.SupportsEffortLevel ?? false;
|
|
1748
2255
|
if (vendorId) {
|
|
1749
|
-
|
|
2256
|
+
// Find the AIModelVendor record for this specific vendor - must be an inference provider
|
|
2257
|
+
const modelVendor = AIEngine.Instance.ModelVendors.find((mv) => mv.ModelID === model.ID && mv.VendorID === vendorId && mv.Status === 'Active' && this.isInferenceProvider(mv));
|
|
1750
2258
|
if (modelVendor) {
|
|
1751
2259
|
driverClass = modelVendor.DriverClass || driverClass;
|
|
1752
2260
|
apiName = modelVendor.APIName || apiName;
|
|
2261
|
+
// Use modelVendor's SupportsEffortLevel if available
|
|
1753
2262
|
supportsEffortLevel = modelVendor.SupportsEffortLevel ?? supportsEffortLevel;
|
|
1754
2263
|
}
|
|
1755
2264
|
else {
|
|
2265
|
+
// Log warning if vendor was specified but not found or not an inference provider
|
|
1756
2266
|
this.logStatus(`⚠️ Vendor ${vendorId} not found or is not an inference provider for model ${model.Name}, using model defaults`, true, params);
|
|
1757
2267
|
}
|
|
1758
2268
|
}
|
|
1759
2269
|
}
|
|
2270
|
+
// Resolve credentials using hierarchical resolution (Credentials system with legacy fallback)
|
|
1760
2271
|
const apiKey = await this.resolveCredentialForExecution(driverClass, prompt.ID, model.ID, vendorId ?? undefined, params);
|
|
1761
|
-
|
|
1762
|
-
|
|
2272
|
+
// Create LLM instance with vendor-specific driver class
|
|
2273
|
+
llm = MJGlobal.Instance.ClassFactory.CreateInstance(BaseLLM, driverClass, apiKey);
|
|
2274
|
+
// Prepare chat parameters
|
|
2275
|
+
chatParams = new ChatParams();
|
|
1763
2276
|
if (!apiName) {
|
|
1764
2277
|
throw new Error(`No API name found for model ${model.Name}. Please ensure the model or its vendor configuration includes an APIName.`);
|
|
1765
2278
|
}
|
|
1766
2279
|
chatParams.model = apiName;
|
|
1767
2280
|
chatParams.cancellationToken = cancellationToken;
|
|
2281
|
+
// Apply defaults from prompt entity first (if they exist)
|
|
2282
|
+
// These can be overridden by additionalParameters
|
|
1768
2283
|
if (prompt.Temperature != null)
|
|
1769
2284
|
chatParams.temperature = prompt.Temperature;
|
|
1770
2285
|
if (prompt.TopP != null)
|
|
@@ -1780,13 +2295,16 @@ class AIPromptRunner {
|
|
|
1780
2295
|
if (prompt.Seed != null)
|
|
1781
2296
|
chatParams.seed = prompt.Seed;
|
|
1782
2297
|
if (prompt.StopSequences) {
|
|
2298
|
+
// Parse comma-delimited stop sequences
|
|
1783
2299
|
chatParams.stopSequences = prompt.StopSequences.split(',').map((s) => s.trim()).filter((s) => s.length > 0);
|
|
1784
2300
|
}
|
|
1785
2301
|
if (prompt.IncludeLogProbs != null)
|
|
1786
2302
|
chatParams.includeLogProbs = prompt.IncludeLogProbs;
|
|
1787
2303
|
if (prompt.TopLogProbs != null)
|
|
1788
2304
|
chatParams.topLogProbs = prompt.TopLogProbs;
|
|
2305
|
+
// Apply additional parameters if provided (these override prompt defaults)
|
|
1789
2306
|
if (params.additionalParameters) {
|
|
2307
|
+
// Apply chat-specific parameters from additionalParameters
|
|
1790
2308
|
if (params.additionalParameters.temperature !== undefined) {
|
|
1791
2309
|
chatParams.temperature = params.additionalParameters.temperature;
|
|
1792
2310
|
}
|
|
@@ -1818,11 +2336,18 @@ class AIPromptRunner {
|
|
|
1818
2336
|
chatParams.topLogProbs = params.additionalParameters.topLogProbs;
|
|
1819
2337
|
}
|
|
1820
2338
|
}
|
|
2339
|
+
// Apply effortLevel with precedence hierarchy
|
|
2340
|
+
// 1. params.effortLevel (runtime override - highest priority)
|
|
2341
|
+
// 2. modelEffortLevel (model-specific override from AIPromptModel - second priority)
|
|
2342
|
+
// 3. Agent DefaultPromptEffortLevel (passed via params.effortLevel by BaseAgent - third priority)
|
|
2343
|
+
// 4. prompt.EffortLevel (prompt default - fourth priority)
|
|
2344
|
+
// 5. No effort level (provider default - lowest priority)
|
|
1821
2345
|
const hasEffortLevel = (params.effortLevel !== undefined && params.effortLevel !== null) ||
|
|
1822
2346
|
(modelEffortLevel !== undefined && modelEffortLevel !== null) ||
|
|
1823
2347
|
(prompt.EffortLevel !== undefined && prompt.EffortLevel !== null);
|
|
1824
2348
|
if (hasEffortLevel) {
|
|
1825
2349
|
if (supportsEffortLevel) {
|
|
2350
|
+
// Vendor/model supports effort level, apply it with precedence
|
|
1826
2351
|
if (params.effortLevel !== undefined && params.effortLevel !== null) {
|
|
1827
2352
|
chatParams.effortLevel = params.effortLevel.toString();
|
|
1828
2353
|
}
|
|
@@ -1834,18 +2359,25 @@ class AIPromptRunner {
|
|
|
1834
2359
|
}
|
|
1835
2360
|
}
|
|
1836
2361
|
else {
|
|
2362
|
+
// Vendor/model does not support effort level, log warning
|
|
1837
2363
|
const effortValue = params.effortLevel ?? modelEffortLevel ?? prompt.EffortLevel;
|
|
1838
2364
|
console.log(`⚠️ Effort Level ${effortValue} specified but will be ignored - model ${model.Name} does not support effort levels`);
|
|
1839
2365
|
}
|
|
1840
2366
|
}
|
|
2367
|
+
// If none are set, effortLevel remains undefined and providers use their defaults
|
|
2368
|
+
// Apply response format from prompt settings
|
|
1841
2369
|
if (prompt.ResponseFormat && prompt.ResponseFormat !== 'Any') {
|
|
1842
|
-
chatParams.responseFormat = prompt.ResponseFormat;
|
|
2370
|
+
chatParams.responseFormat = prompt.ResponseFormat; //as 'Any' | 'Text' | 'Markdown' | 'JSON' | 'ModelSpecific';
|
|
1843
2371
|
}
|
|
1844
2372
|
else {
|
|
2373
|
+
// if chatParams.responseFormat is not set or set to Any, stay silent on response format
|
|
1845
2374
|
chatParams.responseFormat = undefined;
|
|
1846
2375
|
}
|
|
2376
|
+
// Build message array with rendered prompt and conversation messages
|
|
1847
2377
|
chatParams.messages = this.buildMessageArray(renderedPrompt, conversationMessages, templateMessageRole);
|
|
2378
|
+
// Execute the model with cancellation support
|
|
1848
2379
|
if (cancellationToken) {
|
|
2380
|
+
// If cancellation token is provided, wrap the execution to handle cancellation
|
|
1849
2381
|
return await Promise.race([
|
|
1850
2382
|
llm.ChatCompletion(chatParams),
|
|
1851
2383
|
new Promise((_, reject) => {
|
|
@@ -1861,11 +2393,12 @@ class AIPromptRunner {
|
|
|
1861
2393
|
]);
|
|
1862
2394
|
}
|
|
1863
2395
|
else {
|
|
2396
|
+
// No cancellation token, execute normally
|
|
1864
2397
|
return await llm.ChatCompletion(chatParams);
|
|
1865
2398
|
}
|
|
1866
2399
|
}
|
|
1867
2400
|
catch (error) {
|
|
1868
|
-
const errorInfo =
|
|
2401
|
+
const errorInfo = ErrorAnalyzer.analyzeError(error, driverClass);
|
|
1869
2402
|
this.logError(error, {
|
|
1870
2403
|
category: 'ModelExecution',
|
|
1871
2404
|
model: model,
|
|
@@ -1878,51 +2411,70 @@ class AIPromptRunner {
|
|
|
1878
2411
|
throw error;
|
|
1879
2412
|
}
|
|
1880
2413
|
}
|
|
2414
|
+
/**
|
|
2415
|
+
* Builds the message array combining rendered prompt with conversation messages
|
|
2416
|
+
*/
|
|
1881
2417
|
buildMessageArray(renderedPrompt, conversationMessages, templateMessageRole = 'system') {
|
|
1882
2418
|
const messages = [];
|
|
2419
|
+
// Add rendered template as system or user message if not 'none'
|
|
1883
2420
|
if (renderedPrompt && templateMessageRole !== 'none') {
|
|
1884
2421
|
messages.push({
|
|
1885
|
-
role: templateMessageRole === 'system' ?
|
|
2422
|
+
role: templateMessageRole === 'system' ? ChatMessageRole.system : ChatMessageRole.user,
|
|
1886
2423
|
content: renderedPrompt,
|
|
1887
2424
|
});
|
|
1888
2425
|
}
|
|
2426
|
+
// Add conversation messages if provided
|
|
1889
2427
|
if (conversationMessages && conversationMessages.length > 0) {
|
|
1890
2428
|
messages.push(...conversationMessages);
|
|
1891
2429
|
}
|
|
2430
|
+
// If no conversation messages and no rendered prompt as user message,
|
|
2431
|
+
// add a default user message to ensure we have at least one user message
|
|
1892
2432
|
if ((!conversationMessages || conversationMessages.length === 0) && templateMessageRole !== 'user' && renderedPrompt) {
|
|
2433
|
+
// If we only have a system message, we need a user message too
|
|
1893
2434
|
if (templateMessageRole === 'system') {
|
|
1894
2435
|
messages.push({
|
|
1895
|
-
role:
|
|
2436
|
+
role: ChatMessageRole.user,
|
|
1896
2437
|
content: 'Please proceed with the above instructions.',
|
|
1897
2438
|
});
|
|
1898
2439
|
}
|
|
1899
2440
|
}
|
|
1900
2441
|
else if ((!conversationMessages || conversationMessages.length === 0) && !renderedPrompt) {
|
|
2442
|
+
// Fallback: if no conversation and no rendered prompt, add a basic user message
|
|
1901
2443
|
messages.push({
|
|
1902
|
-
role:
|
|
2444
|
+
role: ChatMessageRole.user,
|
|
1903
2445
|
content: 'Hello',
|
|
1904
2446
|
});
|
|
1905
2447
|
}
|
|
1906
2448
|
return messages;
|
|
1907
2449
|
}
|
|
2450
|
+
/**
|
|
2451
|
+
* Executes the model with retry logic for validation failures
|
|
2452
|
+
*/
|
|
1908
2453
|
async executeWithValidationRetries(selectedModel, renderedPromptText, prompt, params, promptRun, allCandidates, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel) {
|
|
1909
2454
|
const validationAttempts = [];
|
|
1910
2455
|
const maxRetries = Math.max(0, prompt.MaxRetries || 0);
|
|
1911
2456
|
let lastError = null;
|
|
2457
|
+
// Track cumulative token usage across all attempts
|
|
1912
2458
|
let cumulativePromptTokens = 0;
|
|
1913
2459
|
let cumulativeCompletionTokens = 0;
|
|
1914
2460
|
let cumulativeCost = 0;
|
|
1915
2461
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
1916
2462
|
try {
|
|
2463
|
+
// Check for cancellation before each attempt
|
|
1917
2464
|
if (params.cancellationToken?.aborted) {
|
|
1918
2465
|
throw new Error('Execution was cancelled during validation retries');
|
|
1919
2466
|
}
|
|
1920
2467
|
if (attempt > 0) {
|
|
1921
|
-
|
|
2468
|
+
LogStatus(` 🔄 Retrying execution due to validation failure, attempt ${attempt + 1}/${maxRetries + 1}`);
|
|
1922
2469
|
await this.applyRetryDelay(prompt, attempt);
|
|
1923
2470
|
}
|
|
1924
|
-
|
|
2471
|
+
// Execute the AI model with failover support
|
|
2472
|
+
const modelResult = await this.executeModelWithFailover(selectedModel, renderedPromptText, prompt, params, promptRun.VendorID, params.conversationMessages, params.templateMessageRole || 'system', params.cancellationToken, allCandidates, // Pass the candidates from initial selection
|
|
2473
|
+
promptRun, vendorDriverClass, vendorApiName, vendorSupportsEffortLevel, modelEffortLevel);
|
|
2474
|
+
// Check for fatal errors - don't attempt validation/retry on these
|
|
2475
|
+
// Fatal errors (like ContextLengthExceeded when all models exhausted) cannot be resolved by retrying
|
|
1925
2476
|
if (!modelResult.success && modelResult.errorInfo?.severity === 'Fatal') {
|
|
2477
|
+
// Record the fatal error attempt
|
|
1926
2478
|
const validationAttempt = {
|
|
1927
2479
|
attemptNumber: attempt + 1,
|
|
1928
2480
|
success: false,
|
|
@@ -1931,6 +2483,7 @@ class AIPromptRunner {
|
|
|
1931
2483
|
timestamp: new Date(),
|
|
1932
2484
|
};
|
|
1933
2485
|
validationAttempts.push(validationAttempt);
|
|
2486
|
+
// Return immediately - no point in validation or retries for fatal errors
|
|
1934
2487
|
return {
|
|
1935
2488
|
modelResult,
|
|
1936
2489
|
parsedResult: {
|
|
@@ -1945,12 +2498,15 @@ class AIPromptRunner {
|
|
|
1945
2498
|
},
|
|
1946
2499
|
};
|
|
1947
2500
|
}
|
|
2501
|
+
// Accumulate token usage from this attempt
|
|
1948
2502
|
if (modelResult.data?.usage) {
|
|
1949
2503
|
cumulativePromptTokens += modelResult.data.usage.promptTokens || 0;
|
|
1950
2504
|
cumulativeCompletionTokens += modelResult.data.usage.completionTokens || 0;
|
|
1951
2505
|
cumulativeCost += modelResult.data.usage.cost || 0;
|
|
1952
2506
|
}
|
|
2507
|
+
// Parse and validate the result
|
|
1953
2508
|
const { result, validationResult, validationErrors } = await this.parseAndValidateResultEnhanced(modelResult, prompt, params.skipValidation, params.cleanValidationSyntax, promptRun, params);
|
|
2509
|
+
// Record this validation attempt
|
|
1954
2510
|
const validationAttempt = {
|
|
1955
2511
|
attemptNumber: attempt + 1,
|
|
1956
2512
|
success: validationResult?.Success || false,
|
|
@@ -1962,6 +2518,7 @@ class AIPromptRunner {
|
|
|
1962
2518
|
};
|
|
1963
2519
|
validationAttempts.push(validationAttempt);
|
|
1964
2520
|
if (validationResult?.Success !== false) {
|
|
2521
|
+
// Validation succeeded, return the result
|
|
1965
2522
|
return {
|
|
1966
2523
|
modelResult,
|
|
1967
2524
|
parsedResult: { result, validationResult },
|
|
@@ -1973,16 +2530,19 @@ class AIPromptRunner {
|
|
|
1973
2530
|
},
|
|
1974
2531
|
};
|
|
1975
2532
|
}
|
|
2533
|
+
// Validation failed, check if we should retry
|
|
2534
|
+
// BUG FIX: Only retry in Strict mode, not in Warn or None modes
|
|
1976
2535
|
if (prompt.ValidationBehavior === 'Strict' && attempt < maxRetries) {
|
|
1977
2536
|
lastError = new Error(`Validation failed: ${validationErrors?.map(e => e.Message).join('; ')}`);
|
|
1978
|
-
|
|
1979
|
-
continue;
|
|
2537
|
+
LogStatus(` ⚠️ Validation failed on attempt ${attempt + 1}, will retry (Strict mode)`);
|
|
2538
|
+
continue; // Retry
|
|
1980
2539
|
}
|
|
1981
2540
|
else {
|
|
2541
|
+
// Either not strict mode or no more retries, return what we have
|
|
1982
2542
|
const reason = prompt.ValidationBehavior !== 'Strict'
|
|
1983
2543
|
? `${prompt.ValidationBehavior || 'None'} mode - continuing with invalid output (no retry)`
|
|
1984
2544
|
: 'max retries exceeded';
|
|
1985
|
-
|
|
2545
|
+
LogStatus(` ⚠️ Validation failed on attempt ${attempt + 1}, stopping retries (${reason})`);
|
|
1986
2546
|
return {
|
|
1987
2547
|
modelResult,
|
|
1988
2548
|
parsedResult: { result, validationResult },
|
|
@@ -2007,6 +2567,7 @@ class AIPromptRunner {
|
|
|
2007
2567
|
},
|
|
2008
2568
|
maxErrorLength: params.maxErrorLength
|
|
2009
2569
|
});
|
|
2570
|
+
// Record failed attempt
|
|
2010
2571
|
const validationAttempt = {
|
|
2011
2572
|
attemptNumber: attempt + 1,
|
|
2012
2573
|
success: false,
|
|
@@ -2016,17 +2577,26 @@ class AIPromptRunner {
|
|
|
2016
2577
|
};
|
|
2017
2578
|
validationAttempts.push(validationAttempt);
|
|
2018
2579
|
if (attempt === maxRetries) {
|
|
2019
|
-
throw error;
|
|
2580
|
+
throw error; // Last attempt, propagate error
|
|
2020
2581
|
}
|
|
2021
2582
|
}
|
|
2022
2583
|
}
|
|
2584
|
+
// Should not reach here, but just in case
|
|
2023
2585
|
throw lastError || new Error('Execution failed after all retry attempts');
|
|
2024
2586
|
}
|
|
2587
|
+
/**
|
|
2588
|
+
* Applies retry delay based on the prompt's retry strategy
|
|
2589
|
+
*/
|
|
2590
|
+
/**
|
|
2591
|
+
* Calculates retry delay for rate limit and other retriable errors.
|
|
2592
|
+
* Uses the prompt's RetryStrategy and can respect suggested delays from provider.
|
|
2593
|
+
*/
|
|
2025
2594
|
calculateRetryDelay(prompt, attemptNumber, suggestedDelaySeconds) {
|
|
2595
|
+
// Use provider's suggested delay if available
|
|
2026
2596
|
if (suggestedDelaySeconds && suggestedDelaySeconds > 0) {
|
|
2027
|
-
return suggestedDelaySeconds * 1000;
|
|
2597
|
+
return suggestedDelaySeconds * 1000; // Convert to milliseconds
|
|
2028
2598
|
}
|
|
2029
|
-
const baseDelay = prompt.RetryDelayMS || 1000;
|
|
2599
|
+
const baseDelay = prompt.RetryDelayMS || 1000; // Default 1 second
|
|
2030
2600
|
let delay = baseDelay;
|
|
2031
2601
|
switch (prompt.RetryStrategy) {
|
|
2032
2602
|
case 'Fixed':
|
|
@@ -2046,20 +2616,28 @@ class AIPromptRunner {
|
|
|
2046
2616
|
async applyRetryDelay(prompt, attemptNumber, suggestedDelaySeconds) {
|
|
2047
2617
|
const delay = this.calculateRetryDelay(prompt, attemptNumber, suggestedDelaySeconds);
|
|
2048
2618
|
const delaySeconds = (delay / 1000).toFixed(1);
|
|
2049
|
-
|
|
2619
|
+
LogStatus(` Waiting ${delaySeconds}s before retry (strategy: ${prompt.RetryStrategy || 'Fixed'})...`);
|
|
2050
2620
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
2051
2621
|
}
|
|
2622
|
+
/**
|
|
2623
|
+
* Filters out all candidates from a vendor when a vendor-level error occurs.
|
|
2624
|
+
* Vendor-level errors affect all models from that vendor:
|
|
2625
|
+
* - Authentication: Invalid API key
|
|
2626
|
+
* - VendorValidationError: API schema/validation requirements
|
|
2627
|
+
*/
|
|
2052
2628
|
filterVendorCandidates(errorType, currentVendorId, allCandidates) {
|
|
2053
2629
|
if (errorType !== 'Authentication' && errorType !== 'VendorValidationError') {
|
|
2054
|
-
return allCandidates;
|
|
2630
|
+
return allCandidates; // No filtering needed for non-vendor-level errors
|
|
2055
2631
|
}
|
|
2056
2632
|
const failedVendorId = currentVendorId || 'default';
|
|
2057
2633
|
const beforeCount = allCandidates.length;
|
|
2634
|
+
// Filter out ALL candidates from this vendor
|
|
2058
2635
|
const filteredCandidates = allCandidates.filter(c => (c.vendorId || 'default') !== failedVendorId);
|
|
2059
2636
|
const removedCount = beforeCount - filteredCandidates.length;
|
|
2060
2637
|
if (removedCount > 0) {
|
|
2061
|
-
const vendorName =
|
|
2638
|
+
const vendorName = AIEngine.Instance.Vendors.find(v => v.ID === failedVendorId)?.Name || failedVendorId;
|
|
2062
2639
|
const remainingCount = filteredCandidates.length;
|
|
2640
|
+
// Log appropriate message based on error type
|
|
2063
2641
|
let reason;
|
|
2064
2642
|
let icon;
|
|
2065
2643
|
if (errorType === 'Authentication') {
|
|
@@ -2078,38 +2656,110 @@ class AIPromptRunner {
|
|
|
2078
2656
|
}
|
|
2079
2657
|
return filteredCandidates;
|
|
2080
2658
|
}
|
|
2659
|
+
/**
|
|
2660
|
+
* Handles rate limit errors by retrying the same model/vendor with backoff.
|
|
2661
|
+
* Returns true if the caller should continue (retry), false if should proceed to failover.
|
|
2662
|
+
*/
|
|
2081
2663
|
async handleRateLimitRetry(errorAnalysis, currentModel, currentVendorId, failoverAttempts, prompt, attemptNumber, maxAttempts, failoverAttempt) {
|
|
2082
2664
|
const isRateLimit = errorAnalysis.errorType === 'RateLimit';
|
|
2083
2665
|
if (!isRateLimit) {
|
|
2084
|
-
return false;
|
|
2666
|
+
return false; // Not a rate limit error
|
|
2085
2667
|
}
|
|
2668
|
+
// Count how many times we've retried this specific model/vendor for rate limits
|
|
2086
2669
|
const rateLimitRetryCount = failoverAttempts.filter(a => a.modelId === currentModel.ID &&
|
|
2087
2670
|
a.vendorId === currentVendorId &&
|
|
2088
2671
|
a.errorType === 'RateLimit').length;
|
|
2672
|
+
// Use MaxRetries from prompt configuration, default to 3 if not set
|
|
2089
2673
|
const maxRetries = prompt.MaxRetries ?? 3;
|
|
2674
|
+
// Retry up to MaxRetries times before giving up and failing over
|
|
2090
2675
|
const shouldRetry = rateLimitRetryCount <= maxRetries;
|
|
2091
2676
|
if (shouldRetry) {
|
|
2092
2677
|
const modelName = currentModel.Name;
|
|
2093
2678
|
const vendorName = currentVendorId
|
|
2094
|
-
?
|
|
2679
|
+
? AIEngine.Instance.Vendors.find(v => v.ID === currentVendorId)?.Name || 'default'
|
|
2095
2680
|
: 'default';
|
|
2096
2681
|
this.logStatus(` ⏳ Rate limit hit - retrying ${modelName} (${vendorName}) with backoff (attempt ${rateLimitRetryCount}/${maxRetries})`, true);
|
|
2097
2682
|
this.logFailoverAttempt(prompt.ID, failoverAttempt, true);
|
|
2683
|
+
// Apply backoff delay before retry
|
|
2098
2684
|
if (attemptNumber < maxAttempts) {
|
|
2099
2685
|
await this.applyRetryDelay(prompt, rateLimitRetryCount, errorAnalysis.suggestedRetryDelaySeconds);
|
|
2100
2686
|
}
|
|
2101
|
-
return true;
|
|
2102
|
-
}
|
|
2103
|
-
return false;
|
|
2104
|
-
}
|
|
2687
|
+
return true; // Signal to continue with same model/vendor
|
|
2688
|
+
}
|
|
2689
|
+
return false; // Too many retries, proceed to failover
|
|
2690
|
+
}
|
|
2691
|
+
/**
|
|
2692
|
+
* Processes a failover error (either from catch block or from failed ChatResult).
|
|
2693
|
+
* Handles vendor filtering, rate limit retries, fatal error detection, and failover logic.
|
|
2694
|
+
*
|
|
2695
|
+
* @returns Decision object indicating whether to retry same model, continue to next candidate, or stop
|
|
2696
|
+
*/
|
|
2697
|
+
async processFailoverError(error, errorInfo, candidate, attemptStartTime, attemptIndex, allCandidates, failoverAttempts, prompt, failoverConfig) {
|
|
2698
|
+
const attemptDuration = Date.now() - attemptStartTime;
|
|
2699
|
+
// Create failover attempt record
|
|
2700
|
+
const failoverAttempt = {
|
|
2701
|
+
attemptNumber: attemptIndex + 1,
|
|
2702
|
+
modelId: candidate.model.ID,
|
|
2703
|
+
vendorId: candidate.vendorId,
|
|
2704
|
+
error: error,
|
|
2705
|
+
errorType: errorInfo.errorType,
|
|
2706
|
+
duration: attemptDuration,
|
|
2707
|
+
timestamp: new Date()
|
|
2708
|
+
};
|
|
2709
|
+
failoverAttempts.push(failoverAttempt);
|
|
2710
|
+
// Vendor-level errors: filter out all candidates from this vendor
|
|
2711
|
+
let updatedCandidates = allCandidates;
|
|
2712
|
+
if (errorInfo.errorType === 'Authentication' || errorInfo.errorType === 'VendorValidationError') {
|
|
2713
|
+
updatedCandidates = this.filterVendorCandidates(errorInfo.errorType, candidate.vendorId, allCandidates);
|
|
2714
|
+
}
|
|
2715
|
+
const isLastCandidate = attemptIndex === updatedCandidates.length - 1;
|
|
2716
|
+
// Fatal errors: stop immediately
|
|
2717
|
+
if (errorInfo.severity === 'Fatal') {
|
|
2718
|
+
const errorMessage = error?.message || 'Unknown error';
|
|
2719
|
+
LogErrorEx(`Stopping failover: Fatal error (${errorInfo.errorType}): ${errorMessage}`);
|
|
2720
|
+
this.logFailoverAttempt(prompt.ID, failoverAttempt, false);
|
|
2721
|
+
return { shouldRetry: false, shouldContinue: false, updatedCandidates };
|
|
2722
|
+
}
|
|
2723
|
+
// Check errorScope filter if configured
|
|
2724
|
+
if (failoverConfig.errorScope && failoverConfig.errorScope !== 'All') {
|
|
2725
|
+
const matchesScope = this.errorMatchesScope(errorInfo.errorType, failoverConfig.errorScope);
|
|
2726
|
+
if (!matchesScope) {
|
|
2727
|
+
this.logFailoverAttempt(prompt.ID, failoverAttempt, false);
|
|
2728
|
+
return { shouldRetry: false, shouldContinue: false, updatedCandidates };
|
|
2729
|
+
}
|
|
2730
|
+
}
|
|
2731
|
+
// Rate limit errors: check if we should retry the same model before failing over
|
|
2732
|
+
if (errorInfo.errorType === 'RateLimit') {
|
|
2733
|
+
const shouldRetry = await this.handleRateLimitRetry(errorInfo, candidate.model, candidate.vendorId, failoverAttempts, prompt, attemptIndex, updatedCandidates.length, failoverAttempt);
|
|
2734
|
+
if (shouldRetry) {
|
|
2735
|
+
return { shouldRetry: true, shouldContinue: false, updatedCandidates };
|
|
2736
|
+
}
|
|
2737
|
+
}
|
|
2738
|
+
// If this is the last candidate, we're done
|
|
2739
|
+
if (isLastCandidate) {
|
|
2740
|
+
this.logFailoverAttempt(prompt.ID, failoverAttempt, false);
|
|
2741
|
+
return { shouldRetry: false, shouldContinue: false, updatedCandidates };
|
|
2742
|
+
}
|
|
2743
|
+
// Log and signal to continue to next candidate
|
|
2744
|
+
this.logFailoverAttempt(prompt.ID, failoverAttempt, true);
|
|
2745
|
+
return { shouldRetry: false, shouldContinue: true, updatedCandidates };
|
|
2746
|
+
}
|
|
2747
|
+
/**
|
|
2748
|
+
* Transitions to the next failover candidate.
|
|
2749
|
+
* Returns the next candidate info or null if no candidates are available.
|
|
2750
|
+
*/
|
|
2105
2751
|
async transitionToNextCandidate(currentModel, currentVendorId, failoverConfig, allCandidates, failoverAttempts, promptId, failoverAttempt, attemptNumber) {
|
|
2752
|
+
// Select next candidate using failover strategy
|
|
2106
2753
|
const nextCandidates = this.selectFailoverCandidates(currentModel, currentVendorId, failoverConfig.strategy, failoverConfig.modelStrategy, allCandidates, failoverAttempts);
|
|
2107
2754
|
if (nextCandidates.length === 0) {
|
|
2755
|
+
// No more candidates available
|
|
2108
2756
|
this.logFailoverAttempt(promptId, failoverAttempt, false);
|
|
2109
2757
|
return null;
|
|
2110
2758
|
}
|
|
2111
2759
|
const nextCandidate = nextCandidates[0];
|
|
2760
|
+
// Log the successful transition
|
|
2112
2761
|
this.logFailoverAttempt(promptId, failoverAttempt, true);
|
|
2762
|
+
// Apply delay before next attempt (if not the last attempt)
|
|
2113
2763
|
if (attemptNumber < failoverConfig.maxAttempts) {
|
|
2114
2764
|
const delay = this.calculateFailoverDelay(attemptNumber, failoverConfig.delaySeconds);
|
|
2115
2765
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
@@ -2122,6 +2772,9 @@ class AIPromptRunner {
|
|
|
2122
2772
|
supportsEffortLevel: nextCandidate.supportsEffortLevel || false
|
|
2123
2773
|
};
|
|
2124
2774
|
}
|
|
2775
|
+
/**
|
|
2776
|
+
* Provides a human-readable description of the validation decision
|
|
2777
|
+
*/
|
|
2125
2778
|
getValidationDecisionDescription(finalSuccess, totalAttempts, validationBehavior) {
|
|
2126
2779
|
if (finalSuccess) {
|
|
2127
2780
|
return totalAttempts === 1
|
|
@@ -2141,6 +2794,9 @@ class AIPromptRunner {
|
|
|
2141
2794
|
}
|
|
2142
2795
|
}
|
|
2143
2796
|
}
|
|
2797
|
+
/**
|
|
2798
|
+
* Generates a JSON schema from an example object for validation
|
|
2799
|
+
*/
|
|
2144
2800
|
generateSchemaFromExample(example) {
|
|
2145
2801
|
if (typeof example !== 'object' || example === null) {
|
|
2146
2802
|
return { type: 'object' };
|
|
@@ -2149,18 +2805,26 @@ class AIPromptRunner {
|
|
|
2149
2805
|
type: 'object',
|
|
2150
2806
|
properties: {},
|
|
2151
2807
|
required: [],
|
|
2152
|
-
additionalProperties: true,
|
|
2808
|
+
additionalProperties: true, // Allow additional properties for flexibility with examples
|
|
2153
2809
|
};
|
|
2810
|
+
// Check if this entire object appears to be a placeholder/example
|
|
2154
2811
|
const isPlaceholderObject = this.isObjectLikelyPlaceholder(example);
|
|
2155
2812
|
for (const [key, value] of Object.entries(example)) {
|
|
2813
|
+
// For placeholder objects, generate very permissive schemas
|
|
2156
2814
|
if (isPlaceholderObject) {
|
|
2815
|
+
// Don't define specific properties for placeholder objects
|
|
2816
|
+
// Just indicate it should be an object with any properties
|
|
2157
2817
|
schema.properties = {};
|
|
2158
2818
|
schema.required = [];
|
|
2159
2819
|
break;
|
|
2160
2820
|
}
|
|
2821
|
+
// Check if the key ends with '?' to indicate optional property (TypeScript style)
|
|
2161
2822
|
const isOptional = key.endsWith('?');
|
|
2162
2823
|
const cleanKey = isOptional ? key.slice(0, -1) : key;
|
|
2163
2824
|
schema.properties[cleanKey] = this.generateSchemaForValue(value);
|
|
2825
|
+
// Don't make fields required if:
|
|
2826
|
+
// 1. They're marked as optional with '?'
|
|
2827
|
+
// 2. They look like placeholder/example values
|
|
2164
2828
|
const isPlaceholder = this.isLikelyPlaceholder(cleanKey, value);
|
|
2165
2829
|
if (!isOptional && !isPlaceholder) {
|
|
2166
2830
|
schema.required.push(cleanKey);
|
|
@@ -2168,11 +2832,16 @@ class AIPromptRunner {
|
|
|
2168
2832
|
}
|
|
2169
2833
|
return schema;
|
|
2170
2834
|
}
|
|
2835
|
+
/**
|
|
2836
|
+
* Detects if a key/value pair looks like a placeholder or example value
|
|
2837
|
+
*/
|
|
2171
2838
|
isLikelyPlaceholder(key, value) {
|
|
2839
|
+
// Check if key contains common placeholder patterns
|
|
2172
2840
|
const placeholderKeyPatterns = /^(param|example|placeholder|sample|dummy|test)/i;
|
|
2173
2841
|
if (placeholderKeyPatterns.test(key)) {
|
|
2174
2842
|
return true;
|
|
2175
2843
|
}
|
|
2844
|
+
// Check if string value contains common placeholder text
|
|
2176
2845
|
if (typeof value === 'string') {
|
|
2177
2846
|
const placeholderValuePatterns = /(goes here|placeholder|example|sample value|value\d+|UUID|your .* here|insert .* here)/i;
|
|
2178
2847
|
if (placeholderValuePatterns.test(value)) {
|
|
@@ -2181,15 +2850,23 @@ class AIPromptRunner {
|
|
|
2181
2850
|
}
|
|
2182
2851
|
return false;
|
|
2183
2852
|
}
|
|
2853
|
+
/**
|
|
2854
|
+
* Detects if an entire object looks like it contains only placeholder/example data
|
|
2855
|
+
*/
|
|
2184
2856
|
isObjectLikelyPlaceholder(obj) {
|
|
2185
2857
|
if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
|
|
2186
2858
|
return false;
|
|
2187
2859
|
}
|
|
2188
2860
|
const entries = Object.entries(obj);
|
|
2861
|
+
// If object has placeholder-like keys (param1, param2, etc)
|
|
2189
2862
|
const hasPlaceholderKeys = entries.some(([key]) => /^(param\d+|key\d+|value\d+|example\d+|placeholder\d+)$/i.test(key));
|
|
2863
|
+
// If all values are simple placeholders
|
|
2190
2864
|
const allValuesArePlaceholders = entries.every(([key, value]) => this.isLikelyPlaceholder(key, value));
|
|
2191
2865
|
return hasPlaceholderKeys || allValuesArePlaceholders;
|
|
2192
2866
|
}
|
|
2867
|
+
/**
|
|
2868
|
+
* Generates schema for a specific value type
|
|
2869
|
+
*/
|
|
2193
2870
|
generateSchemaForValue(value) {
|
|
2194
2871
|
if (value === null) {
|
|
2195
2872
|
return { type: 'null' };
|
|
@@ -2207,7 +2884,7 @@ class AIPromptRunner {
|
|
|
2207
2884
|
return {
|
|
2208
2885
|
type: 'array',
|
|
2209
2886
|
items: this.generateSchemaForValue(value[0]),
|
|
2210
|
-
minItems: 0,
|
|
2887
|
+
minItems: 0, // Don't require minimum items for example arrays
|
|
2211
2888
|
};
|
|
2212
2889
|
}
|
|
2213
2890
|
else {
|
|
@@ -2218,9 +2895,19 @@ class AIPromptRunner {
|
|
|
2218
2895
|
return this.generateSchemaFromExample(value);
|
|
2219
2896
|
}
|
|
2220
2897
|
default:
|
|
2221
|
-
return { type: 'string' };
|
|
2222
|
-
}
|
|
2223
|
-
}
|
|
2898
|
+
return { type: 'string' }; // Fallback
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
/**
|
|
2902
|
+
* Enhanced parsing and validation with detailed error reporting and JSON repair capabilities.
|
|
2903
|
+
*
|
|
2904
|
+
* @param modelResult - The raw result from the AI model
|
|
2905
|
+
* @param prompt - The AI prompt entity containing configuration
|
|
2906
|
+
* @param skipValidation - Whether to skip validation
|
|
2907
|
+
* @param cleanValidationSyntax - Whether to clean validation syntax from results
|
|
2908
|
+
* @param params - Optional prompt parameters containing additional configuration like attemptJSONRepair
|
|
2909
|
+
* @returns Parsed result with optional validation results and errors
|
|
2910
|
+
*/
|
|
2224
2911
|
async parseAndValidateResultEnhanced(modelResult, prompt, skipValidation = false, cleanValidationSyntax = false, currentPromptRun, params) {
|
|
2225
2912
|
const validationErrors = [];
|
|
2226
2913
|
let rawOutput;
|
|
@@ -2232,6 +2919,7 @@ class AIPromptRunner {
|
|
|
2232
2919
|
if (!rawOutput) {
|
|
2233
2920
|
throw new Error('No output received from model');
|
|
2234
2921
|
}
|
|
2922
|
+
// Parse based on output type
|
|
2235
2923
|
let parsedResult = rawOutput;
|
|
2236
2924
|
try {
|
|
2237
2925
|
switch (prompt.OutputType) {
|
|
@@ -2255,24 +2943,27 @@ class AIPromptRunner {
|
|
|
2255
2943
|
}
|
|
2256
2944
|
}
|
|
2257
2945
|
catch (parseError) {
|
|
2258
|
-
|
|
2946
|
+
// Type parsing failed
|
|
2947
|
+
const validationResult = new ValidationResult();
|
|
2259
2948
|
validationResult.Success = false;
|
|
2260
|
-
const error = new
|
|
2949
|
+
const error = new ValidationErrorInfo('parseAndValidateResultEnhanced', `Invalid OutputExample JSON: ${parseError.message}`, rawOutput, ValidationErrorType.Failure);
|
|
2261
2950
|
validationErrors.push(error);
|
|
2262
2951
|
validationResult.Errors = validationErrors;
|
|
2263
2952
|
return { result: rawOutput, validationResult, validationErrors };
|
|
2264
2953
|
}
|
|
2954
|
+
// Perform JSON schema validation for object types
|
|
2265
2955
|
if (!skipValidation && prompt.OutputExample && prompt.OutputType === 'object' && parsedResult) {
|
|
2266
2956
|
try {
|
|
2267
2957
|
const schemaValidationErrors = await this.validateAgainstSchema(parsedResult, prompt.OutputExample, prompt.ID);
|
|
2268
2958
|
validationErrors.push(...schemaValidationErrors);
|
|
2269
2959
|
}
|
|
2270
2960
|
catch (schemaError) {
|
|
2271
|
-
const error = new
|
|
2961
|
+
const error = new ValidationErrorInfo('schema', `Schema validation failed: ${schemaError.message}`, undefined, ValidationErrorType.Failure);
|
|
2272
2962
|
validationErrors.push(error);
|
|
2273
2963
|
}
|
|
2274
2964
|
}
|
|
2275
|
-
|
|
2965
|
+
// Create validation result
|
|
2966
|
+
const validationResult = new ValidationResult();
|
|
2276
2967
|
validationResult.Success = validationErrors.length === 0;
|
|
2277
2968
|
validationResult.Errors = validationErrors;
|
|
2278
2969
|
return { result: parsedResult, validationResult, validationErrors };
|
|
@@ -2287,10 +2978,11 @@ class AIPromptRunner {
|
|
|
2287
2978
|
},
|
|
2288
2979
|
maxErrorLength: params?.maxErrorLength
|
|
2289
2980
|
});
|
|
2290
|
-
|
|
2981
|
+
// Handle validation behavior
|
|
2982
|
+
const validationResult = new ValidationResult();
|
|
2291
2983
|
validationResult.Success = false;
|
|
2292
2984
|
validationResult.Errors = validationErrors.length > 0 ? validationErrors : [
|
|
2293
|
-
new
|
|
2985
|
+
new ValidationErrorInfo('general', error.message, undefined, ValidationErrorType.Failure)
|
|
2294
2986
|
];
|
|
2295
2987
|
switch (prompt.ValidationBehavior) {
|
|
2296
2988
|
case 'Strict':
|
|
@@ -2309,26 +3001,51 @@ class AIPromptRunner {
|
|
|
2309
3001
|
return { result: modelResult.data?.choices?.[0]?.message?.content, validationResult, validationErrors: validationResult.Errors };
|
|
2310
3002
|
case 'None':
|
|
2311
3003
|
default:
|
|
3004
|
+
// For None, we still return the validation result but mark as successful
|
|
2312
3005
|
validationResult.Success = true;
|
|
2313
3006
|
return { result: modelResult.data?.choices?.[0]?.message?.content, validationResult, validationErrors: [] };
|
|
2314
3007
|
}
|
|
2315
3008
|
}
|
|
2316
3009
|
}
|
|
3010
|
+
/**
|
|
3011
|
+
* Parses a string output value.
|
|
3012
|
+
*
|
|
3013
|
+
* @param rawOutput - The raw output from the model
|
|
3014
|
+
* @returns The parsed string value
|
|
3015
|
+
*/
|
|
2317
3016
|
parseStringOutput(rawOutput) {
|
|
2318
3017
|
return rawOutput.toString();
|
|
2319
3018
|
}
|
|
3019
|
+
/**
|
|
3020
|
+
* Parses a number output value with validation.
|
|
3021
|
+
*
|
|
3022
|
+
* @param rawOutput - The raw output from the model
|
|
3023
|
+
* @param skipValidation - Whether to skip validation
|
|
3024
|
+
* @param validationErrors - Array to collect validation errors
|
|
3025
|
+
* @returns The parsed number value
|
|
3026
|
+
* @throws Error if the value cannot be parsed as a number and validation is enabled
|
|
3027
|
+
*/
|
|
2320
3028
|
parseNumberOutput(rawOutput, skipValidation, validationErrors) {
|
|
2321
3029
|
const numberResult = parseFloat(rawOutput);
|
|
2322
3030
|
if (isNaN(numberResult)) {
|
|
2323
3031
|
if (!skipValidation) {
|
|
2324
|
-
const error = new
|
|
3032
|
+
const error = new ValidationErrorInfo('output', `Expected number output but got: ${rawOutput}`, rawOutput, ValidationErrorType.Failure);
|
|
2325
3033
|
validationErrors.push(error);
|
|
2326
3034
|
throw new Error(error.Message);
|
|
2327
3035
|
}
|
|
2328
|
-
return numberResult;
|
|
3036
|
+
return numberResult; // Will be NaN if skipValidation is true
|
|
2329
3037
|
}
|
|
2330
3038
|
return numberResult;
|
|
2331
3039
|
}
|
|
3040
|
+
/**
|
|
3041
|
+
* Parses a boolean output value with flexible input handling.
|
|
3042
|
+
*
|
|
3043
|
+
* @param rawOutput - The raw output from the model
|
|
3044
|
+
* @param skipValidation - Whether to skip validation
|
|
3045
|
+
* @param validationErrors - Array to collect validation errors
|
|
3046
|
+
* @returns The parsed boolean value
|
|
3047
|
+
* @throws Error if the value cannot be parsed as a boolean and validation is enabled
|
|
3048
|
+
*/
|
|
2332
3049
|
parseBooleanOutput(rawOutput, skipValidation, validationErrors) {
|
|
2333
3050
|
const lowerOutput = rawOutput.toLowerCase().trim();
|
|
2334
3051
|
if (['true', 'yes', '1'].includes(lowerOutput)) {
|
|
@@ -2338,46 +3055,82 @@ class AIPromptRunner {
|
|
|
2338
3055
|
return false;
|
|
2339
3056
|
}
|
|
2340
3057
|
else if (!skipValidation) {
|
|
2341
|
-
const error = new
|
|
3058
|
+
const error = new ValidationErrorInfo('output', `Expected boolean output but got: ${rawOutput}`, rawOutput, ValidationErrorType.Failure);
|
|
2342
3059
|
validationErrors.push(error);
|
|
2343
3060
|
throw new Error(error.Message);
|
|
2344
3061
|
}
|
|
2345
|
-
return false;
|
|
2346
|
-
}
|
|
3062
|
+
return false; // Default to false if skipValidation is true
|
|
3063
|
+
}
|
|
3064
|
+
/**
|
|
3065
|
+
* Parses a date output value with validation.
|
|
3066
|
+
*
|
|
3067
|
+
* @param rawOutput - The raw output from the model
|
|
3068
|
+
* @param skipValidation - Whether to skip validation
|
|
3069
|
+
* @param validationErrors - Array to collect validation errors
|
|
3070
|
+
* @returns The parsed Date value
|
|
3071
|
+
* @throws Error if the value cannot be parsed as a date and validation is enabled
|
|
3072
|
+
*/
|
|
2347
3073
|
parseDateOutput(rawOutput, skipValidation, validationErrors) {
|
|
2348
3074
|
const dateResult = new Date(rawOutput);
|
|
2349
3075
|
if (isNaN(dateResult.getTime()) && !skipValidation) {
|
|
2350
|
-
const error = new
|
|
3076
|
+
const error = new ValidationErrorInfo('output', `Expected date output but got: ${rawOutput}`, rawOutput, ValidationErrorType.Failure);
|
|
2351
3077
|
validationErrors.push(error);
|
|
2352
3078
|
throw new Error(error.Message);
|
|
2353
3079
|
}
|
|
2354
3080
|
return dateResult;
|
|
2355
3081
|
}
|
|
3082
|
+
/**
|
|
3083
|
+
* Parses an object (JSON) output value with optional repair capabilities.
|
|
3084
|
+
*
|
|
3085
|
+
* @param rawOutput - The raw output from the model
|
|
3086
|
+
* @param prompt - The AI prompt entity containing configuration
|
|
3087
|
+
* @param skipValidation - Whether to skip validation
|
|
3088
|
+
* @param cleanValidationSyntax - Whether to clean validation syntax
|
|
3089
|
+
* @param validationErrors - Array to collect validation errors
|
|
3090
|
+
* @param params - Optional prompt parameters containing attemptJSONRepair flag
|
|
3091
|
+
* @returns The parsed object value
|
|
3092
|
+
* @throws Error if the value cannot be parsed as JSON and validation is enabled
|
|
3093
|
+
*/
|
|
2356
3094
|
async parseObjectOutput(rawOutput, prompt, skipValidation, cleanValidationSyntax, validationErrors, currentPromptRun, params) {
|
|
2357
3095
|
let parsedResult;
|
|
2358
3096
|
try {
|
|
2359
|
-
|
|
3097
|
+
// First attempt: Use CleanJSON to handle common JSON issues
|
|
3098
|
+
parsedResult = JSON.parse(CleanJSON(rawOutput));
|
|
2360
3099
|
}
|
|
2361
3100
|
catch (jsonError) {
|
|
3101
|
+
// If attemptJSONRepair is enabled and we're dealing with object output
|
|
2362
3102
|
if (params?.attemptJSONRepair && prompt.OutputType === 'object') {
|
|
2363
3103
|
parsedResult = await this.attemptJSONRepair(rawOutput, jsonError, params, currentPromptRun);
|
|
2364
3104
|
}
|
|
2365
3105
|
else {
|
|
3106
|
+
// Original error handling
|
|
2366
3107
|
if (!skipValidation) {
|
|
2367
|
-
const error = new
|
|
3108
|
+
const error = new ValidationErrorInfo('output', `Expected JSON object but got invalid JSON: ${rawOutput}`, rawOutput, ValidationErrorType.Failure);
|
|
2368
3109
|
validationErrors.push(error);
|
|
2369
3110
|
throw new Error(error.Message);
|
|
2370
3111
|
}
|
|
2371
|
-
return rawOutput;
|
|
3112
|
+
return rawOutput; // Return raw output if skipping validation
|
|
2372
3113
|
}
|
|
2373
3114
|
}
|
|
3115
|
+
// Clean validation syntax if needed
|
|
2374
3116
|
if (parsedResult && (cleanValidationSyntax || (!skipValidation && prompt.OutputExample))) {
|
|
2375
|
-
const validator = new
|
|
3117
|
+
const validator = new JSONValidator();
|
|
2376
3118
|
parsedResult = validator.cleanValidationSyntax(parsedResult);
|
|
2377
3119
|
}
|
|
2378
3120
|
return parsedResult;
|
|
2379
3121
|
}
|
|
3122
|
+
/**
|
|
3123
|
+
* Attempts to repair malformed JSON using a two-step process.
|
|
3124
|
+
*
|
|
3125
|
+
* @param rawOutput - The malformed JSON string
|
|
3126
|
+
* @param originalError - The original parsing error
|
|
3127
|
+
* @param params - Prompt parameters containing contextUser
|
|
3128
|
+
* @returns The repaired and parsed JSON object
|
|
3129
|
+
* @throws Error if JSON repair fails
|
|
3130
|
+
*/
|
|
2380
3131
|
async attemptJSONRepair(rawOutput, originalError, params, currentPromptRun) {
|
|
3132
|
+
// Step 0: First, see if the raw output has any { } [ ] characters at all
|
|
3133
|
+
// if not, we KNOW it is not JSON and we should not attempt to repair it
|
|
2381
3134
|
if (!rawOutput.includes('{') && !rawOutput.includes('[')) {
|
|
2382
3135
|
this.logError(new Error('Raw output does not contain any JSON-like characters'), {
|
|
2383
3136
|
category: 'JSONRepairSkipped',
|
|
@@ -2389,11 +3142,13 @@ class AIPromptRunner {
|
|
|
2389
3142
|
});
|
|
2390
3143
|
throw new Error(`JSON repair skipped: raw output does not contain JSON-like characters. Original error: ${originalError.message}`);
|
|
2391
3144
|
}
|
|
3145
|
+
// Step 1: Try JSON5 parsing
|
|
2392
3146
|
try {
|
|
2393
3147
|
this.logStatus(' 🔧 Attempting JSON repair with JSON5...', true, params);
|
|
3148
|
+
// first try to clean JSON in case we have it in a markdown block
|
|
2394
3149
|
let jsonToParse = rawOutput;
|
|
2395
3150
|
try {
|
|
2396
|
-
jsonToParse =
|
|
3151
|
+
jsonToParse = CleanJSON(rawOutput);
|
|
2397
3152
|
}
|
|
2398
3153
|
catch (cleanError) {
|
|
2399
3154
|
if (params.verbose) {
|
|
@@ -2414,14 +3169,17 @@ class AIPromptRunner {
|
|
|
2414
3169
|
return json5Result;
|
|
2415
3170
|
}
|
|
2416
3171
|
catch (json5Error) {
|
|
3172
|
+
// Step 2: Use AI to repair the JSON
|
|
2417
3173
|
if (params.verbose) {
|
|
2418
3174
|
this.logStatus(' 🤖 JSON5 failed, attempting AI-based JSON repair...', true, params);
|
|
2419
3175
|
}
|
|
2420
3176
|
try {
|
|
2421
|
-
|
|
3177
|
+
// Find the "Repair JSON" prompt in the "MJ: System" category
|
|
3178
|
+
const repairPrompt = AIEngine.Instance.Prompts.find(p => p.Name.trim().toLowerCase() === 'repair json' && p.Category.trim().toLowerCase() === 'mj: system');
|
|
2422
3179
|
if (!repairPrompt) {
|
|
2423
3180
|
throw new Error('Repair JSON prompt not found in MJ: System category');
|
|
2424
3181
|
}
|
|
3182
|
+
// Run the repair prompt
|
|
2425
3183
|
const repairResult = await this.ExecutePrompt({
|
|
2426
3184
|
parentPromptRunId: currentPromptRun.ID,
|
|
2427
3185
|
agentRunId: currentPromptRun.AgentRunID,
|
|
@@ -2431,19 +3189,23 @@ class AIPromptRunner {
|
|
|
2431
3189
|
ERROR_MESSAGE: originalError.message,
|
|
2432
3190
|
MALFORMED_JSON: rawOutput
|
|
2433
3191
|
},
|
|
2434
|
-
skipValidation: true
|
|
3192
|
+
skipValidation: true // don't want to validate as this would cause recursive infinity scenario if the JSON is invalid. Just one shot, fix or no fix
|
|
2435
3193
|
});
|
|
2436
3194
|
if (!repairResult.success || !repairResult.result) {
|
|
2437
3195
|
throw new Error('AI-based JSON repair failed' + (repairResult.errorMessage ? `: ${repairResult.errorMessage}` : ''));
|
|
2438
3196
|
}
|
|
3197
|
+
// if we get here we have the text result in the reapairResult.result so let's try to parse it
|
|
2439
3198
|
const repairedJSON = JSON.parse(repairResult.result);
|
|
3199
|
+
// make sure repairedJSON is not this object: { error: "not_json" } -- if it is that means the LLM said it isn't JSOn
|
|
2440
3200
|
if (repairedJSON && typeof repairedJSON === 'object' && Object.keys(repairedJSON).length === 1 && repairedJSON.error?.trim().toLowerCase() === 'not_json') {
|
|
2441
3201
|
throw new Error('AI-based JSON repair returned a non-JSON response indicating it could not repair the JSON');
|
|
2442
3202
|
}
|
|
3203
|
+
// if we get here, we successfully repaired the JSON!!!
|
|
2443
3204
|
this.logStatus(' ✅ AI successfully repaired the JSON', true, params);
|
|
2444
3205
|
return repairedJSON;
|
|
2445
3206
|
}
|
|
2446
3207
|
catch (aiRepairError) {
|
|
3208
|
+
// Both repair attempts failed
|
|
2447
3209
|
if (params.verbose) {
|
|
2448
3210
|
this.logError(aiRepairError, {
|
|
2449
3211
|
category: 'JSONRepairFailed',
|
|
@@ -2460,48 +3222,115 @@ class AIPromptRunner {
|
|
|
2460
3222
|
}
|
|
2461
3223
|
}
|
|
2462
3224
|
}
|
|
3225
|
+
/**
|
|
3226
|
+
* Validates parsed result against JSON schema derived from OutputExample
|
|
3227
|
+
*/
|
|
2463
3228
|
async validateAgainstSchema(parsedResult, outputExample, promptId) {
|
|
2464
3229
|
const validationErrors = [];
|
|
2465
3230
|
try {
|
|
3231
|
+
// Parse the output example
|
|
2466
3232
|
let exampleObject;
|
|
2467
3233
|
try {
|
|
2468
3234
|
exampleObject = JSON.parse(outputExample);
|
|
2469
3235
|
}
|
|
2470
3236
|
catch (parseError) {
|
|
2471
|
-
const error = new
|
|
3237
|
+
const error = new ValidationErrorInfo('outputExample', `Invalid OutputExample JSON: ${parseError.message}`, outputExample, ValidationErrorType.Failure);
|
|
2472
3238
|
validationErrors.push(error);
|
|
2473
3239
|
return validationErrors;
|
|
2474
3240
|
}
|
|
3241
|
+
// Use the JSONValidator to validate against the example
|
|
2475
3242
|
const validationResult = this._jsonValidator.validate(parsedResult, exampleObject);
|
|
2476
3243
|
validationErrors.push(...validationResult.Errors);
|
|
2477
3244
|
if (validationErrors.length !== 0) {
|
|
2478
|
-
|
|
3245
|
+
LogStatus(`⚠️ Validation found ${validationErrors.length} issues for prompt ${promptId}:`);
|
|
2479
3246
|
validationErrors.forEach((error, index) => {
|
|
2480
|
-
|
|
3247
|
+
LogStatus(` ${index + 1}. ${error.Source}: ${error.Message}`);
|
|
2481
3248
|
});
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
}
|
|
3249
|
+
LogStatus(` Note: Validation syntax in OutputExample:`);
|
|
3250
|
+
LogStatus(` - '?' = optional field (e.g., "reasoning?": "...")`);
|
|
3251
|
+
LogStatus(` - '*' = required but any content (e.g., "payload*": {})`);
|
|
3252
|
+
LogStatus(` - ':type' = type validation (e.g., "age:number": 25)`);
|
|
3253
|
+
LogStatus(` - ':[N+]' = array length (e.g., "items:[2+]": [])`);
|
|
3254
|
+
}
|
|
3255
|
+
/* FUTURE IMPLEMENTATION - Keep this commented for reference
|
|
3256
|
+
// Get or create cached validator for this prompt using static cache
|
|
3257
|
+
let validator = AIPromptRunner._schemaCache.get(promptId);
|
|
3258
|
+
|
|
3259
|
+
if (!validator) {
|
|
3260
|
+
// Parse the output example
|
|
3261
|
+
let exampleObject: unknown;
|
|
3262
|
+
try {
|
|
3263
|
+
exampleObject = JSON.parse(outputExample);
|
|
3264
|
+
} catch (parseError) {
|
|
3265
|
+
const error = new ValidationErrorInfo('outputExample', `Invalid OutputExample JSON: ${parseError.message}`, outputExample, ValidationErrorType.Failure);
|
|
3266
|
+
validationErrors.push(error);
|
|
3267
|
+
return validationErrors;
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3270
|
+
// Generate schema from example
|
|
3271
|
+
const schema = this.generateSchemaFromExample(exampleObject);
|
|
3272
|
+
|
|
3273
|
+
// Compile and cache the validator
|
|
3274
|
+
try {
|
|
3275
|
+
validator = this._ajv.compile(schema);
|
|
3276
|
+
AIPromptRunner._schemaCache.set(promptId, validator);
|
|
3277
|
+
const cacheStats = AIPromptRunner.getSchemaCacheStats();
|
|
3278
|
+
LogStatus(`📋 Compiled and cached JSON schema for prompt ${promptId} (global cache size: ${cacheStats.size})`);
|
|
3279
|
+
} catch (compileError) {
|
|
3280
|
+
const error = new ValidationErrorInfo('schema', `Failed to compile schema: ${compileError.message}`, schema, ValidationErrorType.Failure);
|
|
3281
|
+
validationErrors.push(error);
|
|
3282
|
+
return validationErrors;
|
|
3283
|
+
}
|
|
3284
|
+
}
|
|
3285
|
+
|
|
3286
|
+
// Validate the result
|
|
3287
|
+
const isValid = validator(parsedResult);
|
|
3288
|
+
|
|
3289
|
+
if (!isValid && validator.errors) {
|
|
3290
|
+
for (const ajvError of validator.errors) {
|
|
3291
|
+
const fieldPath = ajvError.instancePath || ajvError.schemaPath || 'root';
|
|
3292
|
+
const message = `${ajvError.instancePath || 'root'}: ${ajvError.message}`;
|
|
3293
|
+
const error = new ValidationErrorInfo(fieldPath, message, ajvError.data, ValidationErrorType.Failure);
|
|
3294
|
+
validationErrors.push(error);
|
|
3295
|
+
}
|
|
3296
|
+
}
|
|
3297
|
+
|
|
3298
|
+
if (validationErrors.length === 0) {
|
|
3299
|
+
//LogStatus(`✅ Schema validation passed for prompt ${promptId}`);
|
|
3300
|
+
} else {
|
|
3301
|
+
LogStatus(`⚠️ Schema validation found ${validationErrors.length} potential issues for prompt ${promptId}:`);
|
|
3302
|
+
validationErrors.forEach((error, index) => {
|
|
3303
|
+
LogStatus(` ${index + 1}. ${error.Source}: ${error.Message}`);
|
|
3304
|
+
});
|
|
3305
|
+
// Log additional context to help with debugging
|
|
3306
|
+
LogStatus(` Note: The schema was generated from OutputExample. Consider:`);
|
|
3307
|
+
LogStatus(` - Mark optional properties with '?' suffix (e.g., "subAgent?": {...})`)
|
|
3308
|
+
LogStatus(` - Example values like "param1", "value1" are treated as placeholders`);
|
|
3309
|
+
}
|
|
3310
|
+
*/
|
|
2488
3311
|
}
|
|
2489
3312
|
catch (error) {
|
|
2490
|
-
const validationError = new
|
|
3313
|
+
const validationError = new ValidationErrorInfo('validation', `Unexpected validation error: ${error.message}`, undefined, ValidationErrorType.Failure);
|
|
2491
3314
|
validationErrors.push(validationError);
|
|
2492
3315
|
}
|
|
2493
3316
|
return validationErrors;
|
|
2494
3317
|
}
|
|
3318
|
+
/**
|
|
3319
|
+
* Updates the AIPromptRun entity with execution results
|
|
3320
|
+
*/
|
|
2495
3321
|
async updatePromptRun(promptRun, prompt, modelResult, parsedResult, endTime, executionTimeMS, validationAttempts, cumulativeTokens) {
|
|
2496
3322
|
try {
|
|
2497
3323
|
promptRun.CompletedAt = endTime;
|
|
2498
3324
|
promptRun.ExecutionTimeMS = executionTimeMS;
|
|
3325
|
+
// Determine what to save as the result
|
|
2499
3326
|
let resultToSave;
|
|
2500
3327
|
const rawResult = modelResult.data?.choices?.[0]?.message?.content || '';
|
|
2501
3328
|
if (parsedResult.result === undefined ||
|
|
2502
3329
|
parsedResult.result === null ||
|
|
2503
3330
|
(typeof parsedResult.result === 'string' && parsedResult.result.trim().length === 0)) {
|
|
3331
|
+
// Use raw result as fallback when parsed result is undefined, null, or empty string
|
|
2504
3332
|
resultToSave = rawResult;
|
|
3333
|
+
// Also set error message when we have to fall back to raw result
|
|
2505
3334
|
if (!promptRun.ErrorMessage) {
|
|
2506
3335
|
const validationErrors = parsedResult.validationResult?.Errors;
|
|
2507
3336
|
if (validationErrors && validationErrors.length > 0) {
|
|
@@ -2519,25 +3348,31 @@ class AIPromptRunner {
|
|
|
2519
3348
|
resultToSave = JSON.stringify(parsedResult.result);
|
|
2520
3349
|
}
|
|
2521
3350
|
promptRun.Result = resultToSave;
|
|
3351
|
+
// Extract token usage and cost - use cumulative if retries occurred
|
|
2522
3352
|
if (cumulativeTokens && validationAttempts && validationAttempts.length > 1) {
|
|
3353
|
+
// Multiple attempts occurred, use cumulative totals
|
|
2523
3354
|
promptRun.TokensPrompt = cumulativeTokens.promptTokens;
|
|
2524
3355
|
promptRun.TokensCompletion = cumulativeTokens.completionTokens;
|
|
2525
3356
|
promptRun.TokensUsed = cumulativeTokens.promptTokens + cumulativeTokens.completionTokens;
|
|
2526
3357
|
promptRun.Cost = cumulativeTokens.totalCost;
|
|
3358
|
+
// Cost currency from the last model result
|
|
2527
3359
|
if (modelResult.data?.usage?.costCurrency !== undefined) {
|
|
2528
3360
|
promptRun.CostCurrency = modelResult.data.usage.costCurrency;
|
|
2529
3361
|
}
|
|
2530
3362
|
}
|
|
2531
3363
|
else if (modelResult.data?.usage) {
|
|
3364
|
+
// Single attempt, use standard token tracking
|
|
2532
3365
|
promptRun.TokensUsed = modelResult.data.usage.totalTokens;
|
|
2533
3366
|
promptRun.TokensPrompt = modelResult.data.usage.promptTokens;
|
|
2534
3367
|
promptRun.TokensCompletion = modelResult.data.usage.completionTokens;
|
|
3368
|
+
// Save cost information if available
|
|
2535
3369
|
if (modelResult.data.usage.cost !== undefined) {
|
|
2536
3370
|
promptRun.Cost = modelResult.data.usage.cost;
|
|
2537
3371
|
}
|
|
2538
3372
|
if (modelResult.data.usage.costCurrency !== undefined) {
|
|
2539
3373
|
promptRun.CostCurrency = modelResult.data.usage.costCurrency;
|
|
2540
3374
|
}
|
|
3375
|
+
// Save timing information if available
|
|
2541
3376
|
if (modelResult.data.usage.queueTime !== undefined) {
|
|
2542
3377
|
promptRun.QueueTime = modelResult.data.usage.queueTime;
|
|
2543
3378
|
}
|
|
@@ -2548,14 +3383,18 @@ class AIPromptRunner {
|
|
|
2548
3383
|
promptRun.CompletionTime = modelResult.data.usage.completionTime;
|
|
2549
3384
|
}
|
|
2550
3385
|
}
|
|
3386
|
+
// Save model-specific response details if available
|
|
2551
3387
|
if (modelResult.modelSpecificResponseDetails) {
|
|
2552
3388
|
promptRun.ModelSpecificResponseDetails = JSON.stringify(modelResult.modelSpecificResponseDetails);
|
|
2553
3389
|
}
|
|
3390
|
+
// Populate retry tracking columns
|
|
2554
3391
|
if (validationAttempts && validationAttempts.length > 0) {
|
|
3392
|
+
// Update retry tracking columns
|
|
2555
3393
|
promptRun.ValidationAttemptCount = validationAttempts.length;
|
|
2556
3394
|
promptRun.SuccessfulValidationCount = validationAttempts.filter(a => a.success).length;
|
|
2557
3395
|
promptRun.FinalValidationPassed = parsedResult.validationResult?.Success === true;
|
|
2558
3396
|
promptRun.LastAttemptAt = endTime;
|
|
3397
|
+
// Calculate total retry duration (excluding first attempt)
|
|
2559
3398
|
if (validationAttempts.length > 1) {
|
|
2560
3399
|
const firstAttemptTime = validationAttempts[0].timestamp;
|
|
2561
3400
|
const lastAttemptTime = validationAttempts[validationAttempts.length - 1].timestamp;
|
|
@@ -2564,11 +3403,13 @@ class AIPromptRunner {
|
|
|
2564
3403
|
else {
|
|
2565
3404
|
promptRun.TotalRetryDurationMS = 0;
|
|
2566
3405
|
}
|
|
3406
|
+
// Get final validation error if any
|
|
2567
3407
|
const finalAttempt = validationAttempts[validationAttempts.length - 1];
|
|
2568
3408
|
if (!finalAttempt.success && finalAttempt.errorMessage) {
|
|
2569
|
-
promptRun.FinalValidationError = finalAttempt.errorMessage.substring(0, 500);
|
|
3409
|
+
promptRun.FinalValidationError = finalAttempt.errorMessage.substring(0, 500); // Truncate to fit column
|
|
2570
3410
|
promptRun.ValidationErrorCount = finalAttempt.validationErrors?.length || 0;
|
|
2571
3411
|
}
|
|
3412
|
+
// Find most common validation error
|
|
2572
3413
|
if (validationAttempts.some(a => !a.success)) {
|
|
2573
3414
|
const errorCounts = new Map();
|
|
2574
3415
|
validationAttempts.forEach(attempt => {
|
|
@@ -2579,9 +3420,10 @@ class AIPromptRunner {
|
|
|
2579
3420
|
});
|
|
2580
3421
|
if (errorCounts.size > 0) {
|
|
2581
3422
|
const [commonError] = [...errorCounts.entries()].sort((a, b) => b[1] - a[1])[0];
|
|
2582
|
-
promptRun.CommonValidationError = commonError.substring(0, 255);
|
|
3423
|
+
promptRun.CommonValidationError = commonError.substring(0, 255); // Truncate to fit column
|
|
2583
3424
|
}
|
|
2584
3425
|
}
|
|
3426
|
+
// Store detailed attempts in JSON columns
|
|
2585
3427
|
promptRun.ValidationAttempts = JSON.stringify(validationAttempts.map(a => ({
|
|
2586
3428
|
attemptNumber: a.attemptNumber,
|
|
2587
3429
|
success: a.success,
|
|
@@ -2613,14 +3455,18 @@ class AIPromptRunner {
|
|
|
2613
3455
|
});
|
|
2614
3456
|
}
|
|
2615
3457
|
else {
|
|
2616
|
-
|
|
3458
|
+
// No validation attempts (possibly skipped validation)
|
|
3459
|
+
promptRun.ValidationAttemptCount = 1; // At least one attempt was made
|
|
2617
3460
|
promptRun.SuccessfulValidationCount = parsedResult.validationResult?.Success !== false ? 1 : 0;
|
|
2618
3461
|
promptRun.FinalValidationPassed = parsedResult.validationResult?.Success !== false;
|
|
2619
3462
|
promptRun.LastAttemptAt = endTime;
|
|
2620
3463
|
promptRun.TotalRetryDurationMS = 0;
|
|
2621
3464
|
}
|
|
3465
|
+
// Set Success flag based on validation result
|
|
2622
3466
|
promptRun.Success = modelResult.success && (parsedResult.validationResult?.Success !== false);
|
|
3467
|
+
// Set final Status based on success
|
|
2623
3468
|
promptRun.Status = promptRun.Success ? 'Completed' : 'Failed';
|
|
3469
|
+
// Set ErrorDetails if failed
|
|
2624
3470
|
if (!promptRun.Success) {
|
|
2625
3471
|
if (!modelResult.success && modelResult.errorMessage) {
|
|
2626
3472
|
promptRun.ErrorDetails = modelResult.errorMessage;
|
|
@@ -2629,6 +3475,9 @@ class AIPromptRunner {
|
|
|
2629
3475
|
promptRun.ErrorDetails = `Validation failed: ${parsedResult.validationResult.Errors?.map(e => e.Message).join(', ')}`;
|
|
2630
3476
|
}
|
|
2631
3477
|
}
|
|
3478
|
+
// Note: Failover tracking fields are now updated directly in executeModelWithFailover
|
|
3479
|
+
// The promptRun entity already has the failover information set
|
|
3480
|
+
// With template composition, we only execute once so rollup equals regular fields
|
|
2632
3481
|
promptRun.TokensPromptRollup = promptRun.TokensPrompt;
|
|
2633
3482
|
promptRun.TokensCompletionRollup = promptRun.TokensCompletion;
|
|
2634
3483
|
promptRun.TokensUsedRollup = promptRun.TokensUsed;
|
|
@@ -2637,6 +3486,7 @@ class AIPromptRunner {
|
|
|
2637
3486
|
}
|
|
2638
3487
|
const saveResult = await promptRun.Save();
|
|
2639
3488
|
if (!saveResult) {
|
|
3489
|
+
// Safely extract error message using CompleteMessage getter
|
|
2640
3490
|
let errorMsg = 'Unknown error';
|
|
2641
3491
|
try {
|
|
2642
3492
|
if (promptRun.LatestResult?.CompleteMessage) {
|
|
@@ -2666,6 +3516,27 @@ class AIPromptRunner {
|
|
|
2666
3516
|
});
|
|
2667
3517
|
}
|
|
2668
3518
|
}
|
|
3519
|
+
// ==================== CONTEXT LENGTH METHODS ====================
|
|
3520
|
+
/**
|
|
3521
|
+
* Estimates the number of tokens in a rendered prompt and conversation messages.
|
|
3522
|
+
* This is a rough estimation based on character count and typical token ratios.
|
|
3523
|
+
*
|
|
3524
|
+
* @param renderedPrompt - The rendered prompt text
|
|
3525
|
+
* @param conversationMessages - Optional conversation messages
|
|
3526
|
+
* @returns Estimated token count
|
|
3527
|
+
*/
|
|
3528
|
+
// ==================== FAILOVER METHODS ====================
|
|
3529
|
+
/**
|
|
3530
|
+
* Retrieves failover configuration from the prompt entity.
|
|
3531
|
+
*
|
|
3532
|
+
* @param prompt - The AI prompt entity containing failover settings
|
|
3533
|
+
* @returns FailoverConfiguration object with strategy and settings
|
|
3534
|
+
*
|
|
3535
|
+
* @remarks
|
|
3536
|
+
* This method extracts failover configuration from the prompt entity and provides
|
|
3537
|
+
* default values when configuration is not specified. Override this method to
|
|
3538
|
+
* implement custom failover configuration logic.
|
|
3539
|
+
*/
|
|
2669
3540
|
getFailoverConfiguration(prompt) {
|
|
2670
3541
|
return {
|
|
2671
3542
|
strategy: prompt.FailoverStrategy || 'None',
|
|
@@ -2675,14 +3546,31 @@ class AIPromptRunner {
|
|
|
2675
3546
|
errorScope: prompt.FailoverErrorScope || 'All'
|
|
2676
3547
|
};
|
|
2677
3548
|
}
|
|
3549
|
+
/**
|
|
3550
|
+
* Determines whether a failover attempt should be made based on the error and configuration.
|
|
3551
|
+
*
|
|
3552
|
+
* @param error - The error that occurred during execution
|
|
3553
|
+
* @param config - The failover configuration
|
|
3554
|
+
* @param attemptNumber - The current attempt number (1-based)
|
|
3555
|
+
* @returns True if failover should be attempted, false otherwise
|
|
3556
|
+
*
|
|
3557
|
+
* @remarks
|
|
3558
|
+
* This method uses the ErrorAnalyzer to classify errors and determine if they are
|
|
3559
|
+
* eligible for failover based on the configured error scope. Override this method
|
|
3560
|
+
* to implement custom failover decision logic.
|
|
3561
|
+
*/
|
|
2678
3562
|
shouldAttemptFailover(error, config, attemptNumber) {
|
|
3563
|
+
// Don't failover if strategy is None or we've exceeded max attempts
|
|
2679
3564
|
if (config.strategy === 'None' || attemptNumber > config.maxAttempts) {
|
|
2680
3565
|
return false;
|
|
2681
3566
|
}
|
|
2682
|
-
|
|
3567
|
+
// Analyze the error to determine if it's eligible for failover
|
|
3568
|
+
const errorAnalysis = ErrorAnalyzer.analyzeError(error);
|
|
3569
|
+
// Check if error analysis allows failover
|
|
2683
3570
|
if (!errorAnalysis.canFailover) {
|
|
2684
3571
|
return false;
|
|
2685
3572
|
}
|
|
3573
|
+
// Check error scope configuration
|
|
2686
3574
|
switch (config.errorScope) {
|
|
2687
3575
|
case 'NetworkOnly':
|
|
2688
3576
|
return errorAnalysis.errorType === 'NetworkError';
|
|
@@ -2696,6 +3584,13 @@ class AIPromptRunner {
|
|
|
2696
3584
|
return true;
|
|
2697
3585
|
}
|
|
2698
3586
|
}
|
|
3587
|
+
/**
|
|
3588
|
+
* Checks if an error type matches the configured error scope
|
|
3589
|
+
*
|
|
3590
|
+
* @param errorType - The error type from ErrorAnalyzer
|
|
3591
|
+
* @param scope - The configured error scope
|
|
3592
|
+
* @returns True if the error matches the scope
|
|
3593
|
+
*/
|
|
2699
3594
|
errorMatchesScope(errorType, scope) {
|
|
2700
3595
|
switch (scope) {
|
|
2701
3596
|
case 'NetworkOnly':
|
|
@@ -2709,31 +3604,74 @@ class AIPromptRunner {
|
|
|
2709
3604
|
return true;
|
|
2710
3605
|
}
|
|
2711
3606
|
}
|
|
3607
|
+
/**
|
|
3608
|
+
* Calculates the delay before the next failover attempt.
|
|
3609
|
+
*
|
|
3610
|
+
* @param attemptNumber - The current attempt number (1-based)
|
|
3611
|
+
* @param baseDelaySeconds - The base delay in seconds from configuration
|
|
3612
|
+
* @param previousError - The error from the previous attempt
|
|
3613
|
+
* @returns Delay in milliseconds before the next attempt
|
|
3614
|
+
*
|
|
3615
|
+
* @remarks
|
|
3616
|
+
* Implements exponential backoff with jitter by default. The delay increases
|
|
3617
|
+
* exponentially with each attempt and includes random jitter to prevent
|
|
3618
|
+
* thundering herd problems. Override this method to implement custom delay logic.
|
|
3619
|
+
*/
|
|
2712
3620
|
calculateFailoverDelay(attemptNumber, baseDelaySeconds) {
|
|
3621
|
+
// Exponential backoff: delay = base * 2^(attempt-1)
|
|
2713
3622
|
const exponentialDelay = baseDelaySeconds * Math.pow(2, attemptNumber - 1);
|
|
3623
|
+
// Add jitter (0-25% of delay) to prevent thundering herd
|
|
2714
3624
|
const jitter = exponentialDelay * 0.25 * Math.random();
|
|
3625
|
+
// Cap at 30 seconds to prevent excessive delays
|
|
2715
3626
|
const totalDelay = Math.min(exponentialDelay + jitter, 30);
|
|
2716
|
-
return totalDelay * 1000;
|
|
2717
|
-
}
|
|
3627
|
+
return totalDelay * 1000; // Convert to milliseconds
|
|
3628
|
+
}
|
|
3629
|
+
/**
|
|
3630
|
+
* Selects candidate models for failover based on the strategy and current failure.
|
|
3631
|
+
*
|
|
3632
|
+
* @param currentModel - The model that just failed
|
|
3633
|
+
* @param currentVendorId - The vendor ID that just failed
|
|
3634
|
+
* @param strategy - The failover strategy to use
|
|
3635
|
+
* @param modelStrategy - The model selection preference
|
|
3636
|
+
* @param allCandidates - All available model-vendor candidates
|
|
3637
|
+
* @param attemptHistory - History of previous failover attempts
|
|
3638
|
+
* @returns Array of candidates sorted by priority (highest first)
|
|
3639
|
+
*
|
|
3640
|
+
* @remarks
|
|
3641
|
+
* This method implements different strategies for selecting failover candidates:
|
|
3642
|
+
* - SameModelDifferentVendor: Try the same model with different vendors
|
|
3643
|
+
* - NextBestModel: Try different models in order of preference
|
|
3644
|
+
* - PowerRank: Use the global power ranking of models
|
|
3645
|
+
*
|
|
3646
|
+
* Override this method to implement custom candidate selection logic.
|
|
3647
|
+
*/
|
|
2718
3648
|
selectFailoverCandidates(currentModel, currentVendorId, strategy, modelStrategy, allCandidates, attemptHistory) {
|
|
3649
|
+
// Filter out candidates that have already failed
|
|
3650
|
+
// Note: Authentication errors are already filtered from allCandidates upstream,
|
|
3651
|
+
// so we only need to filter out specific model/vendor pairs that have failed
|
|
2719
3652
|
const failedPairs = new Set(attemptHistory.map(a => `${a.modelId}:${a.vendorId || 'default'}`));
|
|
2720
3653
|
const availableCandidates = allCandidates.filter(c => {
|
|
2721
3654
|
const key = `${c.model.ID}:${c.vendorId || 'default'}`;
|
|
2722
3655
|
return !failedPairs.has(key);
|
|
2723
3656
|
});
|
|
3657
|
+
// Check if we have context length exceeded errors in the attempt history
|
|
2724
3658
|
const hasContextLengthError = attemptHistory.some(a => a.errorType === 'ContextLengthExceeded' ||
|
|
2725
|
-
|
|
3659
|
+
ErrorAnalyzer.analyzeError(a.error).errorType === 'ContextLengthExceeded');
|
|
3660
|
+
// Apply strategy-specific filtering and sorting
|
|
2726
3661
|
let candidates;
|
|
2727
3662
|
switch (strategy) {
|
|
2728
3663
|
case 'SameModelDifferentVendor':
|
|
3664
|
+
// Only consider same model with different vendors
|
|
2729
3665
|
candidates = availableCandidates.filter(c => c.model.ID === currentModel.ID && c.vendorId !== currentVendorId);
|
|
2730
3666
|
break;
|
|
2731
3667
|
case 'NextBestModel':
|
|
3668
|
+
// Consider all models, apply model strategy preference
|
|
2732
3669
|
candidates = availableCandidates;
|
|
2733
3670
|
if (modelStrategy === 'RequireSameModel') {
|
|
2734
3671
|
candidates = candidates.filter(c => c.model.ID === currentModel.ID);
|
|
2735
3672
|
}
|
|
2736
3673
|
else if (modelStrategy === 'PreferSameModel') {
|
|
3674
|
+
// Sort to put same model first
|
|
2737
3675
|
candidates.sort((a, b) => {
|
|
2738
3676
|
const aSameModel = a.model.ID === currentModel.ID ? 1 : 0;
|
|
2739
3677
|
const bSameModel = b.model.ID === currentModel.ID ? 1 : 0;
|
|
@@ -2741,6 +3679,7 @@ class AIPromptRunner {
|
|
|
2741
3679
|
});
|
|
2742
3680
|
}
|
|
2743
3681
|
else if (modelStrategy === 'PreferDifferentModel') {
|
|
3682
|
+
// Sort to put different models first
|
|
2744
3683
|
candidates.sort((a, b) => {
|
|
2745
3684
|
const aDiffModel = a.model.ID !== currentModel.ID ? 1 : 0;
|
|
2746
3685
|
const bDiffModel = b.model.ID !== currentModel.ID ? 1 : 0;
|
|
@@ -2749,21 +3688,25 @@ class AIPromptRunner {
|
|
|
2749
3688
|
}
|
|
2750
3689
|
break;
|
|
2751
3690
|
case 'PowerRank':
|
|
3691
|
+
// Use all candidates, they're already sorted by power rank
|
|
2752
3692
|
candidates = availableCandidates;
|
|
2753
3693
|
break;
|
|
2754
3694
|
default:
|
|
2755
3695
|
candidates = [];
|
|
2756
3696
|
}
|
|
3697
|
+
// If we have context length errors, prioritize models with larger context windows
|
|
2757
3698
|
if (hasContextLengthError) {
|
|
2758
3699
|
const currentMaxTokens = currentModel.ModelVendors?.length > 0 ?
|
|
2759
3700
|
Math.max(...currentModel.ModelVendors.map(mv => mv.MaxInputTokens || 0)) : 0;
|
|
3701
|
+
// Filter out models with same or smaller context windows
|
|
2760
3702
|
candidates = candidates.filter(c => {
|
|
2761
3703
|
const candidateMaxTokens = c.model.ModelVendors?.length > 0 ?
|
|
2762
3704
|
Math.max(...c.model.ModelVendors.map(mv => mv.MaxInputTokens || 0)) : 0;
|
|
2763
3705
|
return candidateMaxTokens > currentMaxTokens;
|
|
2764
3706
|
});
|
|
3707
|
+
// If no larger models exist, this is a fatal error - return empty to stop retrying
|
|
2765
3708
|
if (candidates.length === 0) {
|
|
2766
|
-
|
|
3709
|
+
LogStatusEx({
|
|
2767
3710
|
message: `❌ Context length exceeded and no models with larger context windows available. Current model: ${currentModel.Name} (${currentMaxTokens} max tokens). This is a fatal error.`,
|
|
2768
3711
|
category: 'AI',
|
|
2769
3712
|
additionalArgs: [{
|
|
@@ -2773,22 +3716,27 @@ class AIPromptRunner {
|
|
|
2773
3716
|
reason: 'No models with larger context windows available for failover'
|
|
2774
3717
|
}]
|
|
2775
3718
|
});
|
|
3719
|
+
// Return empty array - caller will see no candidates and stop retrying
|
|
2776
3720
|
return [];
|
|
2777
3721
|
}
|
|
3722
|
+
// Sort by priority first (existing algorithm), then by context window size as tiebreaker
|
|
2778
3723
|
candidates.sort((a, b) => {
|
|
3724
|
+
// Primary sort: priority (higher is better) - maintains existing algorithm
|
|
2779
3725
|
if (a.priority !== b.priority) {
|
|
2780
3726
|
return b.priority - a.priority;
|
|
2781
3727
|
}
|
|
3728
|
+
// Secondary sort: context window size (largest first) - only as tiebreaker
|
|
2782
3729
|
const aMaxTokens = a.model.ModelVendors?.length > 0 ?
|
|
2783
3730
|
Math.max(...a.model.ModelVendors.map((mv) => mv.MaxInputTokens || 0)) : 0;
|
|
2784
3731
|
const bMaxTokens = b.model.ModelVendors?.length > 0 ?
|
|
2785
3732
|
Math.max(...b.model.ModelVendors.map((mv) => mv.MaxInputTokens || 0)) : 0;
|
|
2786
3733
|
return bMaxTokens - aMaxTokens;
|
|
2787
3734
|
});
|
|
3735
|
+
// Log context-aware failover selection
|
|
2788
3736
|
const bestCandidate = candidates[0];
|
|
2789
3737
|
const bestCandidateMaxTokens = bestCandidate.model.ModelVendors?.length > 0 ?
|
|
2790
3738
|
Math.max(...bestCandidate.model.ModelVendors.map((mv) => mv.MaxInputTokens || 0)) : 0;
|
|
2791
|
-
|
|
3739
|
+
LogStatusEx({
|
|
2792
3740
|
message: `🔄 Context-aware failover: Selected model ${bestCandidate.model.Name} with ${bestCandidateMaxTokens} max input tokens (vs ${currentMaxTokens} for failed model)`,
|
|
2793
3741
|
category: 'AI',
|
|
2794
3742
|
additionalArgs: [{
|
|
@@ -2801,10 +3749,23 @@ class AIPromptRunner {
|
|
|
2801
3749
|
});
|
|
2802
3750
|
}
|
|
2803
3751
|
else {
|
|
3752
|
+
// Final sort by priority (higher is better) for non-context-length errors
|
|
2804
3753
|
candidates.sort((a, b) => b.priority - a.priority);
|
|
2805
3754
|
}
|
|
2806
3755
|
return candidates;
|
|
2807
3756
|
}
|
|
3757
|
+
/**
|
|
3758
|
+
* Logs a failover attempt for tracking and debugging.
|
|
3759
|
+
*
|
|
3760
|
+
* @param promptId - The ID of the prompt being executed
|
|
3761
|
+
* @param attempt - The failover attempt details
|
|
3762
|
+
* @param willRetry - Whether another attempt will be made
|
|
3763
|
+
*
|
|
3764
|
+
* @remarks
|
|
3765
|
+
* This method logs detailed information about each failover attempt to help with
|
|
3766
|
+
* debugging and monitoring. Override this method to implement custom logging or
|
|
3767
|
+
* integrate with external monitoring systems.
|
|
3768
|
+
*/
|
|
2808
3769
|
logFailoverAttempt(promptId, attempt, willRetry) {
|
|
2809
3770
|
const message = `Failover attempt ${attempt.attemptNumber} for prompt ${promptId}`;
|
|
2810
3771
|
const metadata = {
|
|
@@ -2818,14 +3779,14 @@ class AIPromptRunner {
|
|
|
2818
3779
|
error: attempt.error.message
|
|
2819
3780
|
};
|
|
2820
3781
|
if (willRetry) {
|
|
2821
|
-
|
|
3782
|
+
LogStatusEx({
|
|
2822
3783
|
message: `⚡ ${message}`,
|
|
2823
3784
|
category: 'AI',
|
|
2824
3785
|
additionalArgs: [metadata]
|
|
2825
3786
|
});
|
|
2826
3787
|
}
|
|
2827
3788
|
else {
|
|
2828
|
-
|
|
3789
|
+
LogErrorEx({
|
|
2829
3790
|
message: message,
|
|
2830
3791
|
error: attempt.error,
|
|
2831
3792
|
category: 'AI',
|
|
@@ -2835,8 +3796,4 @@ class AIPromptRunner {
|
|
|
2835
3796
|
}
|
|
2836
3797
|
}
|
|
2837
3798
|
}
|
|
2838
|
-
exports.AIPromptRunner = AIPromptRunner;
|
|
2839
|
-
function LoadAIPromptRunner() {
|
|
2840
|
-
}
|
|
2841
|
-
exports.LoadAIPromptRunner = LoadAIPromptRunner;
|
|
2842
3799
|
//# sourceMappingURL=AIPromptRunner.js.map
|