@ax-llm/ax 11.0.50 → 11.0.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.cjs CHANGED
@@ -172,6 +172,7 @@ __export(index_exports, {
172
172
  module.exports = __toCommonJS(index_exports);
173
173
 
174
174
  // ai/base.ts
175
+ var import_crypto2 = __toESM(require("crypto"), 1);
175
176
  var import_api2 = require("@opentelemetry/api");
176
177
 
177
178
  // trace/trace.ts
@@ -239,6 +240,7 @@ var AxSpanKindValues = /* @__PURE__ */ ((AxSpanKindValues2) => {
239
240
  })(AxSpanKindValues || {});
240
241
 
241
242
  // util/apicall.ts
243
+ var import_crypto = __toESM(require("crypto"), 1);
242
244
  var import_web3 = require("stream/web");
243
245
  var import_api = require("@opentelemetry/api");
244
246
 
@@ -379,7 +381,7 @@ var AxAIServiceError = class extends Error {
379
381
  this.responseBody = responseBody;
380
382
  this.name = this.constructor.name;
381
383
  this.timestamp = (/* @__PURE__ */ new Date()).toISOString();
382
- this.errorId = crypto.randomUUID();
384
+ this.errorId = import_crypto.default.randomUUID();
383
385
  this.context = context3;
384
386
  this.stack = this.toString();
385
387
  }
@@ -500,7 +502,7 @@ var apiCall = async (api, json) => {
500
502
  const baseUrl = new URL(process.env["PROXY"] ?? api.url);
501
503
  const apiPath = `${[baseUrl.pathname, api.name].filter(Boolean).join("/").replace(/\/+/g, "/")}${baseUrl.search}`;
502
504
  const apiUrl = new URL(apiPath, baseUrl);
503
- const requestId = crypto.randomUUID();
505
+ const requestId = import_crypto.default.randomUUID();
504
506
  if (api.validateRequest) {
505
507
  const isValid = await api.validateRequest(json);
506
508
  if (!isValid) {
@@ -883,13 +885,11 @@ var logResponseDelta = (delta) => {
883
885
 
884
886
  // ai/base.ts
885
887
  var axBaseAIDefaultConfig = () => structuredClone({
886
- maxTokens: 2e3,
887
888
  temperature: 0,
888
889
  topK: 40,
889
890
  topP: 0.9
890
891
  });
891
892
  var axBaseAIDefaultCreativeConfig = () => structuredClone({
892
- maxTokens: 2e3,
893
893
  temperature: 0.4,
894
894
  topP: 0.7,
895
895
  frequencyPenalty: 0.2
@@ -913,7 +913,7 @@ var AxBaseAI = class {
913
913
  this.tracer = options.tracer;
914
914
  this.modelInfo = modelInfo;
915
915
  this.models = models;
916
- this.id = crypto.randomUUID();
916
+ this.id = import_crypto2.default.randomUUID();
917
917
  const model = this.getModel(defaults.model) ?? defaults.model;
918
918
  const embedModel = this.getEmbedModel(defaults.embedModel) ?? defaults.embedModel;
919
919
  this.defaults = { model, embedModel };
@@ -1112,14 +1112,14 @@ var AxBaseAI = class {
1112
1112
  [axSpanAttributes.LLM_SYSTEM]: this.name,
1113
1113
  [axSpanAttributes.LLM_OPERATION_NAME]: "chat",
1114
1114
  [axSpanAttributes.LLM_REQUEST_MODEL]: model,
1115
- [axSpanAttributes.LLM_REQUEST_MAX_TOKENS]: modelConfig.maxTokens,
1115
+ [axSpanAttributes.LLM_REQUEST_MAX_TOKENS]: modelConfig.maxTokens ?? "Not set",
1116
1116
  [axSpanAttributes.LLM_REQUEST_TEMPERATURE]: modelConfig.temperature,
1117
- [axSpanAttributes.LLM_REQUEST_TOP_P]: modelConfig.topP,
1118
- [axSpanAttributes.LLM_REQUEST_TOP_K]: modelConfig.topK,
1119
- [axSpanAttributes.LLM_REQUEST_FREQUENCY_PENALTY]: modelConfig.frequencyPenalty,
1120
- [axSpanAttributes.LLM_REQUEST_PRESENCE_PENALTY]: modelConfig.presencePenalty,
1121
- [axSpanAttributes.LLM_REQUEST_STOP_SEQUENCES]: modelConfig.stopSequences?.join(", "),
1122
- [axSpanAttributes.LLM_REQUEST_LLM_IS_STREAMING]: modelConfig.stream
1117
+ [axSpanAttributes.LLM_REQUEST_TOP_P]: modelConfig.topP ?? "Not set",
1118
+ [axSpanAttributes.LLM_REQUEST_TOP_K]: modelConfig.topK ?? "Not set",
1119
+ [axSpanAttributes.LLM_REQUEST_FREQUENCY_PENALTY]: modelConfig.frequencyPenalty ?? "Not set",
1120
+ [axSpanAttributes.LLM_REQUEST_PRESENCE_PENALTY]: modelConfig.presencePenalty ?? "Not set",
1121
+ [axSpanAttributes.LLM_REQUEST_STOP_SEQUENCES]: modelConfig.stopSequences?.join(", ") ?? "Not set",
1122
+ [axSpanAttributes.LLM_REQUEST_LLM_IS_STREAMING]: modelConfig.stream ?? "Not set"
1123
1123
  }
1124
1124
  },
1125
1125
  options?.traceContext ?? import_api2.context.active(),
@@ -2218,7 +2218,7 @@ var AxAIOpenAIImpl = class {
2218
2218
  response_format: this.config?.responseFormat ? { type: this.config.responseFormat } : void 0,
2219
2219
  tools,
2220
2220
  tool_choice: toolsChoice,
2221
- max_completion_tokens: req.modelConfig?.maxTokens ?? this.config.maxTokens ?? 500,
2221
+ max_completion_tokens: req.modelConfig?.maxTokens ?? this.config.maxTokens,
2222
2222
  temperature: req.modelConfig?.temperature ?? this.config.temperature,
2223
2223
  top_p: req.modelConfig?.topP ?? this.config.topP ?? 1,
2224
2224
  n: req.modelConfig?.n ?? this.config.n,
@@ -2262,6 +2262,9 @@ var AxAIOpenAIImpl = class {
2262
2262
  }
2263
2263
  if (config.thinkingTokenBudget) {
2264
2264
  switch (config.thinkingTokenBudget) {
2265
+ case "none":
2266
+ reqValue.reasoning_effort = void 0;
2267
+ break;
2265
2268
  case "minimal":
2266
2269
  reqValue.reasoning_effort = "low";
2267
2270
  break;
@@ -3323,6 +3326,9 @@ var AxAIGoogleGeminiImpl = class {
3323
3326
  }
3324
3327
  if (config.thinkingTokenBudget) {
3325
3328
  switch (config.thinkingTokenBudget) {
3329
+ case "none":
3330
+ thinkingConfig.thinkingBudget = 0;
3331
+ break;
3326
3332
  case "minimal":
3327
3333
  thinkingConfig.thinkingBudget = 200;
3328
3334
  break;
@@ -4768,7 +4774,6 @@ var axAIOpenAIResponsesDefaultConfig = () => ({
4768
4774
  model: "gpt-4o" /* GPT4O */,
4769
4775
  embedModel: "text-embedding-ada-002" /* TextEmbeddingAda002 */,
4770
4776
  temperature: 0.7,
4771
- maxTokens: 2048,
4772
4777
  topP: 1,
4773
4778
  stream: true
4774
4779
  // reasoningEffort: 'medium',
@@ -4935,7 +4940,7 @@ var AxAIRekaImpl = class {
4935
4940
  const reqValue = {
4936
4941
  model,
4937
4942
  messages,
4938
- max_tokens: req.modelConfig?.maxTokens ?? this.config.maxTokens ?? 500,
4943
+ max_tokens: req.modelConfig?.maxTokens ?? this.config.maxTokens,
4939
4944
  temperature: req.modelConfig?.temperature ?? this.config.temperature,
4940
4945
  top_k: req.modelConfig?.n ?? this.config.n,
4941
4946
  top_p: req.modelConfig?.topP ?? this.config.topP ?? 1,
@@ -5914,16 +5919,20 @@ var AxPromptTemplate = class {
5914
5919
  sig;
5915
5920
  fieldTemplates;
5916
5921
  task;
5917
- constructor(sig, functions, fieldTemplates) {
5922
+ thoughtFieldName;
5923
+ functions;
5924
+ constructor(sig, options, fieldTemplates) {
5918
5925
  this.sig = sig;
5919
5926
  this.fieldTemplates = fieldTemplates;
5927
+ this.thoughtFieldName = options?.thoughtFieldName ?? "thought";
5928
+ this.functions = options?.functions;
5920
5929
  const task = [];
5921
5930
  const inArgs = renderDescFields(this.sig.getInputFields());
5922
5931
  const outArgs = renderDescFields(this.sig.getOutputFields());
5923
5932
  task.push(
5924
5933
  `You will be provided with the following fields: ${inArgs}. Your task is to generate new fields: ${outArgs}.`
5925
5934
  );
5926
- const funcs = functions?.map((f) => "toFunction" in f ? f.toFunction() : f)?.flat();
5935
+ const funcs = this.functions?.map((f) => "toFunction" in f ? f.toFunction() : f)?.flat();
5927
5936
  const funcList = funcs?.map((fn) => `- \`${fn.name}\`: ${formatDescription(fn.description)}`).join("\n");
5928
5937
  if (funcList && funcList.length > 0) {
5929
5938
  task.push(`## Available Functions
@@ -5958,7 +5967,6 @@ ${outputFields}`);
5958
5967
  ...this.renderExamples(examples)
5959
5968
  ] : [];
5960
5969
  const renderedDemos = demos ? this.renderDemos(demos) : [];
5961
- const completion = this.renderInputFields(values);
5962
5970
  const allTextExamples = renderedExamples.every((v) => v.type === "text");
5963
5971
  const allTextDemos = renderedDemos.every((v) => v.type === "text");
5964
5972
  const examplesInSystemPrompt = allTextExamples && allTextDemos;
@@ -5978,14 +5986,73 @@ ${outputFields}`);
5978
5986
  role: "system",
5979
5987
  content: systemContent
5980
5988
  };
5981
- const promptList = examplesInSystemPrompt ? completion : [...renderedExamples, ...renderedDemos, ...completion];
5982
- const prompt = promptList.filter((v) => v !== void 0);
5983
- const userContent = prompt.every((v) => v.type === "text") ? prompt.map((v) => v.text).join("\n") : prompt.reduce(combineConsecutiveStrings("\n"), []);
5984
- const userPrompt = {
5985
- role: "user",
5986
- content: userContent
5987
- };
5988
- return [systemPrompt, userPrompt];
5989
+ let userMessages = [];
5990
+ if (Array.isArray(values)) {
5991
+ const history = values;
5992
+ let lastRole = void 0;
5993
+ for (const message of history) {
5994
+ let messageContent = "";
5995
+ if (message.role === "user") {
5996
+ const userMsgParts = this.renderInputFields(
5997
+ message.values
5998
+ // Cast message.values (AxGenIn) to T (which extends AxGenIn)
5999
+ );
6000
+ messageContent = userMsgParts.map((part) => part.type === "text" ? part.text : "").join("").trim();
6001
+ } else if (message.role === "assistant") {
6002
+ const assistantValues = message.values;
6003
+ let assistantContentParts = [];
6004
+ const outputFields = this.sig.getOutputFields();
6005
+ for (const field of outputFields) {
6006
+ const value = assistantValues[field.name];
6007
+ if (value !== void 0 && value !== null && (typeof value === "string" ? value !== "" : true)) {
6008
+ const renderedValue = processValue(field, value);
6009
+ assistantContentParts.push(`${field.name}: ${renderedValue}`);
6010
+ } else {
6011
+ const isThoughtField = field.name === this.thoughtFieldName;
6012
+ if (!field.isOptional && !field.isInternal && !isThoughtField) {
6013
+ throw new Error(
6014
+ `Value for output field '${field.name}' ('${field.title}') is required in assistant message history but was not found or was empty.`
6015
+ );
6016
+ }
6017
+ }
6018
+ }
6019
+ messageContent = assistantContentParts.join("\n");
6020
+ }
6021
+ if (messageContent) {
6022
+ if (lastRole === message.role && userMessages.length > 0) {
6023
+ const lastMessage = userMessages[userMessages.length - 1];
6024
+ if (lastMessage) {
6025
+ lastMessage.content += "\n" + messageContent;
6026
+ }
6027
+ } else {
6028
+ if (message.role === "user") {
6029
+ userMessages.push({ role: "user", content: messageContent });
6030
+ } else if (message.role === "assistant") {
6031
+ userMessages.push({ role: "assistant", content: messageContent });
6032
+ }
6033
+ }
6034
+ lastRole = message.role;
6035
+ }
6036
+ }
6037
+ } else {
6038
+ const currentValues = values;
6039
+ const completion = this.renderInputFields(currentValues);
6040
+ const promptList = examplesInSystemPrompt ? completion : [...renderedExamples, ...renderedDemos, ...completion];
6041
+ const promptFilter = promptList.filter((v) => v !== void 0);
6042
+ let userContent;
6043
+ if (promptFilter.every((v) => v.type === "text")) {
6044
+ userContent = promptFilter.map((v) => v.text).join("\n");
6045
+ } else {
6046
+ userContent = promptFilter.map((part) => {
6047
+ if (part.type === "text") return part.text;
6048
+ if (part.type === "image") return "[IMAGE]";
6049
+ if (part.type === "audio") return "[AUDIO]";
6050
+ return "";
6051
+ }).join("\n").trim();
6052
+ }
6053
+ userMessages.push({ role: "user", content: userContent });
6054
+ }
6055
+ return [systemPrompt, ...userMessages];
5989
6056
  };
5990
6057
  renderExtraFields = (extraFields) => {
5991
6058
  const prompt = [];
@@ -6029,8 +6096,8 @@ ${outputFields}`);
6029
6096
  renderExamples = (data) => {
6030
6097
  const list = [];
6031
6098
  for (const [index, item] of data.entries()) {
6032
- const renderedInputItem = this.sig.getInputFields().map((field) => this.renderInField(field, item, true)).filter((v) => v !== void 0).flat();
6033
- const renderedOutputItem = this.sig.getOutputFields().map((field) => this.renderInField(field, item, true)).filter((v) => v !== void 0).flat();
6099
+ const renderedInputItem = this.sig.getInputFields().map((field) => this.renderInField(field, item)).filter((v) => v !== void 0).flat();
6100
+ const renderedOutputItem = this.sig.getOutputFields().map((field) => this.renderInField(field, item)).filter((v) => v !== void 0).flat();
6034
6101
  if (renderedOutputItem.length === 0) {
6035
6102
  throw new Error(
6036
6103
  `Output fields are required in examples: index: ${index}, data: ${JSON.stringify(item)}`
@@ -6056,7 +6123,7 @@ ${outputFields}`);
6056
6123
  const list = [];
6057
6124
  const fields = [...this.sig.getInputFields(), ...this.sig.getOutputFields()];
6058
6125
  for (const item of data) {
6059
- const renderedItem = fields.map((field) => this.renderInField(field, item, true)).filter((v) => v !== void 0).flat();
6126
+ const renderedItem = fields.map((field) => this.renderInField(field, item)).filter((v) => v !== void 0).flat();
6060
6127
  renderedItem.slice(0, -1).forEach((v) => {
6061
6128
  if ("text" in v) {
6062
6129
  v.text = v.text + "\n";
@@ -6076,11 +6143,8 @@ ${outputFields}`);
6076
6143
  });
6077
6144
  return renderedItems;
6078
6145
  };
6079
- renderInField = (field, values, skipMissing) => {
6146
+ renderInField = (field, values) => {
6080
6147
  const value = values[field.name];
6081
- if (skipMissing && !value) {
6082
- return;
6083
- }
6084
6148
  if (isEmptyValue(field, value)) {
6085
6149
  return;
6086
6150
  }
@@ -6117,20 +6181,20 @@ ${outputFields}`);
6117
6181
  }
6118
6182
  result = result.concat(
6119
6183
  value.map((v) => {
6120
- v = validateImage(v);
6184
+ const validated = validateImage(v);
6121
6185
  return {
6122
6186
  type: "image",
6123
- mimeType: v.mimeType,
6124
- image: v.data
6187
+ mimeType: validated.mimeType,
6188
+ image: validated.data
6125
6189
  };
6126
6190
  })
6127
6191
  );
6128
6192
  } else {
6129
- const v = validateImage(value);
6193
+ const validated = validateImage(value);
6130
6194
  result.push({
6131
6195
  type: "image",
6132
- mimeType: v.mimeType,
6133
- image: v.data
6196
+ mimeType: validated.mimeType,
6197
+ image: validated.data
6134
6198
  });
6135
6199
  }
6136
6200
  return result;
@@ -6153,24 +6217,24 @@ ${outputFields}`);
6153
6217
  ];
6154
6218
  if (field.type.isArray) {
6155
6219
  if (!Array.isArray(value)) {
6156
- throw new Error("Image field value must be an array.");
6220
+ throw new Error("Audio field value must be an array.");
6157
6221
  }
6158
6222
  result = result.concat(
6159
6223
  value.map((v) => {
6160
- v = validateAudio(v);
6224
+ const validated = validateAudio(v);
6161
6225
  return {
6162
6226
  type: "audio",
6163
- format: v.format ?? "wav",
6164
- data: v.data
6227
+ format: validated.format ?? "wav",
6228
+ data: validated.data
6165
6229
  };
6166
6230
  })
6167
6231
  );
6168
6232
  } else {
6169
- const v = validateAudio(value);
6233
+ const validated = validateAudio(value);
6170
6234
  result.push({
6171
6235
  type: "audio",
6172
- format: v.format ?? "wav",
6173
- data: v.data
6236
+ format: validated.format ?? "wav",
6237
+ data: validated.data
6174
6238
  });
6175
6239
  }
6176
6240
  return result;
@@ -6270,7 +6334,7 @@ var isEmptyValue = (field, value) => {
6270
6334
  return false;
6271
6335
  }
6272
6336
  if (!value || (Array.isArray(value) || typeof value === "string") && value.length === 0) {
6273
- if (field.isOptional) {
6337
+ if (field.isOptional || field.isInternal) {
6274
6338
  return true;
6275
6339
  }
6276
6340
  throw new Error(`Value for input field '${field.name}' is required.`);
@@ -7010,7 +7074,7 @@ var AxInstanceRegistry = class {
7010
7074
  };
7011
7075
 
7012
7076
  // dsp/sig.ts
7013
- var import_crypto = require("crypto");
7077
+ var import_crypto3 = require("crypto");
7014
7078
 
7015
7079
  // dsp/parser.ts
7016
7080
  var SignatureParser = class {
@@ -7452,7 +7516,7 @@ var AxSignature = class _AxSignature {
7452
7516
  throw new Error("Image type is not supported in output fields.");
7453
7517
  }
7454
7518
  });
7455
- this.sigHash = (0, import_crypto.createHash)("sha256").update(this.description ?? "").update(JSON.stringify(this.inputFields)).update(JSON.stringify(this.outputFields)).digest("hex");
7519
+ this.sigHash = (0, import_crypto3.createHash)("sha256").update(this.description ?? "").update(JSON.stringify(this.inputFields)).update(JSON.stringify(this.outputFields)).digest("hex");
7456
7520
  this.sigString = renderSignature(
7457
7521
  this.description,
7458
7522
  this.inputFields,
@@ -7724,12 +7788,18 @@ var AxGen = class extends AxProgramWithSignature {
7724
7788
  streamingFieldProcessors = [];
7725
7789
  values = {};
7726
7790
  excludeContentFromTrace = false;
7791
+ thoughtFieldName;
7727
7792
  constructor(signature, options) {
7728
7793
  super(signature, { description: options?.description });
7729
7794
  this.options = options;
7795
+ this.thoughtFieldName = options?.thoughtFieldName ?? "thought";
7796
+ const promptTemplateOptions = {
7797
+ functions: options?.functions,
7798
+ thoughtFieldName: this.thoughtFieldName
7799
+ };
7730
7800
  this.promptTemplate = new (options?.promptTemplate ?? AxPromptTemplate)(
7731
7801
  this.signature,
7732
- options?.functions
7802
+ promptTemplateOptions
7733
7803
  );
7734
7804
  this.asserts = this.options?.asserts ?? [];
7735
7805
  this.streamingAsserts = this.options?.streamingAsserts ?? [];
@@ -7901,7 +7971,9 @@ var AxGen = class extends AxProgramWithSignature {
7901
7971
  );
7902
7972
  } else if (result.content && result.content.length > 0) {
7903
7973
  if (result.thought && result.thought.length > 0) {
7904
- yield { thought: result.thought };
7974
+ yield {
7975
+ [this.thoughtFieldName]: result.thought
7976
+ };
7905
7977
  }
7906
7978
  content += result.content;
7907
7979
  mem.updateResult(
@@ -7943,8 +8015,10 @@ var AxGen = class extends AxProgramWithSignature {
7943
8015
  );
7944
8016
  await assertAssertions(this.asserts, this.values);
7945
8017
  } else if (result.thought && result.thought.length > 0) {
7946
- this.values.thought = this.values.thought ?? "" + result.thought;
7947
- yield { thought: result.thought };
8018
+ this.values[this.thoughtFieldName] = (this.values[this.thoughtFieldName] ?? "") + result.thought;
8019
+ yield {
8020
+ [this.thoughtFieldName]: result.thought
8021
+ };
7948
8022
  }
7949
8023
  if (result.finishReason === "length") {
7950
8024
  throw new Error(
@@ -8044,7 +8118,7 @@ Content: ${content}`
8044
8118
  }
8045
8119
  } else if (result.content) {
8046
8120
  if (result.thought && result.thought.length > 0) {
8047
- this.values.thought = result.thought;
8121
+ this.values[this.thoughtFieldName] = result.thought;
8048
8122
  }
8049
8123
  extractValues(this.signature, this.values, result.content);
8050
8124
  await assertAssertions(this.asserts, this.values);
@@ -8081,16 +8155,29 @@ Content: ${result.content}`
8081
8155
  const mem = options.mem ?? this.options?.mem ?? new AxMemory(1e4, memOptions);
8082
8156
  let err;
8083
8157
  if (options?.functions && options.functions.length > 0) {
8084
- const promptTemplate = this.options?.promptTemplate ?? AxPromptTemplate;
8085
- this.promptTemplate = new promptTemplate(
8158
+ const promptTemplateClass = this.options?.promptTemplate ?? AxPromptTemplate;
8159
+ const currentPromptTemplateOptions = {
8160
+ functions: options.functions,
8161
+ thoughtFieldName: this.thoughtFieldName
8162
+ };
8163
+ this.promptTemplate = new promptTemplateClass(
8086
8164
  this.signature,
8087
- options.functions
8165
+ currentPromptTemplateOptions
8088
8166
  );
8089
8167
  }
8090
- const prompt = this.promptTemplate.render(values, {
8091
- examples: this.examples,
8092
- demos: this.demos
8093
- });
8168
+ let prompt;
8169
+ if (Array.isArray(values)) {
8170
+ prompt = this.promptTemplate.render(values, {
8171
+ examples: this.examples,
8172
+ demos: this.demos
8173
+ });
8174
+ } else {
8175
+ prompt = this.promptTemplate.render(values, {
8176
+ // Cast if necessary
8177
+ examples: this.examples,
8178
+ demos: this.demos
8179
+ });
8180
+ }
8094
8181
  mem.add(prompt, options?.sessionId);
8095
8182
  multiStepLoop: for (let n = 0; n < maxSteps; n++) {
8096
8183
  const firstStep = n === 0;
@@ -8952,7 +9039,8 @@ var AxBootstrapFewShot = class {
8952
9039
  if (this.costMonitoring) {
8953
9040
  this.stats.estimatedTokenUsage += JSON.stringify(ex).length / 4 + JSON.stringify(res).length / 4;
8954
9041
  }
8955
- const success = metricFn({ prediction: res, example: ex });
9042
+ const score = metricFn({ prediction: res, example: ex });
9043
+ const success = score >= 0.5;
8956
9044
  if (success) {
8957
9045
  this.traces = [...this.traces, ...this.program.getTraces()];
8958
9046
  this.stats.successfulDemos++;
@@ -10940,19 +11028,20 @@ ${dataContext}
10940
11028
  }
10941
11029
  evalSet = minibatchEvalSet;
10942
11030
  }
10943
- let correctCount = 0;
11031
+ let sumOfScores = 0;
10944
11032
  for (const example of evalSet) {
10945
11033
  try {
10946
11034
  const prediction = await this.program.forward(this.ai, example);
10947
- const correct = metricFn({ prediction, example });
10948
- if (correct) correctCount++;
11035
+ const score = metricFn({ prediction, example });
11036
+ sumOfScores += score;
10949
11037
  } catch (err) {
10950
11038
  if (this.verbose) {
10951
11039
  console.error("Error evaluating example:", err);
10952
11040
  }
10953
11041
  }
10954
11042
  }
10955
- return correctCount / evalSet.length;
11043
+ if (evalSet.length === 0) return 0;
11044
+ return sumOfScores / evalSet.length;
10956
11045
  }
10957
11046
  /**
10958
11047
  * Run full evaluation on the entire validation set
@@ -10964,19 +11053,20 @@ ${dataContext}
10964
11053
  bootstrappedDemos,
10965
11054
  labeledExamples
10966
11055
  );
10967
- let fullCorrectCount = 0;
11056
+ let sumOfScores = 0;
10968
11057
  for (const example of valset) {
10969
11058
  try {
10970
11059
  const prediction = await this.program.forward(this.ai, example);
10971
- const correct = metricFn({ prediction, example });
10972
- if (correct) fullCorrectCount++;
11060
+ const score = metricFn({ prediction, example });
11061
+ sumOfScores += score;
10973
11062
  } catch (err) {
10974
11063
  if (this.verbose) {
10975
11064
  console.error("Error evaluating example:", err);
10976
11065
  }
10977
11066
  }
10978
11067
  }
10979
- return fullCorrectCount / valset.length;
11068
+ if (valset.length === 0) return 0;
11069
+ return sumOfScores / valset.length;
10980
11070
  }
10981
11071
  /**
10982
11072
  * Implements a Bayesian-inspired selection of the next configuration to try
@@ -11148,10 +11238,11 @@ ${dataContext}
11148
11238
  };
11149
11239
 
11150
11240
  // ai/mock/api.ts
11241
+ var import_crypto4 = __toESM(require("crypto"), 1);
11151
11242
  var AxMockAIService = class {
11152
11243
  constructor(config = {}) {
11153
11244
  this.config = config;
11154
- this.config.id = this.config.id ?? crypto.randomUUID();
11245
+ this.config.id = this.config.id ?? import_crypto4.default.randomUUID();
11155
11246
  }
11156
11247
  metrics = {
11157
11248
  latency: {
@@ -11362,27 +11453,26 @@ var AxTestPrompt = class {
11362
11453
  async run(metricFn) {
11363
11454
  const st = (/* @__PURE__ */ new Date()).getTime();
11364
11455
  const total = this.examples.length;
11365
- let successCount = 0;
11456
+ let sumOfScores = 0;
11366
11457
  for (let i = 0; i < total; i++) {
11367
11458
  const ex = this.examples[i];
11368
11459
  if (!ex) {
11369
11460
  throw new Error("Invalid example");
11370
11461
  }
11371
11462
  const res = await this.program.forward(this.ai, ex);
11372
- const success = metricFn({ prediction: res, example: ex });
11373
- if (success) {
11374
- successCount++;
11375
- }
11463
+ const score = metricFn({ prediction: res, example: ex });
11464
+ sumOfScores += score;
11376
11465
  const et = (/* @__PURE__ */ new Date()).getTime() - st;
11377
- updateProgressBar(i, total, successCount, et, "Testing Prompt", 30);
11466
+ updateProgressBar(i, total, sumOfScores, et, "Testing Prompt", 30);
11378
11467
  }
11468
+ const averageScore = total > 0 ? sumOfScores / total : 0;
11379
11469
  console.log(
11380
11470
  "\nPerformance: ",
11381
- successCount,
11471
+ sumOfScores,
11382
11472
  "/",
11383
11473
  total,
11384
- "Accuracy: ",
11385
- successCount / total,
11474
+ "Average Score: ",
11475
+ averageScore,
11386
11476
  "\n"
11387
11477
  );
11388
11478
  }
@@ -11396,7 +11486,8 @@ var AxChainOfThought = class extends AxGen {
11396
11486
  sig.setOutputFields([
11397
11487
  {
11398
11488
  name: "reason",
11399
- description
11489
+ description,
11490
+ isInternal: options?.setVisibleReasoning !== true
11400
11491
  },
11401
11492
  ...sig.getOutputFields()
11402
11493
  ]);
@@ -12725,7 +12816,7 @@ function normalizeText(s) {
12725
12816
  return s.toLowerCase();
12726
12817
  }
12727
12818
  function emScore(prediction, groundTruth) {
12728
- return normalizeText(prediction) === normalizeText(groundTruth);
12819
+ return normalizeText(prediction) === normalizeText(groundTruth) ? 1 : 0;
12729
12820
  }
12730
12821
  function f1Score(prediction, groundTruth) {
12731
12822
  const predictionTokens = normalizeText(prediction).split(" ");
@@ -12765,12 +12856,12 @@ var AxEvalUtil = {
12765
12856
  };
12766
12857
 
12767
12858
  // ../../node_modules/uuid/dist/esm-node/rng.js
12768
- var import_crypto2 = __toESM(require("crypto"));
12859
+ var import_crypto5 = __toESM(require("crypto"));
12769
12860
  var rnds8Pool = new Uint8Array(256);
12770
12861
  var poolPtr = rnds8Pool.length;
12771
12862
  function rng() {
12772
12863
  if (poolPtr > rnds8Pool.length - 16) {
12773
- import_crypto2.default.randomFillSync(rnds8Pool);
12864
+ import_crypto5.default.randomFillSync(rnds8Pool);
12774
12865
  poolPtr = 0;
12775
12866
  }
12776
12867
  return rnds8Pool.slice(poolPtr, poolPtr += 16);
@@ -12786,9 +12877,9 @@ function unsafeStringify(arr, offset = 0) {
12786
12877
  }
12787
12878
 
12788
12879
  // ../../node_modules/uuid/dist/esm-node/native.js
12789
- var import_crypto3 = __toESM(require("crypto"));
12880
+ var import_crypto6 = __toESM(require("crypto"));
12790
12881
  var native_default = {
12791
- randomUUID: import_crypto3.default.randomUUID
12882
+ randomUUID: import_crypto6.default.randomUUID
12792
12883
  };
12793
12884
 
12794
12885
  // ../../node_modules/uuid/dist/esm-node/v4.js