@botpress/vai 0.0.1-beta.1

package/dist/index.js

@@ -0,0 +1,476 @@
+var __defProp = Object.defineProperty;
+var __defProps = Object.defineProperties;
+var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __typeError = (msg) => {
+  throw TypeError(msg);
+};
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
+var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
+var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
+var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), setter ? setter.call(obj, value) : member.set(obj, value), value);
+
+// src/task/compare.ts
+import { z } from "@botpress/sdk";
+import { createTaskCollector, getCurrentSuite } from "vitest/suite";
+
+// src/utils/deferred.ts
+var _Deferred = class _Deferred {
+  constructor() {
+    this.promise = new Promise((resolve, reject) => {
+      this._resolve = resolve;
+      this._reject = reject;
+    });
+  }
+  resolve(value) {
+    this._resolve(value);
+  }
+  reject(reason) {
+    this._reject(reason);
+  }
+};
+__name(_Deferred, "Deferred");
+var Deferred = _Deferred;
+
+// src/task/compare.ts
+var scenarioId = z.string().trim().min(1, "Scenario ID/name must not be empty").max(50, "Scenario ID/name is too long");
+var ScenarioLike = z.union([
+  scenarioId,
+  z.object({ name: scenarioId }).passthrough(),
+  z.object({ id: scenarioId }).passthrough()
+]);
+var getScenarioName = /* @__PURE__ */ __name((scenario) => typeof scenario === "string" ? scenario : "name" in scenario ? scenario == null ? void 0 : scenario.name : scenario == null ? void 0 : scenario.id, "getScenarioName");
+var scenarioArgs = z.array(ScenarioLike).min(2, "You need at least two scenarios to compare").max(10, "You can only compare up to 10 scenarios").refine((scenarios) => {
+  const set = /* @__PURE__ */ new Set();
+  scenarios.forEach((scenario) => set.add(getScenarioName(scenario)));
+  return set.size === scenarios.length;
+}, "Scenarios names must be unique");
+function compare(name, scenarios, fn) {
+  scenarios = scenarioArgs.parse(scenarios);
+  return createTaskCollector((_name, fn2, timeout) => {
+    const currentSuite = getCurrentSuite();
+    let completedCount = 0;
+    const finished = new Deferred();
+    for (const scenario of scenarios) {
+      const key = getScenarioName(scenario);
+      currentSuite.task(key, {
+        meta: {
+          scenario: key,
+          isVaiTest: true
+        },
+        handler: /* @__PURE__ */ __name(async (context) => {
+          const extendedContext = Object.freeze({
+            scenario
+          });
+          context.onTestFinished(() => {
+            if (++completedCount === scenarios.length) {
+              finished.resolve();
+            }
+          });
+          await fn2(__spreadValues(__spreadValues({}, context), extendedContext));
+        }, "handler"),
+        timeout: timeout != null ? timeout : 1e4
+      });
+    }
+  })(name, fn);
+}
+__name(compare, "compare");
+
+// src/assertions/check.ts
+import { z as z3 } from "@botpress/sdk";
+
+// src/context.ts
+import { onTestFinished } from "vitest";
+import { getCurrentTest } from "vitest/suite";
+var getTestMetadata = /* @__PURE__ */ __name(() => {
+  var _a;
+  const test = getCurrentTest();
+  return (_a = test == null ? void 0 : test.meta) != null ? _a : {
+    isVaiTest: false
+  };
+}, "getTestMetadata");
+var _client, _wrapError;
+var _VaiContext = class _VaiContext {
+  constructor() {
+    __privateAdd(this, _client, null);
+    __privateAdd(this, _wrapError, false);
+  }
+  get wrapError() {
+    return __privateGet(this, _wrapError);
+  }
+  get client() {
+    if (!__privateGet(this, _client)) {
+      throw new Error("Botpress client is not set");
+    }
+    return __privateGet(this, _client);
+  }
+  get evaluatorModel() {
+    var _a;
+    return (_a = getTestMetadata().evaluatorModel) != null ? _a : "openai__gpt-4o-mini-2024-07-18";
+  }
+  get scenario() {
+    return getTestMetadata().scenario;
+  }
+  get isVaiTest() {
+    return getTestMetadata().isVaiTest;
+  }
+  setClient(cognitive) {
+    __privateSet(this, _client, cognitive);
+  }
+  swallowErrors() {
+    if (!getCurrentTest()) {
+      throw new Error("cancelBail is a Vitest hook and must be called within a test");
+    }
+    __privateSet(this, _wrapError, true);
+    onTestFinished(() => {
+      __privateSet(this, _wrapError, false);
+    });
+  }
+};
+_client = new WeakMap();
+_wrapError = new WeakMap();
+__name(_VaiContext, "VaiContext");
+var VaiContext = _VaiContext;
+var Context = new VaiContext();
+
+// src/utils/asyncAssertion.ts
+import { expect } from "vitest";
+import { getCurrentTest as getCurrentTest2 } from "vitest/suite";
+var _AsyncExpectError = class _AsyncExpectError extends Error {
+  constructor(message, output) {
+    super(message);
+    this.output = output;
+    this.name = "AsyncExpectError";
+  }
+};
+__name(_AsyncExpectError, "AsyncExpectError");
+var AsyncExpectError = _AsyncExpectError;
+var getErrorMessages = /* @__PURE__ */ __name((e) => {
+  if (e instanceof Error) {
+    return e.message;
+  } else if (typeof e === "string") {
+    return e;
+  } else if (typeof e === "object" && e !== null) {
+    return JSON.stringify(e);
+  }
+  return `Unknown error: ${e}`;
+}, "getErrorMessages");
+var asyncExpect = /* @__PURE__ */ __name((output, assertion) => {
+  var _a, _b;
+  const promise = output.then((x) => {
+    try {
+      assertion(expect(x.result, x.reason));
+    } catch (e) {
+      if (Context.wrapError) {
+        return new AsyncExpectError(getErrorMessages(e), x);
+      }
+      throw e;
+    }
+    return x;
+  });
+  (_b = (_a = getCurrentTest2()).promises) != null ? _b : _a.promises = [];
+  getCurrentTest2().promises.push(promise);
+  return promise;
+}, "asyncExpect");
+
+// src/utils/predictJson.ts
+import { z as z2, ZodSchema } from "@botpress/sdk";
+import JSON5 from "json5";
+var nonEmptyString = z2.string().trim().min(1);
+var nonEmptyObject = z2.object({}).passthrough().refine((value) => Object.keys(value).length > 0, {
+  message: "Expected a non-empty object"
+});
+var Input = nonEmptyString.or(nonEmptyObject).or(z2.array(z2.any()));
+var Output = z2.object({
+  reason: nonEmptyString.describe("A human-readable explanation of the result"),
+  result: z2.any().describe(
+    "Your best guess at the output according to the instructions provided, rooted in the context of the input and the reason above"
+  )
+});
+var Example = z2.object({
+  input: Input,
+  output: Output
+});
+var Options = z2.object({
+  systemMessage: z2.string(),
+  examples: z2.array(Example).default([]),
+  input: Input,
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  outputSchema: z2.custom((value) => value instanceof ZodSchema),
+  model: z2.string()
+});
+var isValidExample = /* @__PURE__ */ __name((outputSchema) => (example) => Input.safeParse(example.input).success && Output.safeParse(example.output).success && outputSchema.safeParse(example.output.result).success, "isValidExample");
+async function predictJson(_options) {
+  var _a, _b;
+  const options = Options.parse(_options);
+  const [integration, model] = options.model.split("__");
+  if (!(model == null ? void 0 : model.length)) {
+    throw new Error("Invalid model");
+  }
+  const exampleMessages = options.examples.filter(isValidExample(options.outputSchema)).flatMap(({ input, output: output2 }) => [
+    { role: "user", content: JSON.stringify(input, null, 2) },
+    { role: "assistant", content: JSON.stringify(output2, null, 2) }
+  ]);
+  const outputSchema = Output.extend({
+    result: options.outputSchema.describe(Output.shape.result.description)
+  });
+  const result = await Context.client.callAction({
+    type: `${integration}:generateContent`,
+    input: {
+      systemPrompt: `
+${options.systemMessage}
+
+---
+Please generate a JSON response with the following format:
+\`\`\`typescript
+${await outputSchema.toTypescriptAsync()}
+\`\`\`
+`.trim(),
+      messages: [
+        ...exampleMessages,
+        {
+          role: "user",
+          content: JSON.stringify(options.input, null, 2)
+        }
+      ],
+      temperature: 0,
+      responseFormat: "json_object",
+      model: { id: model }
+    }
+  });
+  const output = result.output;
+  if (!output.choices.length || typeof ((_b = (_a = output.choices) == null ? void 0 : _a[0]) == null ? void 0 : _b.content) !== "string") {
+    throw new Error("Invalid response from the model");
+  }
+  const json = output.choices[0].content.trim();
+  if (!json.length) {
+    throw new Error("No response from the model");
+  }
+  return outputSchema.parse(JSON5.parse(json));
+}
+__name(predictJson, "predictJson");
+
+// src/assertions/extension.ts
+import json5 from "json5";
+import { expect as expect2 } from "vitest";
+import { getCurrentTest as getCurrentTest3 } from "vitest/suite";
+var toAssertion = /* @__PURE__ */ __name((promise) => {
+  return {
+    then: promise.then.bind(promise),
+    value: promise.then((value) => value.result)
+  };
+}, "toAssertion");
+var makeToMatchInlineSnapshot = /* @__PURE__ */ __name((promise) => async (expected) => {
+  const stack = new Error().stack.split("\n")[2];
+  const newStack = `
+at __INLINE_SNAPSHOT__ (node:internal/process/task_queues:1:1)
+at randomLine (node:internal/process/task_queues:1:1)
+${stack}
+`.trim();
+  const obj = json5.parse(expected != null ? expected : '""');
+  const expectation = asyncExpect(promise, (expect3) => expect3.toMatchObject(obj)).catch(() => {
+  });
+  try {
+    expect2((await promise).result).toMatchObject(obj);
+  } catch (err) {
+    const newError = new Error();
+    newError.stack = newStack;
+    expect2.getState().snapshotState.match({
+      isInline: true,
+      received: (await promise).result,
+      testName: getCurrentTest3().name,
+      error: newError,
+      inlineSnapshot: expected
+    });
+  }
+  return expectation;
+}, "makeToMatchInlineSnapshot");
+
+// src/assertions/check.ts
+function check(value, condition, options) {
+  var _a;
+  const promise = predictJson({
+    systemMessage: `Check that the value meets the condition: ${condition}`,
+    examples: (_a = options == null ? void 0 : options.examples) == null ? void 0 : _a.map(({ value: value2, reason, expected }) => ({
+      input: value2,
+      output: { reason, result: expected }
+    })),
+    outputSchema: z3.boolean(),
+    model: Context.evaluatorModel,
+    input: value
+  });
+  return __spreadProps(__spreadValues({}, toAssertion(promise)), {
+    toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toEqual(expected)), "toBe"),
+    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise)
+  });
+}
+__name(check, "check");
+
+// src/assertions/extract.ts
+function extract(value, shape, options) {
+  var _a;
+  const additionalMessage = (options == null ? void 0 : options.description) ? `
+In order to extract the right information, follow these instructions:
+${options.description}
+` : "";
+  const promise = predictJson({
+    systemMessage: "From the given input, extract the required information into the requested format." + additionalMessage.trim(),
+    examples: (_a = options == null ? void 0 : options.examples) == null ? void 0 : _a.map(({ value: value2, reason, extracted }) => ({
+      input: value2,
+      output: { reason, result: extracted }
+    })),
+    outputSchema: shape,
+    model: Context.evaluatorModel,
+    input: value
+  });
+  return __spreadProps(__spreadValues({}, toAssertion(promise)), {
+    toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toEqual(expected)), "toBe"),
+    toMatchObject: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toMatchObject(expected)), "toMatchObject"),
+    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise)
+  });
+}
+__name(extract, "extract");
+
+// src/assertions/filter.ts
+import { literal, z as z4 } from "@botpress/sdk";
+function filter(values, condition, options) {
+  var _a, _b;
+  const mappedValues = values.map(
+    (_, idx) => z4.object({
+      index: literal(idx),
+      reason: z4.string(),
+      keep: z4.boolean()
+    })
+  );
+  const input = values.map((value, idx) => ({
+    index: idx,
+    value
+  }));
+  const schema = z4.tuple(mappedValues).describe(
+    "An array of the objects with the index and a boolean value indicating if the object should be kept or not"
+  );
+  const promise = predictJson({
+    systemMessage: `
+Based on the following qualification criteria, you need to filter the given list of objects.
+Citeria: ${condition}
+
+---
+You need to return an array of objects with the index and a boolean value indicating if the object should be kept or not.
+`.trim(),
+    examples: (options == null ? void 0 : options.examples) ? [
+      {
+        input: (_a = options == null ? void 0 : options.examples) == null ? void 0 : _a.map((v, index) => ({
+          index,
+          value: v.value
+        })),
+        output: {
+          reason: "Here are some examples",
+          result: (_b = options == null ? void 0 : options.examples) == null ? void 0 : _b.map((v, idx) => ({
+            index: idx,
+            reason: v.reason,
+            keep: v.keep
+          }))
+        }
+      }
+    ] : void 0,
+    outputSchema: schema,
+    model: Context.evaluatorModel,
+    input
+  }).then((x) => {
+    const results = schema.parse(x.result);
+    return {
+      result: values.filter((_, idx) => {
+        var _a2;
+        return (_a2 = results.find((r) => r.index === idx)) == null ? void 0 : _a2.keep;
+      }),
+      reason: x.reason
+    };
+  });
+  return __spreadProps(__spreadValues({}, toAssertion(promise)), {
+    toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toEqual(expected)), "toBe"),
+    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise),
+    toHaveNoneFiltered: /* @__PURE__ */ __name(() => asyncExpect(promise, (expect3) => expect3.toEqual(values)), "toHaveNoneFiltered"),
+    toHaveSomeFiltered: /* @__PURE__ */ __name(() => asyncExpect(promise, (expect3) => expect3.not.toEqual(values)), "toHaveSomeFiltered"),
+    toBeEmpty: /* @__PURE__ */ __name(() => asyncExpect(promise, (expect3) => expect3.toHaveLength(0)), "toBeEmpty"),
+    length: {
+      toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toHaveLength(expected)), "toBe"),
+      toBeGreaterThanOrEqual: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.length.greaterThanOrEqual(expected)), "toBeGreaterThanOrEqual"),
+      toBeLessThanOrEqual: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.length.lessThanOrEqual(expected)), "toBeLessThanOrEqual"),
+      toBeBetween: /* @__PURE__ */ __name((min, max) => asyncExpect(promise, (expect3) => expect3.length.within(min, max)), "toBeBetween")
+    }
+  });
+}
+__name(filter, "filter");
+
+// src/assertions/rate.ts
+import { z as z5 } from "@botpress/sdk";
+function rate(value, condition, options) {
+  var _a;
+  const schema = z5.number().min(1).max(5).describe("Rating score, higher is better (1 is the worst, 5 is the best)");
+  const promise = predictJson({
+    systemMessage: `Based on the following qualification criteria, you need to rate the given situation from a score of 1 to 5.
+Scoring: 1 is the worst score, 5 is the best score possible.
+Criteria: ${condition}`,
+    examples: (_a = options == null ? void 0 : options.examples) == null ? void 0 : _a.map(({ value: value2, reason, rating }) => ({
+      input: value2,
+      output: { reason, result: rating }
+    })),
+    outputSchema: schema,
+    model: Context.evaluatorModel,
+    input: value
+  }).then((x) => {
+    return {
+      result: typeof x.result === "number" ? x.result : parseInt(x.result, 10),
+      reason: x.reason
+    };
+  });
+  return __spreadProps(__spreadValues({}, toAssertion(promise)), {
+    toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toEqual(expected)), "toBe"),
+    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise),
+    toBeGreaterThanOrEqual: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toBeGreaterThanOrEqual(expected)), "toBeGreaterThanOrEqual"),
+    toBeLessThanOrEqual: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toBeLessThanOrEqual(expected)), "toBeLessThanOrEqual")
+  });
+}
+__name(rate, "rate");
+
+// src/hooks/setEvaluator.ts
+import { getCurrentTest as getCurrentTest4 } from "vitest/suite";
+var setEvaluator = /* @__PURE__ */ __name((model) => {
+  const test = getCurrentTest4();
+  if (!test) {
+    throw new Error("setEvaluator is a Vitest hook and must be called within a test");
+  }
+  const meta = test.meta;
+  meta.evaluatorModel = model;
+}, "setEvaluator");
+
+// src/hooks/setupClient.ts
+var setupClient = /* @__PURE__ */ __name((client) => {
+  Context.setClient(client);
+}, "setupClient");
+export {
+  check,
+  compare,
+  extract,
+  filter,
+  rate,
+  setEvaluator,
+  setupClient
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/task/compare.ts","../src/utils/deferred.ts","../src/assertions/check.ts","../src/context.ts","../src/utils/asyncAssertion.ts","../src/utils/predictJson.ts","../src/assertions/extension.ts","../src/assertions/extract.ts","../src/assertions/filter.ts","../src/assertions/rate.ts","../src/hooks/setEvaluator.ts","../src/hooks/setupClient.ts"],"sourcesContent":["import { z } from '@botpress/sdk'\nimport { TestFunction } from 'vitest'\nimport { createTaskCollector, getCurrentSuite } from 'vitest/suite'\nimport { TestMetadata } from '../context'\nimport { Deferred } from '../utils/deferred'\n\nconst scenarioId = z\n  .string()\n  .trim()\n  .min(1, 'Scenario ID/name must not be empty')\n  .max(50, 'Scenario ID/name is too long')\n\nexport type ScenarioLike = z.infer<typeof ScenarioLike>\nconst ScenarioLike = z.union([\n  scenarioId,\n  z.object({ name: scenarioId }).passthrough(),\n  z.object({ id: scenarioId }).passthrough()\n])\n\nconst getScenarioName = (scenario: ScenarioLike) =>\n  (typeof scenario === 'string' ? scenario : 'name' in scenario ? scenario?.name : scenario?.id) as string\n\nconst scenarioArgs = z\n  .array(ScenarioLike)\n  .min(2, 'You need at least two scenarios to compare')\n  .max(10, 'You can only compare up to 10 scenarios')\n  .refine((scenarios) => {\n    const set = new Set<string>()\n    scenarios.forEach((scenario) => set.add(getScenarioName(scenario)))\n    return set.size === scenarios.length\n  }, 'Scenarios names must be unique')\n\nexport function compare<T extends ReadonlyArray<ScenarioLike>>(\n  name: string | Function,\n  scenarios: T,\n  fn?: TestFunction<{\n    scenario: T[number]\n  }>\n) {\n  scenarios = scenarioArgs.parse(scenarios) as unknown as T\n\n  return createTaskCollector((_name, fn, timeout) => {\n    const currentSuite = getCurrentSuite()\n\n    let completedCount = 0\n    const finished = new Deferred<void>()\n\n    for (const scenario of scenarios) {\n      const key = getScenarioName(scenario)\n\n      currentSuite.task(key, {\n        meta: {\n          scenario: key,\n          isVaiTest: true\n        } satisfies TestMetadata,\n        handler: async (context) => {\n          const extendedContext = Object.freeze({\n            scenario\n          })\n          context.onTestFinished(() => {\n            if (++completedCount === scenarios.length) {\n              finished.resolve()\n            }\n          })\n\n          await fn({ ...context, ...extendedContext })\n        },\n        timeout: timeout ?? 10_000\n      })\n    }\n  })(name, fn)\n}\n","export class Deferred<T> {\n  promise: Promise<T>\n  private _resolve!: (value: T | PromiseLike<T>) => void\n  private _reject!: (reason?: unknown) => void\n\n  constructor() {\n    this.promise = new Promise<T>((resolve, reject) => {\n      this._resolve = resolve\n      this._reject = reject\n    })\n  }\n\n  resolve(value: T | PromiseLike<T>): void {\n    this._resolve(value)\n  }\n\n  reject(reason?: unknown): void {\n    this._reject(reason)\n  }\n}\n","import { z } from '@botpress/sdk'\nimport { Context } from '../context'\nimport { asyncExpect } from '../utils/asyncAssertion'\nimport { Input, predictJson } from '../utils/predictJson'\nimport { makeToMatchInlineSnapshot, toAssertion } from './extension'\n\nexport type CheckOptions<T> = {\n  examples?: { value: T; expected: boolean; reason: string }[]\n}\n\nexport function check<T extends Input>(value: T, condition: string, options?: CheckOptions<T>) {\n  const promise = predictJson({\n    systemMessage: `Check that the value meets the condition: ${condition}`,\n    examples: options?.examples?.map(({ value, reason, expected }) => ({\n      input: value,\n      output: { reason, result: expected }\n    })),\n    outputSchema: z.boolean(),\n    model: Context.evaluatorModel,\n    input: value\n  })\n\n  return {\n    ...toAssertion(promise),\n    toBe: (expected: boolean) => asyncExpect(promise, (expect) => expect.toEqual(expected)),\n    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise)\n  }\n}\n","import type { Client } from '@botpress/client'\nimport { onTestFinished } from 'vitest'\nimport { getCurrentTest } from 'vitest/suite'\nimport { Models } from './models'\n\nexport type EvaluatorModel = (typeof Models)[number]['id']\n\nexport type TestMetadata = {\n  isVaiTest: boolean\n  scenario?: string\n  evaluatorModel?: EvaluatorModel\n}\n\nconst getTestMetadata = (): TestMetadata => {\n  const test = getCurrentTest()\n  return (test?.meta ?? {\n    isVaiTest: false\n  }) as TestMetadata\n}\n\nclass VaiContext {\n  #client: Client | null = null\n  #wrapError = false\n\n  get wrapError() {\n    return this.#wrapError\n  }\n\n  get client() {\n    if (!this.#client) {\n      throw new Error('Botpress client is not set')\n    }\n\n    return this.#client\n  }\n\n  get evaluatorModel(): EvaluatorModel {\n    return getTestMetadata().evaluatorModel ?? 'openai__gpt-4o-mini-2024-07-18'\n  }\n\n  get scenario() {\n    return getTestMetadata().scenario\n  }\n\n  get isVaiTest() {\n    return getTestMetadata().isVaiTest\n  }\n\n  setClient(cognitive: Client) {\n    this.#client = cognitive\n  }\n\n  swallowErrors() {\n    if (!getCurrentTest()) {\n      throw new Error('cancelBail is a Vitest hook and must be called within a test')\n    }\n\n    this.#wrapError = true\n    onTestFinished(() => {\n      this.#wrapError = false\n    })\n  }\n}\n\nexport const Context = new VaiContext()\n","import { Assertion, expect } from 'vitest'\nimport { getCurrentTest } from 'vitest/suite'\nimport { Context } from '../context'\nimport { Output } from './predictJson'\n\nexport class AsyncExpectError<T> extends Error {\n  constructor(message: string, public readonly output: Output<T>) {\n    super(message)\n    this.name = 'AsyncExpectError'\n  }\n}\n\nconst getErrorMessages = (e: unknown): string => {\n  if (e instanceof Error) {\n    return e.message\n  } else if (typeof e === 'string') {\n    return e\n  } else if (typeof e === 'object' && e !== null) {\n    return JSON.stringify(e)\n  }\n\n  return `Unknown error: ${e}`\n}\n\nexport const asyncExpect = <T>(output: Promise<Output<T>>, assertion: (assert: Assertion<T>) => void) => {\n  const promise = output.then((x) => {\n    try {\n      assertion(expect(x.result, x.reason))\n    } catch (e: unknown) {\n      if (Context.wrapError) {\n        return new AsyncExpectError<T>(getErrorMessages(e), x)\n      }\n      throw e\n    }\n    return x\n  })\n  getCurrentTest()!.promises ??= []\n  getCurrentTest()!.promises!.push(promise)\n  return promise\n}\n","import { z, ZodSchema } from '@botpress/sdk'\nimport JSON5 from 'json5'\nimport { Context } from '../context'\nimport { llm } from '../sdk-interfaces/llm/generateContent'\n\nconst nonEmptyString = z.string().trim().min(1)\nconst nonEmptyObject = z\n  .object({})\n  .passthrough()\n  .refine((value) => Object.keys(value).length > 0, {\n    message: 'Expected a non-empty object'\n  })\n\nexport type Input = z.infer<typeof Input>\nconst Input = nonEmptyString.or(nonEmptyObject).or(z.array(z.any()))\n\n// eslint-disable-next-line @typescript-eslint/no-explicit-any\nexport type Output<T = any> = z.infer<typeof Output> & { result: T }\nconst Output = z.object({\n  reason: nonEmptyString.describe('A human-readable explanation of the result'),\n  result: z\n    .any()\n    .describe(\n      'Your best guess at the output according to the instructions provided, rooted in the context of the input and the reason above'\n    )\n})\n\ntype Example = z.infer<typeof Example>\nconst Example = z.object({\n  input: Input,\n  output: Output\n})\n\ntype InputOptions<T extends ZodSchema = ZodSchema> = z.input<typeof Options> & { outputSchema: T }\ntype Options = z.infer<typeof Options>\nconst Options = z.object({\n  systemMessage: z.string(),\n  examples: z.array(Example).default([]),\n  input: Input,\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  outputSchema: z.custom<ZodSchema<any>>((value) => value instanceof ZodSchema),\n  model: z.string()\n})\n\ntype Message = {\n  role: 'user' | 'assistant' | 'system'\n  content: string\n}\n\nconst isValidExample =\n  (outputSchema: ZodSchema) =>\n  (example: Example): example is Example =>\n    Input.safeParse(example.input).success &&\n    Output.safeParse(example.output).success &&\n    outputSchema.safeParse(example.output.result).success\n\nexport async function predictJson<T extends ZodSchema>(_options: InputOptions<T>): Promise<Output<z.infer<T>>> {\n  const options = Options.parse(_options)\n  const [integration, model] = options.model.split('__')\n\n  if (!model?.length) {\n    throw new Error('Invalid model')\n  }\n\n  const exampleMessages = options.examples\n    .filter(isValidExample(options.outputSchema))\n    .flatMap(({ input, output }) => [\n      { role: 'user', content: JSON.stringify(input, null, 2) } satisfies Message,\n      { role: 'assistant', content: JSON.stringify(output, null, 2) } satisfies Message\n    ])\n\n  const outputSchema = Output.extend({\n    result: options.outputSchema.describe(Output.shape.result.description!)\n  })\n\n  const result = await Context.client.callAction({\n    type: `${integration}:generateContent`,\n    input: {\n      systemPrompt: `\n${options.systemMessage}\n\n---\nPlease generate a JSON response with the following format:\n\\`\\`\\`typescript\n${await outputSchema.toTypescriptAsync()}\n\\`\\`\\`\n`.trim(),\n      messages: [\n        ...exampleMessages,\n        {\n          role: 'user',\n          content: JSON.stringify(options.input, null, 2)\n        }\n      ],\n      temperature: 0,\n      responseFormat: 'json_object',\n      model: { id: model! }\n    } satisfies llm.generateContent.Input\n  })\n\n  const output = result.output as llm.generateContent.Output\n\n  if (!output.choices.length || typeof output.choices?.[0]?.content !== 'string') {\n    throw new Error('Invalid response from the model')\n  }\n\n  const json = output.choices[0].content.trim()\n\n  if (!json.length) {\n    throw new Error('No response from the model')\n  }\n\n  return outputSchema.parse(JSON5.parse(json)) as Output<z.infer<T>>\n}\n","import json5 from 'json5'\nimport { expect } from 'vitest'\nimport { getCurrentTest } from 'vitest/suite'\n\nimport { asyncExpect } from '../utils/asyncAssertion'\nimport { Output } from '../utils/predictJson'\n\nexport type ExtendedPromise<T> = PromiseLike<Output<T>> & {\n  value: PromiseLike<T>\n}\n\nexport const toAssertion = <T>(promise: Promise<Output<T>>): ExtendedPromise<T> => {\n  return {\n    then: promise.then.bind(promise),\n    value: promise.then((value) => value.result)\n  }\n}\n\nexport const makeToMatchInlineSnapshot =\n  <T>(promise: Promise<Output<T>>) =>\n  async (expected?: string) => {\n    const stack = new Error().stack!.split('\\n')[2]\n    const newStack = `\nat __INLINE_SNAPSHOT__ (node:internal/process/task_queues:1:1)\nat randomLine (node:internal/process/task_queues:1:1)\n${stack}\n`.trim()\n\n    const obj = json5.parse(expected ?? '\"\"')\n    const expectation = asyncExpect(promise, (expect) => expect.toMatchObject(obj)).catch(() => {\n      // we swallow the error here, as we're going to throw a new one with the correct stack\n      // this is just to make vitest happy and show a nice error message\n    })\n\n    try {\n      expect((await promise).result).toMatchObject(obj)\n    } catch (err) {\n      const newError = new Error()\n      newError.stack = newStack\n\n      expect.getState().snapshotState.match({\n        isInline: true,\n        received: (await promise).result,\n        testName: getCurrentTest()!.name,\n        error: newError,\n        inlineSnapshot: expected\n      })\n    }\n\n    return expectation\n  }\n","import { z } from '@botpress/sdk'\n\nimport { Context } from '../context'\nimport { asyncExpect } from '../utils/asyncAssertion'\nimport { Input, predictJson } from '../utils/predictJson'\nimport { makeToMatchInlineSnapshot, toAssertion } from './extension'\n\nexport type ExtractOptions<T, S> = {\n  description?: string\n  examples?: { value: T; extracted: S; reason: string }[]\n}\n\nexport function extract<T extends Input, S extends z.AnyZodObject>(\n  value: T,\n  shape: S,\n  options?: ExtractOptions<T, z.infer<S>>\n) {\n  const additionalMessage = options?.description\n    ? `\\nIn order to extract the right information, follow these instructions:\\n${options.description}\\n`\n    : ''\n  const promise = predictJson({\n    systemMessage:\n      'From the given input, extract the required information into the requested format.' + additionalMessage.trim(),\n    examples: options?.examples?.map(({ value, reason, extracted }) => ({\n      input: value,\n      output: { reason, result: extracted }\n    })),\n    outputSchema: shape,\n    model: Context.evaluatorModel,\n    input: value\n  })\n\n  return {\n    ...toAssertion(promise),\n    toBe: (expected: z.infer<S>) => asyncExpect(promise, (expect) => expect.toEqual(expected)),\n    toMatchObject: (expected: Partial<z.infer<S>>) => asyncExpect(promise, (expect) => expect.toMatchObject(expected)),\n    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise)\n  }\n}\n","import { literal, z } from '@botpress/sdk'\n\nimport { Context } from '../context'\nimport { asyncExpect } from '../utils/asyncAssertion'\nimport { predictJson } from '../utils/predictJson'\nimport { makeToMatchInlineSnapshot, toAssertion } from './extension'\n\nexport type FilterOptions<T> = {\n  examples?: { value: T; reason: string; keep: boolean }[]\n}\n\nexport function filter<U>(values: U[], condition: string, options?: FilterOptions<U>) {\n  const mappedValues = values.map((_, idx) =>\n    z.object({\n      index: literal(idx),\n      reason: z.string(),\n      keep: z.boolean()\n    })\n  )\n\n  const input = values.map((value, idx) => ({\n    index: idx,\n    value\n  }))\n\n  const schema = z\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    .tuple(mappedValues as any)\n    .describe(\n      'An array of the objects with the index and a boolean value indicating if the object should be kept or not'\n    )\n\n  const promise = predictJson({\n    systemMessage: `\nBased on the following qualification criteria, you need to filter the given list of objects.\nCiteria: ${condition}\n\n---\nYou need to return an array of objects with the index and a boolean value indicating if the object should be kept or not.\n`.trim(),\n    examples: options?.examples\n      ? [\n          {\n            input: options?.examples?.map((v, index) => ({\n              index,\n              value: v.value\n            })),\n            output: {\n              reason: 'Here are some examples',\n              result: options?.examples?.map((v, idx) => ({\n                index: idx,\n                reason: v.reason,\n                keep: v.keep\n              }))\n            }\n          }\n        ]\n      : undefined,\n    outputSchema: schema,\n    model: Context.evaluatorModel,\n    input\n  }).then((x) => {\n    const results = schema.parse(x.result) as { index: number; keep: boolean }[]\n    return {\n      result: values.filter((_, idx) => results.find((r) => r.index === idx)?.keep),\n      reason: x.reason\n    }\n  })\n\n  return {\n    ...toAssertion(promise),\n    toBe: (expected: U[]) => asyncExpect(promise, (expect) => expect.toEqual(expected)),\n    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise),\n    toHaveNoneFiltered: () => asyncExpect(promise, (expect) => expect.toEqual(values)),\n    toHaveSomeFiltered: () => asyncExpect(promise, (expect) => expect.not.toEqual(values)),\n    toBeEmpty: () => asyncExpect(promise, (expect) => expect.toHaveLength(0)),\n    length: {\n      toBe: (expected: number) => asyncExpect(promise, (expect) => expect.toHaveLength(expected)),\n      toBeGreaterThanOrEqual: (expected: number) =>\n        asyncExpect(promise, (expect) => expect.length.greaterThanOrEqual(expected)),\n      toBeLessThanOrEqual: (expected: number) =>\n        asyncExpect(promise, (expect) => expect.length.lessThanOrEqual(expected)),\n      toBeBetween: (min: number, max: number) => asyncExpect(promise, (expect) => expect.length.within(min, max))\n    }\n  }\n}\n","import { z } from '@botpress/sdk'\n\nimport { Context } from '../context'\nimport { asyncExpect } from '../utils/asyncAssertion'\nimport { Input, predictJson } from '../utils/predictJson'\nimport { makeToMatchInlineSnapshot, toAssertion } from './extension'\n\nexport type RatingScore = 1 | 2 | 3 | 4 | 5\nexport type RateOptions<T> = {\n  examples?: { value: T; rating: number; reason: string }[]\n}\n\nexport function rate<T extends Input>(value: T, condition: string, options?: RateOptions<T>) {\n  const schema = z.number().min(1).max(5).describe('Rating score, higher is better (1 is the worst, 5 is the best)')\n  const promise = predictJson({\n    systemMessage: `Based on the following qualification criteria, you need to rate the given situation from a score of 1 to 5.\\nScoring: 1 is the worst score, 5 is the best score possible.\\nCriteria: ${condition}`,\n    examples: options?.examples?.map(({ value, reason, rating }) => ({\n      input: value,\n      output: { reason, result: rating }\n    })),\n    outputSchema: schema,\n    model: Context.evaluatorModel,\n    input: value\n  }).then((x) => {\n    return {\n      result: typeof x.result === 'number' ? x.result : parseInt(x.result, 10),\n      reason: x.reason\n    }\n  })\n\n  return {\n    ...toAssertion(promise),\n    toBe: (expected: number) => asyncExpect(promise, (expect) => expect.toEqual(expected)),\n    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise),\n    toBeGreaterThanOrEqual: (expected: RatingScore) =>\n      asyncExpect(promise, (expect) => expect.toBeGreaterThanOrEqual(expected)),\n    toBeLessThanOrEqual: (expected: RatingScore) =>\n      asyncExpect(promise, (expect) => expect.toBeLessThanOrEqual(expected))\n  }\n}\n","import { getCurrentTest } from 'vitest/suite'\nimport { EvaluatorModel, TestMetadata } from '../context'\n\nexport const setEvaluator = (model: EvaluatorModel) => {\n  const test = getCurrentTest()\n\n  if (!test) {\n    throw new Error('setEvaluator is a Vitest hook and must be called within a test')\n  }\n\n  const meta = test.meta as TestMetadata\n  meta.evaluatorModel = model\n}\n","import { Client } from '@botpress/client'\nimport { Context } from '../context'\n\nexport const setupClient = (client: Client) => {\n  Context.setClient(client)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,SAAS;AAElB,SAAS,qBAAqB,uBAAuB;;;ACF9C,IAAM,YAAN,MAAM,UAAY;AAAA,EAKvB,cAAc;AACZ,SAAK,UAAU,IAAI,QAAW,CAAC,SAAS,WAAW;AACjD,WAAK,WAAW;AAChB,WAAK,UAAU;AAAA,IACjB,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ,OAAiC;AACvC,SAAK,SAAS,KAAK;AAAA,EACrB;AAAA,EAEA,OAAO,QAAwB;AAC7B,SAAK,QAAQ,MAAM;AAAA,EACrB;AACF;AAnByB;AAAlB,IAAM,WAAN;;;ADMP,IAAM,aAAa,EAChB,OAAO,EACP,KAAK,EACL,IAAI,GAAG,oCAAoC,EAC3C,IAAI,IAAI,8BAA8B;AAGzC,IAAM,eAAe,EAAE,MAAM;AAAA,EAC3B;AAAA,EACA,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC,EAAE,YAAY;AAAA,EAC3C,EAAE,OAAO,EAAE,IAAI,WAAW,CAAC,EAAE,YAAY;AAC3C,CAAC;AAED,IAAM,kBAAkB,wBAAC,aACtB,OAAO,aAAa,WAAW,WAAW,UAAU,WAAW,qCAAU,OAAO,qCAAU,IADrE;AAGxB,IAAM,eAAe,EAClB,MAAM,YAAY,EAClB,IAAI,GAAG,4CAA4C,EACnD,IAAI,IAAI,yCAAyC,EACjD,OAAO,CAAC,cAAc;AACrB,QAAM,MAAM,oBAAI,IAAY;AAC5B,YAAU,QAAQ,CAAC,aAAa,IAAI,IAAI,gBAAgB,QAAQ,CAAC,CAAC;AAClE,SAAO,IAAI,SAAS,UAAU;AAChC,GAAG,gCAAgC;AAE9B,SAAS,QACd,MACA,WACA,IAGA;AACA,cAAY,aAAa,MAAM,SAAS;AAExC,SAAO,oBAAoB,CAAC,OAAOA,KAAI,YAAY;AACjD,UAAM,eAAe,gBAAgB;AAErC,QAAI,iBAAiB;AACrB,UAAM,WAAW,IAAI,SAAe;AAEpC,eAAW,YAAY,WAAW;AAChC,YAAM,MAAM,gBAAgB,QAAQ;AAEpC,mBAAa,KAAK,KAAK;AAAA,QACrB,MAAM;AAAA,UACJ,UAAU;AAAA,UACV,WAAW;AAAA,QACb;AAAA,QACA,SAAS,8BAAO,YAAY;AAC1B,gBAAM,kBAAkB,OAAO,OAAO;AAAA,YACpC;AAAA,UACF,CAAC;AACD,kBAAQ,eAAe,MAAM;AAC3B,gBAAI,EAAE,mBAAmB,UAAU,QAAQ;AACzC,uBAAS,QAAQ;AAAA,YACnB;AAAA,UACF,CAAC;AAED,gBAAMA,IAAG,kCAAK,UAAY,gBAAiB;AAAA,QAC7C,GAXS;AAAA,QAYT,SAAS,4BAAW;AAAA,MACtB,CAAC;AAAA,IACH;AAAA,EACF,CAAC,EAAE,MAAM,EAAE;AACb;AAvCgB;;;AEhChB,SAAS,KAAAC,UAAS;;;ACClB,SAAS,sBAAsB;AAC/B,SAAS,sBAAsB;AAW/B,IAAM,kBAAkB,6BAAoB;AAb5C;AAcE,QAAM,OAAO,eAAe;AAC5B,UAAQ,kCAAM,SAAN,YAAc;AAAA,IACpB,WAAW;AAAA,EACb;AACF,GALwB;AAbxB;AAoBA,IAAM,cAAN,MAAM,YAAW;AAAA,EAAjB;AACE,gCAAyB;AACzB,mCAAa;AAAA;AAAA,EAEb,IAAI,YAAY;AACd,WAAO,mBAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAS;AACX,QAAI,CAAC,mBAAK,UAAS;AACjB,YAAM,IAAI,MAAM,4BAA4B;AAAA,IAC9C;AAEA,WAAO,mBAAK;AAAA,EACd;AAAA,EAEA,IAAI,iBAAiC;AApCvC;AAqCI,YAAO,qBAAgB,EAAE,mBAAlB,YAAoC;AAAA,EAC7C;AAAA,EAEA,IAAI,WAAW;AACb,WAAO,gBAAgB,EAAE;AAAA,EAC3B;AAAA,EAEA,IAAI,YAAY;AACd,WAAO,gBAAgB,EAAE;AAAA,EAC3B;AAAA,EAEA,UAAU,WAAmB;AAC3B,uBAAK,SAAU;AAAA,EACjB;AAAA,EAEA,gBAAgB;AACd,QAAI,CAAC,eAAe,GAAG;AACrB,YAAM,IAAI,MAAM,8DAA8D;AAAA,IAChF;AAEA,uBAAK,YAAa;AAClB,mBAAe,MAAM;AACnB,yBAAK,YAAa;AAAA,IACpB,CAAC;AAAA,EACH;AACF;AAzCE;AACA;AAFe;AAAjB,IAAM,aAAN;AA4CO,IAAM,UAAU,IAAI,WAAW;;;AChEtC,SAAoB,cAAc;AAClC,SAAS,kBAAAC,uBAAsB;AAIxB,IAAM,oBAAN,MAAM,0BAA4B,MAAM;AAAA,EAC7C,YAAY,SAAiC,QAAmB;AAC9D,UAAM,OAAO;AAD8B;AAE3C,SAAK,OAAO;AAAA,EACd;AACF;AAL+C;AAAxC,IAAM,mBAAN;AAOP,IAAM,mBAAmB,wBAAC,MAAuB;AAC/C,MAAI,aAAa,OAAO;AACtB,WAAO,EAAE;AAAA,EACX,WAAW,OAAO,MAAM,UAAU;AAChC,WAAO;AAAA,EACT,WAAW,OAAO,MAAM,YAAY,MAAM,MAAM;AAC9C,WAAO,KAAK,UAAU,CAAC;AAAA,EACzB;AAEA,SAAO,kBAAkB,CAAC;AAC5B,GAVyB;AAYlB,IAAM,cAAc,wBAAI,QAA4B,cAA8C;AAxBzG;AAyBE,QAAM,UAAU,OAAO,KAAK,CAAC,MAAM;AACjC,QAAI;AACF,gBAAU,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC;AAAA,IACtC,SAAS,GAAY;AACnB,UAAI,QAAQ,WAAW;AACrB,eAAO,IAAI,iBAAoB,iBAAiB,CAAC,GAAG,CAAC;AAAA,MACvD;AACA,YAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT,CAAC;AACD,cAAAC,gBAAe,GAAG,aAAlB,eAAkB,WAAa,CAAC;AAChC,EAAAA,gBAAe,EAAG,SAAU,KAAK,OAAO;AACxC,SAAO;AACT,GAf2B;;;ACxB3B,SAAS,KAAAC,IAAG,iBAAiB;AAC7B,OAAO,WAAW;AAIlB,IAAM,iBAAiBC,GAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC;AAC9C,IAAM,iBAAiBA,GACpB,OAAO,CAAC,CAAC,EACT,YAAY,EACZ,OAAO,CAAC,UAAU,OAAO,KAAK,KAAK,EAAE,SAAS,GAAG;AAAA,EAChD,SAAS;AACX,CAAC;AAGH,IAAM,QAAQ,eAAe,GAAG,cAAc,EAAE,GAAGA,GAAE,MAAMA,GAAE,IAAI,CAAC,CAAC;AAInE,IAAM,SAASA,GAAE,OAAO;AAAA,EACtB,QAAQ,eAAe,SAAS,4CAA4C;AAAA,EAC5E,QAAQA,GACL,IAAI,EACJ;AAAA,IACC;AAAA,EACF;AACJ,CAAC;AAGD,IAAM,UAAUA,GAAE,OAAO;AAAA,EACvB,OAAO;AAAA,EACP,QAAQ;AACV,CAAC;AAID,IAAM,UAAUA,GAAE,OAAO;AAAA,EACvB,eAAeA,GAAE,OAAO;AAAA,EACxB,UAAUA,GAAE,MAAM,OAAO,EAAE,QAAQ,CAAC,CAAC;AAAA,EACrC,OAAO;AAAA;AAAA,EAEP,cAAcA,GAAE,OAAuB,CAAC,UAAU,iBAAiB,SAAS;AAAA,EAC5E,OAAOA,GAAE,OAAO;AAClB,CAAC;AAOD,IAAM,iBACJ,wBAAC,iBACD,CAAC,YACC,MAAM,UAAU,QAAQ,KAAK,EAAE,WAC/B,OAAO,UAAU,QAAQ,MAAM,EAAE,WACjC,aAAa,UAAU,QAAQ,OAAO,MAAM,EAAE,SAJhD;AAMF,eAAsB,YAAiC,UAAwD;AAxD/G;AAyDE,QAAM,UAAU,QAAQ,MAAM,QAAQ;AACtC,QAAM,CAAC,aAAa,KAAK,IAAI,QAAQ,MAAM,MAAM,IAAI;AAErD,MAAI,EAAC,+BAAO,SAAQ;AAClB,UAAM,IAAI,MAAM,eAAe;AAAA,EACjC;AAEA,QAAM,kBAAkB,QAAQ,SAC7B,OAAO,eAAe,QAAQ,YAAY,CAAC,EAC3C,QAAQ,CAAC,EAAE,OAAO,QAAAC,QAAO,MAAM;AAAA,IAC9B,EAAE,MAAM,QAAQ,SAAS,KAAK,UAAU,OAAO,MAAM,CAAC,EAAE;AAAA,IACxD,EAAE,MAAM,aAAa,SAAS,KAAK,UAAUA,SAAQ,MAAM,CAAC,EAAE;AAAA,EAChE,CAAC;AAEH,QAAM,eAAe,OAAO,OAAO;AAAA,IACjC,QAAQ,QAAQ,aAAa,SAAS,OAAO,MAAM,OAAO,WAAY;AAAA,EACxE,CAAC;AAED,QAAM,SAAS,MAAM,QAAQ,OAAO,WAAW;AAAA,IAC7C,MAAM,GAAG,WAAW;AAAA,IACpB,OAAO;AAAA,MACL,cAAc;AAAA,EAClB,QAAQ,aAAa;AAAA;AAAA;AAAA;AAAA;AAAA,EAKrB,MAAM,aAAa,kBAAkB,CAAC;AAAA;AAAA,EAEtC,KAAK;AAAA,MACD,UAAU;AAAA,QACR,GAAG;AAAA,QACH;AAAA,UACE,MAAM;AAAA,UACN,SAAS,KAAK,UAAU,QAAQ,OAAO,MAAM,CAAC;AAAA,QAChD;AAAA,MACF;AAAA,MACA,aAAa;AAAA,MACb,gBAAgB;AAAA,MAChB,OAAO,EAAE,IAAI,MAAO;AAAA,IACtB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO;AAEtB,MAAI,CAAC,OAAO,QAAQ,UAAU,SAAO,kBAAO,YAAP,mBAAiB,OAAjB,mBAAqB,aAAY,UAAU;AAC9E,UAAM,IAAI,MAAM,iCAAiC;AAAA,EACnD;AAEA,QAAM,OAAO,OAAO,QAAQ,CAAC,EAAE,QAAQ,KAAK;AAE5C,MAAI,CAAC,KAAK,QAAQ;AAChB,UAAM,IAAI,MAAM,4BAA4B;AAAA,EAC9C;AAEA,SAAO,aAAa,MAAM,MAAM,MAAM,IAAI,CAAC;AAC7C;AAzDsB;;;ACxDtB,OAAO,WAAW;AAClB,SAAS,UAAAC,eAAc;AACvB,SAAS,kBAAAC,uBAAsB;AASxB,IAAM,cAAc,wBAAI,YAAoD;AACjF,SAAO;AAAA,IACL,MAAM,QAAQ,KAAK,KAAK,OAAO;AAAA,IAC/B,OAAO,QAAQ,KAAK,CAAC,UAAU,MAAM,MAAM;AAAA,EAC7C;AACF,GAL2B;AAOpB,IAAM,4BACX,wBAAI,YACJ,OAAO,aAAsB;AAC3B,QAAM,QAAQ,IAAI,MAAM,EAAE,MAAO,MAAM,IAAI,EAAE,CAAC;AAC9C,QAAM,WAAW;AAAA;AAAA;AAAA,EAGnB,KAAK;AAAA,EACL,KAAK;AAEH,QAAM,MAAM,MAAM,MAAM,8BAAY,IAAI;AACxC,QAAM,cAAc,YAAY,SAAS,CAACC,YAAWA,QAAO,cAAc,GAAG,CAAC,EAAE,MAAM,MAAM;AAAA,EAG5F,CAAC;AAED,MAAI;AACF,IAAAA,SAAQ,MAAM,SAAS,MAAM,EAAE,cAAc,GAAG;AAAA,EAClD,SAAS,KAAK;AACZ,UAAM,WAAW,IAAI,MAAM;AAC3B,aAAS,QAAQ;AAEjB,IAAAA,QAAO,SAAS,EAAE,cAAc,MAAM;AAAA,MACpC,UAAU;AAAA,MACV,WAAW,MAAM,SAAS;AAAA,MAC1B,UAAUC,gBAAe,EAAG;AAAA,MAC5B,OAAO;AAAA,MACP,gBAAgB;AAAA,IAClB,CAAC;AAAA,EACH;AAEA,SAAO;AACT,GA/BA;;;AJTK,SAAS,MAAuB,OAAU,WAAmB,SAA2B;AAV/F;AAWE,QAAM,UAAU,YAAY;AAAA,IAC1B,eAAe,6CAA6C,SAAS;AAAA,IACrE,WAAU,wCAAS,aAAT,mBAAmB,IAAI,CAAC,EAAE,OAAAC,QAAO,QAAQ,SAAS,OAAO;AAAA,MACjE,OAAOA;AAAA,MACP,QAAQ,EAAE,QAAQ,QAAQ,SAAS;AAAA,IACrC;AAAA,IACA,cAAcC,GAAE,QAAQ;AAAA,IACxB,OAAO,QAAQ;AAAA,IACf,OAAO;AAAA,EACT,CAAC;AAED,SAAO,iCACF,YAAY,OAAO,IADjB;AAAA,IAEL,MAAM,wBAAC,aAAsB,YAAY,SAAS,CAACC,YAAWA,QAAO,QAAQ,QAAQ,CAAC,GAAhF;AAAA,IACN,uBAAuB,0BAA0B,OAAO;AAAA,EAC1D;AACF;AAjBgB;;;AKET,SAAS,QACd,OACA,OACA,SACA;AAhBF;AAiBE,QAAM,qBAAoB,mCAAS,eAC/B;AAAA;AAAA,EAA4E,QAAQ,WAAW;AAAA,IAC/F;AACJ,QAAM,UAAU,YAAY;AAAA,IAC1B,eACE,sFAAsF,kBAAkB,KAAK;AAAA,IAC/G,WAAU,wCAAS,aAAT,mBAAmB,IAAI,CAAC,EAAE,OAAAC,QAAO,QAAQ,UAAU,OAAO;AAAA,MAClE,OAAOA;AAAA,MACP,QAAQ,EAAE,QAAQ,QAAQ,UAAU;AAAA,IACtC;AAAA,IACA,cAAc;AAAA,IACd,OAAO,QAAQ;AAAA,IACf,OAAO;AAAA,EACT,CAAC;AAED,SAAO,iCACF,YAAY,OAAO,IADjB;AAAA,IAEL,MAAM,wBAAC,aAAyB,YAAY,SAAS,CAACC,YAAWA,QAAO,QAAQ,QAAQ,CAAC,GAAnF;AAAA,IACN,eAAe,wBAAC,aAAkC,YAAY,SAAS,CAACA,YAAWA,QAAO,cAAc,QAAQ,CAAC,GAAlG;AAAA,IACf,uBAAuB,0BAA0B,OAAO;AAAA,EAC1D;AACF;AA1BgB;;;ACZhB,SAAS,SAAS,KAAAC,UAAS;AAWpB,SAAS,OAAU,QAAa,WAAmB,SAA4B;AAXtF;AAYE,QAAM,eAAe,OAAO;AAAA,IAAI,CAAC,GAAG,QAClCC,GAAE,OAAO;AAAA,MACP,OAAO,QAAQ,GAAG;AAAA,MAClB,QAAQA,GAAE,OAAO;AAAA,MACjB,MAAMA,GAAE,QAAQ;AAAA,IAClB,CAAC;AAAA,EACH;AAEA,QAAM,QAAQ,OAAO,IAAI,CAAC,OAAO,SAAS;AAAA,IACxC,OAAO;AAAA,IACP;AAAA,EACF,EAAE;AAEF,QAAM,SAASA,GAEZ,MAAM,YAAmB,EACzB;AAAA,IACC;AAAA,EACF;AAEF,QAAM,UAAU,YAAY;AAAA,IAC1B,eAAe;AAAA;AAAA,WAER,SAAS;AAAA;AAAA;AAAA;AAAA,EAIlB,KAAK;AAAA,IACH,WAAU,mCAAS,YACf;AAAA,MACE;AAAA,QACE,QAAO,wCAAS,aAAT,mBAAmB,IAAI,CAAC,GAAG,WAAW;AAAA,UAC3C;AAAA,UACA,OAAO,EAAE;AAAA,QACX;AAAA,QACA,QAAQ;AAAA,UACN,QAAQ;AAAA,UACR,SAAQ,wCAAS,aAAT,mBAAmB,IAAI,CAAC,GAAG,SAAS;AAAA,YAC1C,OAAO;AAAA,YACP,QAAQ,EAAE;AAAA,YACV,MAAM,EAAE;AAAA,UACV;AAAA,QACF;AAAA,MACF;AAAA,IACF,IACA;AAAA,IACJ,cAAc;AAAA,IACd,OAAO,QAAQ;AAAA,IACf;AAAA,EACF,CAAC,EAAE,KAAK,CAAC,MAAM;AACb,UAAM,UAAU,OAAO,MAAM,EAAE,MAAM;AACrC,WAAO;AAAA,MACL,QAAQ,OAAO,OAAO,CAAC,GAAG,QAAK;AAhErC,YAAAC;AAgEwC,gBAAAA,MAAA,QAAQ,KAAK,CAAC,MAAM,EAAE,UAAU,GAAG,MAAnC,gBAAAA,IAAsC;AAAA,OAAI;AAAA,MAC5E,QAAQ,EAAE;AAAA,IACZ;AAAA,EACF,CAAC;AAED,SAAO,iCACF,YAAY,OAAO,IADjB;AAAA,IAEL,MAAM,wBAAC,aAAkB,YAAY,SAAS,CAACC,YAAWA,QAAO,QAAQ,QAAQ,CAAC,GAA5E;AAAA,IACN,uBAAuB,0BAA0B,OAAO;AAAA,IACxD,oBAAoB,6BAAM,YAAY,SAAS,CAACA,YAAWA,QAAO,QAAQ,MAAM,CAAC,GAA7D;AAAA,IACpB,oBAAoB,6BAAM,YAAY,SAAS,CAACA,YAAWA,QAAO,IAAI,QAAQ,MAAM,CAAC,GAAjE;AAAA,IACpB,WAAW,6BAAM,YAAY,SAAS,CAACA,YAAWA,QAAO,aAAa,CAAC,CAAC,GAA7D;AAAA,IACX,QAAQ;AAAA,MACN,MAAM,wBAAC,aAAqB,YAAY,SAAS,CAACA,YAAWA,QAAO,aAAa,QAAQ,CAAC,GAApF;AAAA,MACN,wBAAwB,wBAAC,aACvB,YAAY,SAAS,CAACA,YAAWA,QAAO,OAAO,mBAAmB,QAAQ,CAAC,GADrD;AAAA,MAExB,qBAAqB,wBAAC,aACpB,YAAY,SAAS,CAACA,YAAWA,QAAO,OAAO,gBAAgB,QAAQ,CAAC,GADrD;AAAA,MAErB,aAAa,wBAAC,KAAa,QAAgB,YAAY,SAAS,CAACA,YAAWA,QAAO,OAAO,OAAO,KAAK,GAAG,CAAC,GAA7F;AAAA,IACf;AAAA,EACF;AACF;AA1EgB;;;ACXhB,SAAS,KAAAC,UAAS;AAYX,SAAS,KAAsB,OAAU,WAAmB,SAA0B;AAZ7F;AAaE,QAAM,SAASC,GAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,SAAS,gEAAgE;AACjH,QAAM,UAAU,YAAY;AAAA,IAC1B,eAAe;AAAA;AAAA,YAAwL,SAAS;AAAA,IAChN,WAAU,wCAAS,aAAT,mBAAmB,IAAI,CAAC,EAAE,OAAAC,QAAO,QAAQ,OAAO,OAAO;AAAA,MAC/D,OAAOA;AAAA,MACP,QAAQ,EAAE,QAAQ,QAAQ,OAAO;AAAA,IACnC;AAAA,IACA,cAAc;AAAA,IACd,OAAO,QAAQ;AAAA,IACf,OAAO;AAAA,EACT,CAAC,EAAE,KAAK,CAAC,MAAM;AACb,WAAO;AAAA,MACL,QAAQ,OAAO,EAAE,WAAW,WAAW,EAAE,SAAS,SAAS,EAAE,QAAQ,EAAE;AAAA,MACvE,QAAQ,EAAE;AAAA,IACZ;AAAA,EACF,CAAC;AAED,SAAO,iCACF,YAAY,OAAO,IADjB;AAAA,IAEL,MAAM,wBAAC,aAAqB,YAAY,SAAS,CAACC,YAAWA,QAAO,QAAQ,QAAQ,CAAC,GAA/E;AAAA,IACN,uBAAuB,0BAA0B,OAAO;AAAA,IACxD,wBAAwB,wBAAC,aACvB,YAAY,SAAS,CAACA,YAAWA,QAAO,uBAAuB,QAAQ,CAAC,GADlD;AAAA,IAExB,qBAAqB,wBAAC,aACpB,YAAY,SAAS,CAACA,YAAWA,QAAO,oBAAoB,QAAQ,CAAC,GADlD;AAAA,EAEvB;AACF;AA3BgB;;;ACZhB,SAAS,kBAAAC,uBAAsB;AAGxB,IAAM,eAAe,wBAAC,UAA0B;AACrD,QAAM,OAAOC,gBAAe;AAE5B,MAAI,CAAC,MAAM;AACT,UAAM,IAAI,MAAM,gEAAgE;AAAA,EAClF;AAEA,QAAM,OAAO,KAAK;AAClB,OAAK,iBAAiB;AACxB,GAT4B;;;ACArB,IAAM,cAAc,wBAAC,WAAmB;AAC7C,UAAQ,UAAU,MAAM;AAC1B,GAF2B;","names":["fn","z","getCurrentTest","getCurrentTest","z","z","output","expect","getCurrentTest","expect","getCurrentTest","value","z","expect","value","expect","z","z","_a","expect","z","z","value","expect","getCurrentTest","getCurrentTest"]}

package/ensure-env.cjs

@@ -0,0 +1,9 @@
+;(function () {
+  if (!process.env.CLOUD_BOT_ID) {
+    throw new Error('Env: CLOUD_BOT_ID is required')
+  }
+
+  if (!process.env.CLOUD_PAT) {
+    throw new Error('Env: CLOUD_PAT is required')
+  }
+})()

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@botpress/vai",
+  "version": "0.0.1-beta.1",
+  "type": "module",
+  "description": "Vitest AI (vai) – a vitest extension for testing with LLMs",
+  "main": "src/index.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/src/index.js",
+      "require": "./dist/src/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsup",
+    "watch": "tsup --watch",
+    "test": "vitest run --config vitest.config.ts",
+    "test:update": "vitest -u run --config vitest.config.ts",
+    "test:watch": "vitest --config vitest.config.ts",
+    "build-with-latest-models": "pnpm run update-types && pnpm run update-models && pnpm run build",
+    "update-models": "ts-node ./src/scripts/update-models.ts",
+    "update-types": "ts-node ./src/scripts/update-types.ts"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "json5": "^2.2.1",
+    "jsonrepair": "^3.2.0",
+    "lodash": "^4.17.21"
+  },
+  "devDependencies": {
+    "@types/lodash": "^4.17.0",
+    "dotenv": "^16.3.1",
+    "ts-node": "^10.9.2",
+    "tsup": "^8.3.5",
+    "typescript": "^5.7.2"
+  },
+  "peerDependencies": {
+    "@botpress/client": "^0.36.2",
+    "@botpress/sdk": "^1.6.1",
+    "@botpress/wasm": "^1.0.0",
+    "vitest": "^2.0.5"
+  }
+}

package/src/assertions/check.ts

@@ -0,0 +1,28 @@
+import { z } from '@botpress/sdk'
+import { Context } from '../context'
+import { asyncExpect } from '../utils/asyncAssertion'
+import { Input, predictJson } from '../utils/predictJson'
+import { makeToMatchInlineSnapshot, toAssertion } from './extension'
+
+export type CheckOptions<T> = {
+  examples?: { value: T; expected: boolean; reason: string }[]
+}
+
+export function check<T extends Input>(value: T, condition: string, options?: CheckOptions<T>) {
+  const promise = predictJson({
+    systemMessage: `Check that the value meets the condition: ${condition}`,
+    examples: options?.examples?.map(({ value, reason, expected }) => ({
+      input: value,
+      output: { reason, result: expected }
+    })),
+    outputSchema: z.boolean(),
+    model: Context.evaluatorModel,
+    input: value
+  })
+
+  return {
+    ...toAssertion(promise),
+    toBe: (expected: boolean) => asyncExpect(promise, (expect) => expect.toEqual(expected)),
+    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise)
+  }
+}