@botpress/vai 0.0.1 → 0.0.2-beta.1

package/dist/index.js

@@ -1,473 +1,8 @@
-var __defProp = Object.defineProperty;
-var __defProps = Object.defineProperties;
-var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
-var __getOwnPropSymbols = Object.getOwnPropertySymbols;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __propIsEnum = Object.prototype.propertyIsEnumerable;
-var __typeError = (msg) => {
-  throw TypeError(msg);
-};
-var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
-var __spreadValues = (a, b) => {
-  for (var prop in b || (b = {}))
-    if (__hasOwnProp.call(b, prop))
-      __defNormalProp(a, prop, b[prop]);
-  if (__getOwnPropSymbols)
-    for (var prop of __getOwnPropSymbols(b)) {
-      if (__propIsEnum.call(b, prop))
-        __defNormalProp(a, prop, b[prop]);
-    }
-  return a;
-};
-var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
-var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
-var __accessCheck = (obj, member, msg) => member.has(obj) || __typeError("Cannot " + msg);
-var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read from private field"), getter ? getter.call(obj) : member.get(obj));
-var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
-var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), setter ? setter.call(obj, value) : member.set(obj, value), value);
-
-// src/utils/zui.ts
-import sdk from "@botpress/sdk";
-var z = sdk.z;
-
-// src/task/compare.ts
-import { createTaskCollector, getCurrentSuite } from "vitest/suite";
-
-// src/utils/deferred.ts
-var _Deferred = class _Deferred {
-  constructor() {
-    this.promise = new Promise((resolve, reject) => {
-      this._resolve = resolve;
-      this._reject = reject;
-    });
-  }
-  resolve(value) {
-    this._resolve(value);
-  }
-  reject(reason) {
-    this._reject(reason);
-  }
-};
-__name(_Deferred, "Deferred");
-var Deferred = _Deferred;
-
-// src/task/compare.ts
-var scenarioId = z.string().trim().min(1, "Scenario ID/name must not be empty").max(50, "Scenario ID/name is too long");
-var ScenarioLike = z.union([
-  scenarioId,
-  z.object({ name: scenarioId }).passthrough(),
-  z.object({ id: scenarioId }).passthrough()
-]);
-var getScenarioName = /* @__PURE__ */ __name((scenario) => typeof scenario === "string" ? scenario : "name" in scenario ? scenario == null ? void 0 : scenario.name : scenario == null ? void 0 : scenario.id, "getScenarioName");
-var scenarioArgs = z.array(ScenarioLike).min(2, "You need at least two scenarios to compare").max(10, "You can only compare up to 10 scenarios").refine((scenarios) => {
-  const set = /* @__PURE__ */ new Set();
-  scenarios.forEach((scenario) => set.add(getScenarioName(scenario)));
-  return set.size === scenarios.length;
-}, "Scenarios names must be unique");
-function compare(name, scenarios, fn) {
-  scenarios = scenarioArgs.parse(scenarios);
-  return createTaskCollector((_name, fn2, timeout) => {
-    const currentSuite = getCurrentSuite();
-    let completedCount = 0;
-    const finished = new Deferred();
-    for (const scenario of scenarios) {
-      const key = getScenarioName(scenario);
-      currentSuite.task(key, {
-        meta: {
-          scenario: key,
-          isVaiTest: true
-        },
-        handler: /* @__PURE__ */ __name(async (context) => {
-          const extendedContext = Object.freeze({
-            scenario
-          });
-          context.onTestFinished(() => {
-            if (++completedCount === scenarios.length) {
-              finished.resolve();
-            }
-          });
-          await fn2(__spreadValues(__spreadValues({}, context), extendedContext));
-        }, "handler"),
-        timeout: timeout != null ? timeout : 1e4
-      });
-    }
-  })(name, fn);
-}
-__name(compare, "compare");
-
-// src/context.ts
-import { onTestFinished } from "vitest";
-import { getCurrentTest } from "vitest/suite";
-var getTestMetadata = /* @__PURE__ */ __name(() => {
-  var _a;
-  const test = getCurrentTest();
-  return (_a = test == null ? void 0 : test.meta) != null ? _a : {
-    isVaiTest: false
-  };
-}, "getTestMetadata");
-var _client, _wrapError;
-var _VaiContext = class _VaiContext {
-  constructor() {
-    __privateAdd(this, _client, null);
-    __privateAdd(this, _wrapError, false);
-  }
-  get wrapError() {
-    return __privateGet(this, _wrapError);
-  }
-  get client() {
-    if (!__privateGet(this, _client)) {
-      throw new Error("Botpress client is not set");
-    }
-    return __privateGet(this, _client);
-  }
-  get evaluatorModel() {
-    var _a;
-    return (_a = getTestMetadata().evaluatorModel) != null ? _a : "openai__gpt-4o-mini-2024-07-18";
-  }
-  get scenario() {
-    return getTestMetadata().scenario;
-  }
-  get isVaiTest() {
-    return getTestMetadata().isVaiTest;
-  }
-  setClient(cognitive) {
-    __privateSet(this, _client, cognitive);
-  }
-  swallowErrors() {
-    if (!getCurrentTest()) {
-      throw new Error("cancelBail is a Vitest hook and must be called within a test");
-    }
-    __privateSet(this, _wrapError, true);
-    onTestFinished(() => {
-      __privateSet(this, _wrapError, false);
-    });
-  }
-};
-_client = new WeakMap();
-_wrapError = new WeakMap();
-__name(_VaiContext, "VaiContext");
-var VaiContext = _VaiContext;
-var Context = new VaiContext();
-
-// src/utils/asyncAssertion.ts
-import { expect } from "vitest";
-import { getCurrentTest as getCurrentTest2 } from "vitest/suite";
-var _AsyncExpectError = class _AsyncExpectError extends Error {
-  constructor(message, output) {
-    super(message);
-    this.output = output;
-    this.name = "AsyncExpectError";
-  }
-};
-__name(_AsyncExpectError, "AsyncExpectError");
-var AsyncExpectError = _AsyncExpectError;
-var getErrorMessages = /* @__PURE__ */ __name((e) => {
-  if (e instanceof Error) {
-    return e.message;
-  } else if (typeof e === "string") {
-    return e;
-  } else if (typeof e === "object" && e !== null) {
-    return JSON.stringify(e);
-  }
-  return `Unknown error: ${e}`;
-}, "getErrorMessages");
-var asyncExpect = /* @__PURE__ */ __name((output, assertion) => {
-  var _a, _b;
-  const promise = output.then((x) => {
-    try {
-      assertion(expect(x.result, x.reason));
-    } catch (e) {
-      if (Context.wrapError) {
-        return new AsyncExpectError(getErrorMessages(e), x);
-      }
-      throw e;
-    }
-    return x;
-  });
-  (_b = (_a = getCurrentTest2()).promises) != null ? _b : _a.promises = [];
-  getCurrentTest2().promises.push(promise);
-  return promise;
-}, "asyncExpect");
-
-// src/utils/predictJson.ts
-import JSON5 from "json5";
-var nonEmptyString = z.string().trim().min(1);
-var nonEmptyObject = z.object({}).passthrough().refine((value) => Object.keys(value).length > 0, {
-  message: "Expected a non-empty object"
-});
-var Input = nonEmptyString.or(nonEmptyObject).or(z.array(z.any()));
-var Output = z.object({
-  reason: nonEmptyString.describe("A human-readable explanation of the result"),
-  result: z.any().describe(
-    "Your best guess at the output according to the instructions provided, rooted in the context of the input and the reason above"
-  )
-});
-var Example = z.object({
-  input: Input,
-  output: Output
-});
-var Options = z.object({
-  systemMessage: z.string(),
-  examples: z.array(Example).default([]),
-  input: Input,
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  outputSchema: z.custom((value) => typeof value === "object" && value !== null && "_def" in value),
-  model: z.string()
-});
-var isValidExample = /* @__PURE__ */ __name((outputSchema) => (example) => Input.safeParse(example.input).success && Output.safeParse(example.output).success && outputSchema.safeParse(example.output.result).success, "isValidExample");
-async function predictJson(_options) {
-  var _a, _b;
-  const options = Options.parse(_options);
-  const [integration, model] = options.model.split("__");
-  if (!(model == null ? void 0 : model.length)) {
-    throw new Error("Invalid model");
-  }
-  const exampleMessages = options.examples.filter(isValidExample(options.outputSchema)).flatMap(({ input, output: output2 }) => [
-    { role: "user", content: JSON.stringify(input, null, 2) },
-    { role: "assistant", content: JSON.stringify(output2, null, 2) }
-  ]);
-  const outputSchema = Output.extend({
-    result: options.outputSchema.describe(Output.shape.result.description)
-  });
-  const result = await Context.client.callAction({
-    type: `${integration}:generateContent`,
-    input: {
-      systemPrompt: `
-${options.systemMessage}
-
----
-Please generate a JSON response with the following format:
-\`\`\`typescript
-${await outputSchema.toTypescriptAsync()}
-\`\`\`
-`.trim(),
-      messages: [
-        ...exampleMessages,
-        {
-          role: "user",
-          content: JSON.stringify(options.input, null, 2)
-        }
-      ],
-      temperature: 0,
-      responseFormat: "json_object",
-      model: { id: model }
-    }
-  });
-  const output = result.output;
-  if (!output.choices.length || typeof ((_b = (_a = output.choices) == null ? void 0 : _a[0]) == null ? void 0 : _b.content) !== "string") {
-    throw new Error("Invalid response from the model");
-  }
-  const json = output.choices[0].content.trim();
-  if (!json.length) {
-    throw new Error("No response from the model");
-  }
-  return outputSchema.parse(JSON5.parse(json));
-}
-__name(predictJson, "predictJson");
-
-// src/assertions/extension.ts
-import json5 from "json5";
-import { expect as expect2 } from "vitest";
-import { getCurrentTest as getCurrentTest3 } from "vitest/suite";
-var toAssertion = /* @__PURE__ */ __name((promise) => {
-  return {
-    then: promise.then.bind(promise),
-    value: promise.then((value) => value.result)
-  };
-}, "toAssertion");
-var makeToMatchInlineSnapshot = /* @__PURE__ */ __name((promise) => async (expected) => {
-  const stack = new Error().stack.split("\n")[2];
-  const newStack = `
-at __INLINE_SNAPSHOT__ (node:internal/process/task_queues:1:1)
-at randomLine (node:internal/process/task_queues:1:1)
-${stack}
-`.trim();
-  const obj = json5.parse(expected != null ? expected : '""');
-  const expectation = asyncExpect(promise, (expect3) => expect3.toMatchObject(obj)).catch(() => {
-  });
-  try {
-    expect2((await promise).result).toMatchObject(obj);
-  } catch (err) {
-    const newError = new Error();
-    newError.stack = newStack;
-    expect2.getState().snapshotState.match({
-      isInline: true,
-      received: (await promise).result,
-      testName: getCurrentTest3().name,
-      error: newError,
-      inlineSnapshot: expected
-    });
-  }
-  return expectation;
-}, "makeToMatchInlineSnapshot");
-
-// src/assertions/check.ts
-function check(value, condition, options) {
-  var _a;
-  const promise = predictJson({
-    systemMessage: `Check that the value meets the condition: ${condition}`,
-    examples: (_a = options == null ? void 0 : options.examples) == null ? void 0 : _a.map(({ value: value2, reason, expected }) => ({
-      input: value2,
-      output: { reason, result: expected }
-    })),
-    outputSchema: z.boolean(),
-    model: Context.evaluatorModel,
-    input: value
-  });
-  return __spreadProps(__spreadValues({}, toAssertion(promise)), {
-    toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toEqual(expected)), "toBe"),
-    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise)
-  });
-}
-__name(check, "check");
-
-// src/assertions/extract.ts
-function extract(value, shape, options) {
-  var _a;
-  const additionalMessage = (options == null ? void 0 : options.description) ? `
-In order to extract the right information, follow these instructions:
-${options.description}
-` : "";
-  const promise = predictJson({
-    systemMessage: "From the given input, extract the required information into the requested format." + additionalMessage.trim(),
-    examples: (_a = options == null ? void 0 : options.examples) == null ? void 0 : _a.map(({ value: value2, reason, extracted }) => ({
-      input: value2,
-      output: { reason, result: extracted }
-    })),
-    outputSchema: shape,
-    model: Context.evaluatorModel,
-    input: value
-  });
-  return __spreadProps(__spreadValues({}, toAssertion(promise)), {
-    toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toEqual(expected)), "toBe"),
-    toMatchObject: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toMatchObject(expected)), "toMatchObject"),
-    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise)
-  });
-}
-__name(extract, "extract");
-
-// src/assertions/filter.ts
-function filter(values, condition, options) {
-  var _a, _b;
-  const mappedValues = values.map(
-    (_, idx) => z.object({
-      index: z.literal(idx),
-      reason: z.string(),
-      keep: z.boolean()
-    })
-  );
-  const input = values.map((value, idx) => ({
-    index: idx,
-    value
-  }));
-  const schema = z.tuple(mappedValues).describe(
-    "An array of the objects with the index and a boolean value indicating if the object should be kept or not"
-  );
-  const promise = predictJson({
-    systemMessage: `
-Based on the following qualification criteria, you need to filter the given list of objects.
-Citeria: ${condition}
-
----
-You need to return an array of objects with the index and a boolean value indicating if the object should be kept or not.
-`.trim(),
-    examples: (options == null ? void 0 : options.examples) ? [
-      {
-        input: (_a = options == null ? void 0 : options.examples) == null ? void 0 : _a.map((v, index) => ({
-          index,
-          value: v.value
-        })),
-        output: {
-          reason: "Here are some examples",
-          result: (_b = options == null ? void 0 : options.examples) == null ? void 0 : _b.map((v, idx) => ({
-            index: idx,
-            reason: v.reason,
-            keep: v.keep
-          }))
-        }
-      }
-    ] : void 0,
-    outputSchema: schema,
-    model: Context.evaluatorModel,
-    input
-  }).then((x) => {
-    const results = schema.parse(x.result);
-    return {
-      result: values.filter((_, idx) => {
-        var _a2;
-        return (_a2 = results.find((r) => r.index === idx)) == null ? void 0 : _a2.keep;
-      }),
-      reason: x.reason
-    };
-  });
-  return __spreadProps(__spreadValues({}, toAssertion(promise)), {
-    toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toEqual(expected)), "toBe"),
-    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise),
-    toHaveNoneFiltered: /* @__PURE__ */ __name(() => asyncExpect(promise, (expect3) => expect3.toEqual(values)), "toHaveNoneFiltered"),
-    toHaveSomeFiltered: /* @__PURE__ */ __name(() => asyncExpect(promise, (expect3) => expect3.not.toEqual(values)), "toHaveSomeFiltered"),
-    toBeEmpty: /* @__PURE__ */ __name(() => asyncExpect(promise, (expect3) => expect3.toHaveLength(0)), "toBeEmpty"),
-    length: {
-      toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toHaveLength(expected)), "toBe"),
-      toBeGreaterThanOrEqual: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.length.greaterThanOrEqual(expected)), "toBeGreaterThanOrEqual"),
-      toBeLessThanOrEqual: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.length.lessThanOrEqual(expected)), "toBeLessThanOrEqual"),
-      toBeBetween: /* @__PURE__ */ __name((min, max) => asyncExpect(promise, (expect3) => expect3.length.within(min, max)), "toBeBetween")
-    }
-  });
-}
-__name(filter, "filter");
-
-// src/assertions/rate.ts
-function rate(value, condition, options) {
-  var _a;
-  const schema = z.number().min(1).max(5).describe("Rating score, higher is better (1 is the worst, 5 is the best)");
-  const promise = predictJson({
-    systemMessage: `Based on the following qualification criteria, you need to rate the given situation from a score of 1 to 5.
-Scoring: 1 is the worst score, 5 is the best score possible.
-Criteria: ${condition}`,
-    examples: (_a = options == null ? void 0 : options.examples) == null ? void 0 : _a.map(({ value: value2, reason, rating }) => ({
-      input: value2,
-      output: { reason, result: rating }
-    })),
-    outputSchema: schema,
-    model: Context.evaluatorModel,
-    input: value
-  }).then((x) => {
-    return {
-      result: typeof x.result === "number" ? x.result : parseInt(x.result, 10),
-      reason: x.reason
-    };
-  });
-  return __spreadProps(__spreadValues({}, toAssertion(promise)), {
-    toBe: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toEqual(expected)), "toBe"),
-    toMatchInlineSnapshot: makeToMatchInlineSnapshot(promise),
-    toBeGreaterThanOrEqual: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toBeGreaterThanOrEqual(expected)), "toBeGreaterThanOrEqual"),
-    toBeLessThanOrEqual: /* @__PURE__ */ __name((expected) => asyncExpect(promise, (expect3) => expect3.toBeLessThanOrEqual(expected)), "toBeLessThanOrEqual")
-  });
-}
-__name(rate, "rate");
-
-// src/hooks/setEvaluator.ts
-import { getCurrentTest as getCurrentTest4 } from "vitest/suite";
-var setEvaluator = /* @__PURE__ */ __name((model) => {
-  const test = getCurrentTest4();
-  if (!test) {
-    throw new Error("setEvaluator is a Vitest hook and must be called within a test");
-  }
-  const meta = test.meta;
-  meta.evaluatorModel = model;
-}, "setEvaluator");
-
-// src/hooks/setupClient.ts
-var setupClient = /* @__PURE__ */ __name((client) => {
-  Context.setClient(client);
-}, "setupClient");
-export {
-  check,
-  compare,
-  extract,
-  filter,
-  rate,
-  setEvaluator,
-  setupClient
-};
-//# sourceMappingURL=index.js.map
+"use strict";
+export { compare } from "./task/compare";
+export { check } from "./assertions/check";
+export { extract } from "./assertions/extract";
+export { filter } from "./assertions/filter";
+export { rate } from "./assertions/rate";
+export { setEvaluator } from "./hooks/setEvaluator";
+export { setupClient } from "./hooks/setupClient";