@lmnr-ai/lmnr 0.4.25 → 0.4.27

package/.github/workflows/run-tests.yml

@@ -0,0 +1,15 @@
+name: Run Tests
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    branches:
+      - main
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: npm install
+      - run: npm test

package/dist/cli.js

@@ -36,13 +36,13 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@lmnr-ai/lmnr",
-      version: "0.4.25",
+      version: "0.4.27",
       description: "TypeScript SDK for Laminar AI",
       main: "dist/index.js",
       types: "dist/index.d.ts",
       scripts: {
         build: "tsup src/index.ts src/cli.ts --format esm,cjs --dts",
-        test: 'echo "Error: no test specified" && exit 1'
+        test: "tsx --test test/*.test.ts"
       },
       bin: {
         lmnr: "./dist/cli.js"
@@ -84,6 +84,7 @@ var require_package = __commonJS({
         llamaindex: "^0.7.10",
         openai: "^4.75.0",
         tsup: "^8.3.5",
+        tsx: "^4.19.2",
         typescript: "^5.7.2"
       },
       dependencies: {
@@ -112,6 +113,7 @@ var require_package = __commonJS({
         argparse: "^2.0.1",
         "cli-progress": "^3.12.0",
         esbuild: "^0.24.0",
+        glob: "^11.0.0",
         uuid: "^11.0.3"
       }
     };
@@ -126,6 +128,7 @@ __export(cli_exports, {
 module.exports = __toCommonJS(cli_exports);
 var import_argparse = require("argparse");
 var esbuild = __toESM(require("esbuild"));
+var glob = __toESM(require("glob"));
 var pjson = require_package();
 function loadModule({
   filename,
@@ -140,41 +143,75 @@ async function cli() {
   const [, , ...args] = process.argv;
   const parser = new import_argparse.ArgumentParser({
     prog: "lmnr",
-    description: "CLI for Laminar"
+    description: "CLI for Laminar. Use `lmnr <subcommand> --help` for more information."
   });
   parser.add_argument("-v", "--version", { action: "version", version: pjson.version });
   const subparsers = parser.add_subparsers({
     title: "subcommands",
     dest: "subcommand"
   });
-  const parser_eval = subparsers.add_parser("eval", {
-    description: "Run an evaluation"
+  const parserEval = subparsers.add_parser("eval", {
+    description: "Run an evaluation",
+    help: "Run an evaluation"
   });
-  parser_eval.add_argument("file", {
-    help: "A file containing the evaluation to run"
+  parserEval.add_argument("file", {
+    help: "A file containing the evaluation to run. If no file is provided, the evaluation will run all `*.eval.ts|js` files in the `evals` directory.",
+    nargs: "?"
   });
-  parser_eval.set_defaults({
+  parserEval.add_argument("--fail-on-error", {
+    help: "Fail on error. If specified, will fail if encounters a file that cannot be run",
+    action: "store_true"
+  });
+  parserEval.set_defaults({
     func: async (args2) => {
-      const buildOptions = {
-        entryPoints: [args2.file],
-        outfile: "tmp_out.js",
-        write: false,
-        // will be loaded in memory as a temp file
-        platform: "node",
-        bundle: true,
-        external: ["node_modules/*"]
-      };
-      const result = await esbuild.build(buildOptions);
-      if (!result.outputFiles) {
-        console.error("Error when building: No output files found");
+      const files = args2.file ? [args2.file] : glob.sync("evals/**/*.eval.{ts,js}");
+      files.sort();
+      if (files.length === 0) {
+        console.error("No evaluation files found. Please provide a file or ensure there are files in the `evals` directory.");
         process.exit(1);
       }
-      const outputFileText = result.outputFiles[0].text;
-      const evaluation = loadModule({
-        filename: args2.file,
-        moduleText: outputFileText
-      });
-      await evaluation.run();
+      if (!args2.file) {
+        console.log(`Located ${files.length} evaluation files in evals/`);
+      }
+      for (const file of files) {
+        console.log(`Loading ${file}...`);
+        const buildOptions = {
+          entryPoints: [file],
+          outfile: `tmp_out_${file}.js`,
+          write: false,
+          // will be loaded in memory as a temp file
+          platform: "node",
+          bundle: true,
+          external: ["node_modules/*"]
+        };
+        const result = await esbuild.build(buildOptions);
+        if (!result.outputFiles) {
+          console.error("Error when building: No output files found");
+          if (args2.fail_on_error) {
+            process.exit(1);
+          }
+          continue;
+        }
+        const outputFileText = result.outputFiles[0].text;
+        const evaluation = loadModule({
+          filename: args2.file,
+          moduleText: outputFileText
+        });
+        if (!evaluation?.run) {
+          console.error(`Evaluation ${file} does not properly call evaluate()`);
+          if (args2.fail_on_error) {
+            process.exit(1);
+          }
+          continue;
+        }
+        await evaluation.run();
+      }
+    }
+  });
+  parser.set_defaults({
+    func: async (args2) => {
+      parser.print_help();
+      process.exit(0);
     }
   });
   const parsed = parser.parse_args(args);

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n  \"name\": \"@lmnr-ai/lmnr\",\n  \"version\": \"0.4.25\",\n  \"description\": \"TypeScript SDK for Laminar AI\",\n  \"main\": \"dist/index.js\",\n  \"types\": \"dist/index.d.ts\",\n  \"scripts\": {\n    \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n    \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n  },\n  \"bin\": {\n    \"lmnr\": \"./dist/cli.js\"\n  },\n  \"repository\": {\n    \"type\": \"git\",\n    \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n  },\n  \"keywords\": [\n    \"laminar\",\n    \"lmnr\",\n    \"sdk\",\n    \"lmnr.ai\"\n  ],\n  \"author\": \"founders@lmnr.ai\",\n  \"license\": \"Apache-2.0\",\n  \"bugs\": {\n    \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n  },\n  \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n  \"devDependencies\": {\n    \"@anthropic-ai/sdk\": \"^0.27.3\",\n    \"@aws-sdk/client-bedrock-runtime\": \"^3.705.0\",\n    \"@azure/openai\": \"1.0.0-beta.13\",\n    \"@google-cloud/aiplatform\": \"^3.34.0\",\n    \"@google-cloud/vertexai\": \"^1.9.0\",\n    \"@langchain/core\": \"^0.3.20\",\n    \"@pinecone-database/pinecone\": \"^3.0.3\",\n    \"@qdrant/js-client-rest\": \"^1.12.0\",\n    \"@types/argparse\": \"^2.0.17\",\n    \"@types/cli-progress\": \"^3.11.6\",\n    \"@types/node\": \"^22.10.1\",\n    \"@types/semver\": \"^7.5.8\",\n    \"@types/uuid\": \"^10.0.0\",\n    \"bufferutil\": \"^4.0.8\",\n    \"chromadb\": \"^1.9.4\",\n    \"cohere-ai\": \"^7.15.0\",\n    \"langchain\": \"^0.3.6\",\n    \"llamaindex\": \"^0.7.10\",\n    \"openai\": \"^4.75.0\",\n    \"tsup\": \"^8.3.5\",\n    \"typescript\": \"^5.7.2\"\n  },\n  \"dependencies\": {\n    \"@grpc/grpc-js\": \"^1.12.3\",\n    \"@opentelemetry/api\": \"^1.9.0\",\n    \"@opentelemetry/core\": \"^1.28.0\",\n    \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.53.0\",\n    \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.53.0\",\n    \"@opentelemetry/instrumentation\": \"^0.53.0\",\n    \"@opentelemetry/sdk-node\": \"^0.53.0\",\n    \"@opentelemetry/sdk-trace-base\": \"^1.28.0\",\n    \"@opentelemetry/sdk-trace-node\": \"^1.28.0\",\n    \"@opentelemetry/semantic-conventions\": \"^1.28.0\",\n    \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n    \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n    \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n    \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n    \"@traceloop/instrumentation-chromadb\": \"^0.11.3\",\n    \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n    \"@traceloop/instrumentation-langchain\": \"^0.11.4\",\n    \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n    \"@traceloop/instrumentation-openai\": \"^0.11.3\",\n    \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n    \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n    \"@traceloop/instrumentation-vertexai\": \"^0.11.4\",\n    \"argparse\": \"^2.0.1\",\n    \"cli-progress\": \"^3.12.0\",\n    \"esbuild\": \"^0.24.0\",\n    \"uuid\": \"^11.0.3\"\n  }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n  filename,\n  moduleText,\n}: {\n  filename: string;\n  moduleText: string;\n}) {\n  // TODO: Figure out how to remove all ts-ignores\n  // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n  // @ts-ignore\n  globalThis._evaluation = undefined;  // @ts-ignore\n  globalThis._set_global_evaluation = true;  // @ts-ignore\n\n  // it needs \"require\" to be passed in\n  new Function(\"require\", moduleText)(require);\n\n  // Return the modified _evals global variable\n  // @ts-ignore\n  return globalThis._evaluation;\n}\n\nasync function cli() {\n  const [, , ...args] = process.argv;\n\n  // Use argparse, which is the port of the python library\n  const parser = new ArgumentParser({\n    prog: \"lmnr\",\n    description: \"CLI for Laminar\",\n  });\n\n  parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n  const subparsers = parser.add_subparsers({\n    title: \"subcommands\",\n    dest: \"subcommand\",\n  });\n\n  const parser_eval = subparsers.add_parser(\"eval\", {\n    description: \"Run an evaluation\",\n  });\n\n  parser_eval.add_argument(\"file\", {\n    help: \"A file containing the evaluation to run\",\n  });\n\n  parser_eval.set_defaults({\n    func: async (args: any) => {\n      // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n      const buildOptions = {\n        entryPoints: [args.file],\n        outfile: \"tmp_out.js\",\n        write: false,  // will be loaded in memory as a temp file\n        platform: \"node\" as esbuild.Platform,\n        bundle: true,\n        external: [\"node_modules/*\"],\n      };\n\n      const result = await esbuild.build(buildOptions);\n\n      if (!result.outputFiles) {\n        console.error(\"Error when building: No output files found\");\n        process.exit(1);\n      }\n\n      const outputFileText = result.outputFiles[0].text;\n\n      const evaluation = loadModule({\n        filename: args.file,\n        moduleText: outputFileText,\n      });\n\n      // @ts-ignore\n      await evaluation.run();\n    }\n  });\n\n  const parsed = parser.parse_args(args);\n  await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,iBAAAA,UAAAC,SAAA;AAAA,IAAAA,QAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,MAAQ;AAAA,QACR,YAAc;AAAA,MAChB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;AChFA;AAAA;AAAA;AAAA;AAAA;AAEA,sBAA+B;AAC/B,cAAyB;AAEzB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,OAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,cAAc,WAAW,WAAW,QAAQ;AAAA,IAChD,aAAa;AAAA,EACf,CAAC;AAED,cAAY,aAAa,QAAQ;AAAA,IAC/B,MAAM;AAAA,EACR,CAAC;AAED,cAAY,aAAa;AAAA,IACvB,MAAM,OAAOC,UAAc;AAEzB,YAAM,eAAe;AAAA,QACnB,aAAa,CAACA,MAAK,IAAI;AAAA,QACvB,SAAS;AAAA,QACT,OAAO;AAAA;AAAA,QACP,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU,CAAC,gBAAgB;AAAA,MAC7B;AAEA,YAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,UAAI,CAAC,OAAO,aAAa;AACvB,gBAAQ,MAAM,4CAA4C;AAC1D,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,YAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,YAAM,aAAa,WAAW;AAAA,QAC5B,UAAUA,MAAK;AAAA,QACf,YAAY;AAAA,MACd,CAAC;AAGD,YAAM,WAAW,IAAI;AAAA,IACvB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI;","names":["exports","module","args"]}
+{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n  \"name\": \"@lmnr-ai/lmnr\",\n  \"version\": \"0.4.27\",\n  \"description\": \"TypeScript SDK for Laminar AI\",\n  \"main\": \"dist/index.js\",\n  \"types\": \"dist/index.d.ts\",\n  \"scripts\": {\n    \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n    \"test\": \"tsx --test test/*.test.ts\"\n  },\n  \"bin\": {\n    \"lmnr\": \"./dist/cli.js\"\n  },\n  \"repository\": {\n    \"type\": \"git\",\n    \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n  },\n  \"keywords\": [\n    \"laminar\",\n    \"lmnr\",\n    \"sdk\",\n    \"lmnr.ai\"\n  ],\n  \"author\": \"founders@lmnr.ai\",\n  \"license\": \"Apache-2.0\",\n  \"bugs\": {\n    \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n  },\n  \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n  \"devDependencies\": {\n    \"@anthropic-ai/sdk\": \"^0.27.3\",\n    \"@aws-sdk/client-bedrock-runtime\": \"^3.705.0\",\n    \"@azure/openai\": \"1.0.0-beta.13\",\n    \"@google-cloud/aiplatform\": \"^3.34.0\",\n    \"@google-cloud/vertexai\": \"^1.9.0\",\n    \"@langchain/core\": \"^0.3.20\",\n    \"@pinecone-database/pinecone\": \"^3.0.3\",\n    \"@qdrant/js-client-rest\": \"^1.12.0\",\n    \"@types/argparse\": \"^2.0.17\",\n    \"@types/cli-progress\": \"^3.11.6\",\n    \"@types/node\": \"^22.10.1\",\n    \"@types/semver\": \"^7.5.8\",\n    \"@types/uuid\": \"^10.0.0\",\n    \"bufferutil\": \"^4.0.8\",\n    \"chromadb\": \"^1.9.4\",\n    \"cohere-ai\": \"^7.15.0\",\n    \"langchain\": \"^0.3.6\",\n    \"llamaindex\": \"^0.7.10\",\n    \"openai\": \"^4.75.0\",\n    \"tsup\": \"^8.3.5\",\n    \"tsx\": \"^4.19.2\",\n    \"typescript\": \"^5.7.2\"\n  },\n  \"dependencies\": {\n    \"@grpc/grpc-js\": \"^1.12.3\",\n    \"@opentelemetry/api\": \"^1.9.0\",\n    \"@opentelemetry/core\": \"^1.28.0\",\n    \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.53.0\",\n    \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.53.0\",\n    \"@opentelemetry/instrumentation\": \"^0.53.0\",\n    \"@opentelemetry/sdk-node\": \"^0.53.0\",\n    \"@opentelemetry/sdk-trace-base\": \"^1.28.0\",\n    \"@opentelemetry/sdk-trace-node\": \"^1.28.0\",\n    \"@opentelemetry/semantic-conventions\": \"^1.28.0\",\n    \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n    \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n    \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n    \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n    \"@traceloop/instrumentation-chromadb\": \"^0.11.3\",\n    \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n    \"@traceloop/instrumentation-langchain\": \"^0.11.4\",\n    \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n    \"@traceloop/instrumentation-openai\": \"^0.11.3\",\n    \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n    \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n    \"@traceloop/instrumentation-vertexai\": \"^0.11.4\",\n    \"argparse\": \"^2.0.1\",\n    \"cli-progress\": \"^3.12.0\",\n    \"esbuild\": \"^0.24.0\",\n    \"glob\": \"^11.0.0\",\n    \"uuid\": \"^11.0.3\"\n  }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n  filename,\n  moduleText,\n}: {\n  filename: string;\n  moduleText: string;\n}) {\n  // TODO: Figure out how to remove all ts-ignores\n  // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n  // @ts-ignore\n  globalThis._evaluation = undefined;  // @ts-ignore\n  globalThis._set_global_evaluation = true;  // @ts-ignore\n\n  // it needs \"require\" to be passed in\n  new Function(\"require\", moduleText)(require);\n\n  // Return the modified _evals global variable\n  // @ts-ignore\n  return globalThis._evaluation;\n}\n\nasync function cli() {\n  const [, , ...args] = process.argv;\n\n  // Use argparse, which is the port of the python library\n  const parser = new ArgumentParser({\n    prog: \"lmnr\",\n    description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n  });\n\n  parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n  const subparsers = parser.add_subparsers({\n    title: \"subcommands\",\n    dest: \"subcommand\",\n  });\n\n  const parserEval = subparsers.add_parser(\"eval\", {\n    description: \"Run an evaluation\",\n    help: \"Run an evaluation\",\n  });\n\n  parserEval.add_argument(\"file\", {\n    help: \"A file containing the evaluation to run. If no file is provided, \" +\n      \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n    nargs: \"?\",\n  });\n\n  parserEval.add_argument(\"--fail-on-error\", {\n    help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n    action: \"store_true\",\n  });\n\n  parserEval.set_defaults({\n    func: async (args: any) => {\n      const files = args.file\n        ? [args.file]\n        : glob.sync('evals/**/*.eval.{ts,js}')\n\n      files.sort();\n\n      if (files.length === 0) {\n        console.error(\"No evaluation files found. Please provide a file or \" +\n          \"ensure there are files in the `evals` directory.\");\n        process.exit(1);\n      }\n\n      if (!args.file) {\n        console.log(`Located ${files.length} evaluation files in evals/`);\n      }\n\n      for (const file of files) {\n        console.log(`Loading ${file}...`);\n        // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n        const buildOptions = {\n          entryPoints: [file],\n          outfile: `tmp_out_${file}.js`,\n          write: false,  // will be loaded in memory as a temp file\n          platform: \"node\" as esbuild.Platform,\n          bundle: true,\n          external: [\"node_modules/*\"],\n        };\n\n        const result = await esbuild.build(buildOptions);\n\n        if (!result.outputFiles) {\n          console.error(\"Error when building: No output files found\");\n          if (args.fail_on_error) {\n            process.exit(1);\n          }\n          continue;\n        }\n\n        const outputFileText = result.outputFiles[0].text;\n\n        const evaluation = loadModule({\n          filename: args.file,\n          moduleText: outputFileText,\n        });\n\n        // @ts-ignore\n        if (!evaluation?.run) {\n          console.error(`Evaluation ${file} does not properly call evaluate()`);\n          if (args.fail_on_error) {\n            process.exit(1);\n          }\n          continue;\n        }\n\n        // @ts-ignore\n        await evaluation.run();\n      }\n    }\n  });\n\n  parser.set_defaults({\n    func: async (args: any) => {\n      parser.print_help();\n      process.exit(0);\n    }\n  });\n\n  const parsed = parser.parse_args(args);\n  await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,iBAAAA,UAAAC,SAAA;AAAA,IAAAA,QAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,MAAQ;AAAA,QACR,KAAO;AAAA,QACP,YAAc;AAAA,MAChB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,QACR,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;AClFA;AAAA;AAAA;AAAA;AAAA;AAEA,sBAA+B;AAC/B,cAAyB;AACzB,WAAsB;AAEtB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,OAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,gBAAQ,MAAM,sGACsC;AACpD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACA,MAAK,MAAM;AACd,gBAAQ,IAAI,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAEA,iBAAW,QAAQ,OAAO;AACxB,gBAAQ,IAAI,WAAW,IAAI,KAAK;AAEhC,cAAM,eAAe;AAAA,UACnB,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,UACV,QAAQ;AAAA,UACR,UAAU,CAAC,gBAAgB;AAAA,QAC7B;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,kBAAQ,MAAM,4CAA4C;AAC1D,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,aAAa,WAAW;AAAA,UAC5B,UAAUA,MAAK;AAAA,UACf,YAAY;AAAA,QACd,CAAC;AAGD,YAAI,CAAC,YAAY,KAAK;AACpB,kBAAQ,MAAM,cAAc,IAAI,oCAAoC;AACpE,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAGA,cAAM,WAAW,IAAI;AAAA,MACvB;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,OAAOA,UAAc;AACzB,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI;","names":["exports","module","args"]}

package/dist/cli.mjs

@@ -15,13 +15,13 @@ var require_package = __commonJS({
   "package.json"(exports, module) {
     module.exports = {
       name: "@lmnr-ai/lmnr",
-      version: "0.4.25",
+      version: "0.4.27",
       description: "TypeScript SDK for Laminar AI",
       main: "dist/index.js",
       types: "dist/index.d.ts",
       scripts: {
         build: "tsup src/index.ts src/cli.ts --format esm,cjs --dts",
-        test: 'echo "Error: no test specified" && exit 1'
+        test: "tsx --test test/*.test.ts"
       },
       bin: {
         lmnr: "./dist/cli.js"
@@ -63,6 +63,7 @@ var require_package = __commonJS({
         llamaindex: "^0.7.10",
         openai: "^4.75.0",
         tsup: "^8.3.5",
+        tsx: "^4.19.2",
         typescript: "^5.7.2"
       },
       dependencies: {
@@ -91,6 +92,7 @@ var require_package = __commonJS({
         argparse: "^2.0.1",
         "cli-progress": "^3.12.0",
         esbuild: "^0.24.0",
+        glob: "^11.0.0",
         uuid: "^11.0.3"
       }
     };
@@ -100,6 +102,7 @@ var require_package = __commonJS({
 // src/cli.ts
 import { ArgumentParser } from "argparse";
 import * as esbuild from "esbuild";
+import * as glob from "glob";
 var pjson = require_package();
 function loadModule({
   filename,
@@ -114,41 +117,75 @@ async function cli() {
   const [, , ...args] = process.argv;
   const parser = new ArgumentParser({
     prog: "lmnr",
-    description: "CLI for Laminar"
+    description: "CLI for Laminar. Use `lmnr <subcommand> --help` for more information."
   });
   parser.add_argument("-v", "--version", { action: "version", version: pjson.version });
   const subparsers = parser.add_subparsers({
     title: "subcommands",
     dest: "subcommand"
   });
-  const parser_eval = subparsers.add_parser("eval", {
-    description: "Run an evaluation"
+  const parserEval = subparsers.add_parser("eval", {
+    description: "Run an evaluation",
+    help: "Run an evaluation"
   });
-  parser_eval.add_argument("file", {
-    help: "A file containing the evaluation to run"
+  parserEval.add_argument("file", {
+    help: "A file containing the evaluation to run. If no file is provided, the evaluation will run all `*.eval.ts|js` files in the `evals` directory.",
+    nargs: "?"
   });
-  parser_eval.set_defaults({
+  parserEval.add_argument("--fail-on-error", {
+    help: "Fail on error. If specified, will fail if encounters a file that cannot be run",
+    action: "store_true"
+  });
+  parserEval.set_defaults({
     func: async (args2) => {
-      const buildOptions = {
-        entryPoints: [args2.file],
-        outfile: "tmp_out.js",
-        write: false,
-        // will be loaded in memory as a temp file
-        platform: "node",
-        bundle: true,
-        external: ["node_modules/*"]
-      };
-      const result = await esbuild.build(buildOptions);
-      if (!result.outputFiles) {
-        console.error("Error when building: No output files found");
+      const files = args2.file ? [args2.file] : glob.sync("evals/**/*.eval.{ts,js}");
+      files.sort();
+      if (files.length === 0) {
+        console.error("No evaluation files found. Please provide a file or ensure there are files in the `evals` directory.");
         process.exit(1);
       }
-      const outputFileText = result.outputFiles[0].text;
-      const evaluation = loadModule({
-        filename: args2.file,
-        moduleText: outputFileText
-      });
-      await evaluation.run();
+      if (!args2.file) {
+        console.log(`Located ${files.length} evaluation files in evals/`);
+      }
+      for (const file of files) {
+        console.log(`Loading ${file}...`);
+        const buildOptions = {
+          entryPoints: [file],
+          outfile: `tmp_out_${file}.js`,
+          write: false,
+          // will be loaded in memory as a temp file
+          platform: "node",
+          bundle: true,
+          external: ["node_modules/*"]
+        };
+        const result = await esbuild.build(buildOptions);
+        if (!result.outputFiles) {
+          console.error("Error when building: No output files found");
+          if (args2.fail_on_error) {
+            process.exit(1);
+          }
+          continue;
+        }
+        const outputFileText = result.outputFiles[0].text;
+        const evaluation = loadModule({
+          filename: args2.file,
+          moduleText: outputFileText
+        });
+        if (!evaluation?.run) {
+          console.error(`Evaluation ${file} does not properly call evaluate()`);
+          if (args2.fail_on_error) {
+            process.exit(1);
+          }
+          continue;
+        }
+        await evaluation.run();
+      }
+    }
+  });
+  parser.set_defaults({
+    func: async (args2) => {
+      parser.print_help();
+      process.exit(0);
     }
   });
   const parsed = parser.parse_args(args);

package/dist/cli.mjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n  \"name\": \"@lmnr-ai/lmnr\",\n  \"version\": \"0.4.25\",\n  \"description\": \"TypeScript SDK for Laminar AI\",\n  \"main\": \"dist/index.js\",\n  \"types\": \"dist/index.d.ts\",\n  \"scripts\": {\n    \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n    \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n  },\n  \"bin\": {\n    \"lmnr\": \"./dist/cli.js\"\n  },\n  \"repository\": {\n    \"type\": \"git\",\n    \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n  },\n  \"keywords\": [\n    \"laminar\",\n    \"lmnr\",\n    \"sdk\",\n    \"lmnr.ai\"\n  ],\n  \"author\": \"founders@lmnr.ai\",\n  \"license\": \"Apache-2.0\",\n  \"bugs\": {\n    \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n  },\n  \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n  \"devDependencies\": {\n    \"@anthropic-ai/sdk\": \"^0.27.3\",\n    \"@aws-sdk/client-bedrock-runtime\": \"^3.705.0\",\n    \"@azure/openai\": \"1.0.0-beta.13\",\n    \"@google-cloud/aiplatform\": \"^3.34.0\",\n    \"@google-cloud/vertexai\": \"^1.9.0\",\n    \"@langchain/core\": \"^0.3.20\",\n    \"@pinecone-database/pinecone\": \"^3.0.3\",\n    \"@qdrant/js-client-rest\": \"^1.12.0\",\n    \"@types/argparse\": \"^2.0.17\",\n    \"@types/cli-progress\": \"^3.11.6\",\n    \"@types/node\": \"^22.10.1\",\n    \"@types/semver\": \"^7.5.8\",\n    \"@types/uuid\": \"^10.0.0\",\n    \"bufferutil\": \"^4.0.8\",\n    \"chromadb\": \"^1.9.4\",\n    \"cohere-ai\": \"^7.15.0\",\n    \"langchain\": \"^0.3.6\",\n    \"llamaindex\": \"^0.7.10\",\n    \"openai\": \"^4.75.0\",\n    \"tsup\": \"^8.3.5\",\n    \"typescript\": \"^5.7.2\"\n  },\n  \"dependencies\": {\n    \"@grpc/grpc-js\": \"^1.12.3\",\n    \"@opentelemetry/api\": \"^1.9.0\",\n    \"@opentelemetry/core\": \"^1.28.0\",\n    \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.53.0\",\n    \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.53.0\",\n    \"@opentelemetry/instrumentation\": \"^0.53.0\",\n    \"@opentelemetry/sdk-node\": \"^0.53.0\",\n    \"@opentelemetry/sdk-trace-base\": \"^1.28.0\",\n    \"@opentelemetry/sdk-trace-node\": \"^1.28.0\",\n    \"@opentelemetry/semantic-conventions\": \"^1.28.0\",\n    \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n    \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n    \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n    \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n    \"@traceloop/instrumentation-chromadb\": \"^0.11.3\",\n    \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n    \"@traceloop/instrumentation-langchain\": \"^0.11.4\",\n    \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n    \"@traceloop/instrumentation-openai\": \"^0.11.3\",\n    \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n    \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n    \"@traceloop/instrumentation-vertexai\": \"^0.11.4\",\n    \"argparse\": \"^2.0.1\",\n    \"cli-progress\": \"^3.12.0\",\n    \"esbuild\": \"^0.24.0\",\n    \"uuid\": \"^11.0.3\"\n  }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n  filename,\n  moduleText,\n}: {\n  filename: string;\n  moduleText: string;\n}) {\n  // TODO: Figure out how to remove all ts-ignores\n  // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n  // @ts-ignore\n  globalThis._evaluation = undefined;  // @ts-ignore\n  globalThis._set_global_evaluation = true;  // @ts-ignore\n\n  // it needs \"require\" to be passed in\n  new Function(\"require\", moduleText)(require);\n\n  // Return the modified _evals global variable\n  // @ts-ignore\n  return globalThis._evaluation;\n}\n\nasync function cli() {\n  const [, , ...args] = process.argv;\n\n  // Use argparse, which is the port of the python library\n  const parser = new ArgumentParser({\n    prog: \"lmnr\",\n    description: \"CLI for Laminar\",\n  });\n\n  parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n  const subparsers = parser.add_subparsers({\n    title: \"subcommands\",\n    dest: \"subcommand\",\n  });\n\n  const parser_eval = subparsers.add_parser(\"eval\", {\n    description: \"Run an evaluation\",\n  });\n\n  parser_eval.add_argument(\"file\", {\n    help: \"A file containing the evaluation to run\",\n  });\n\n  parser_eval.set_defaults({\n    func: async (args: any) => {\n      // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n      const buildOptions = {\n        entryPoints: [args.file],\n        outfile: \"tmp_out.js\",\n        write: false,  // will be loaded in memory as a temp file\n        platform: \"node\" as esbuild.Platform,\n        bundle: true,\n        external: [\"node_modules/*\"],\n      };\n\n      const result = await esbuild.build(buildOptions);\n\n      if (!result.outputFiles) {\n        console.error(\"Error when building: No output files found\");\n        process.exit(1);\n      }\n\n      const outputFileText = result.outputFiles[0].text;\n\n      const evaluation = loadModule({\n        filename: args.file,\n        moduleText: outputFileText,\n      });\n\n      // @ts-ignore\n      await evaluation.run();\n    }\n  });\n\n  const parsed = parser.parse_args(args);\n  await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,MAAQ;AAAA,QACR,YAAc;AAAA,MAChB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;AC9EA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AAEzB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,SAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,cAAc,WAAW,WAAW,QAAQ;AAAA,IAChD,aAAa;AAAA,EACf,CAAC;AAED,cAAY,aAAa,QAAQ;AAAA,IAC/B,MAAM;AAAA,EACR,CAAC;AAED,cAAY,aAAa;AAAA,IACvB,MAAM,OAAOA,UAAc;AAEzB,YAAM,eAAe;AAAA,QACnB,aAAa,CAACA,MAAK,IAAI;AAAA,QACvB,SAAS;AAAA,QACT,OAAO;AAAA;AAAA,QACP,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU,CAAC,gBAAgB;AAAA,MAC7B;AAEA,YAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,UAAI,CAAC,OAAO,aAAa;AACvB,gBAAQ,MAAM,4CAA4C;AAC1D,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,YAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,YAAM,aAAa,WAAW;AAAA,QAC5B,UAAUA,MAAK;AAAA,QACf,YAAY;AAAA,MACd,CAAC;AAGD,YAAM,WAAW,IAAI;AAAA,IACvB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI;","names":["args"]}
+{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n  \"name\": \"@lmnr-ai/lmnr\",\n  \"version\": \"0.4.27\",\n  \"description\": \"TypeScript SDK for Laminar AI\",\n  \"main\": \"dist/index.js\",\n  \"types\": \"dist/index.d.ts\",\n  \"scripts\": {\n    \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n    \"test\": \"tsx --test test/*.test.ts\"\n  },\n  \"bin\": {\n    \"lmnr\": \"./dist/cli.js\"\n  },\n  \"repository\": {\n    \"type\": \"git\",\n    \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n  },\n  \"keywords\": [\n    \"laminar\",\n    \"lmnr\",\n    \"sdk\",\n    \"lmnr.ai\"\n  ],\n  \"author\": \"founders@lmnr.ai\",\n  \"license\": \"Apache-2.0\",\n  \"bugs\": {\n    \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n  },\n  \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n  \"devDependencies\": {\n    \"@anthropic-ai/sdk\": \"^0.27.3\",\n    \"@aws-sdk/client-bedrock-runtime\": \"^3.705.0\",\n    \"@azure/openai\": \"1.0.0-beta.13\",\n    \"@google-cloud/aiplatform\": \"^3.34.0\",\n    \"@google-cloud/vertexai\": \"^1.9.0\",\n    \"@langchain/core\": \"^0.3.20\",\n    \"@pinecone-database/pinecone\": \"^3.0.3\",\n    \"@qdrant/js-client-rest\": \"^1.12.0\",\n    \"@types/argparse\": \"^2.0.17\",\n    \"@types/cli-progress\": \"^3.11.6\",\n    \"@types/node\": \"^22.10.1\",\n    \"@types/semver\": \"^7.5.8\",\n    \"@types/uuid\": \"^10.0.0\",\n    \"bufferutil\": \"^4.0.8\",\n    \"chromadb\": \"^1.9.4\",\n    \"cohere-ai\": \"^7.15.0\",\n    \"langchain\": \"^0.3.6\",\n    \"llamaindex\": \"^0.7.10\",\n    \"openai\": \"^4.75.0\",\n    \"tsup\": \"^8.3.5\",\n    \"tsx\": \"^4.19.2\",\n    \"typescript\": \"^5.7.2\"\n  },\n  \"dependencies\": {\n    \"@grpc/grpc-js\": \"^1.12.3\",\n    \"@opentelemetry/api\": \"^1.9.0\",\n    \"@opentelemetry/core\": \"^1.28.0\",\n    \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.53.0\",\n    \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.53.0\",\n    \"@opentelemetry/instrumentation\": \"^0.53.0\",\n    \"@opentelemetry/sdk-node\": \"^0.53.0\",\n    \"@opentelemetry/sdk-trace-base\": \"^1.28.0\",\n    \"@opentelemetry/sdk-trace-node\": \"^1.28.0\",\n    \"@opentelemetry/semantic-conventions\": \"^1.28.0\",\n    \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n    \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n    \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n    \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n    \"@traceloop/instrumentation-chromadb\": \"^0.11.3\",\n    \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n    \"@traceloop/instrumentation-langchain\": \"^0.11.4\",\n    \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n    \"@traceloop/instrumentation-openai\": \"^0.11.3\",\n    \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n    \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n    \"@traceloop/instrumentation-vertexai\": \"^0.11.4\",\n    \"argparse\": \"^2.0.1\",\n    \"cli-progress\": \"^3.12.0\",\n    \"esbuild\": \"^0.24.0\",\n    \"glob\": \"^11.0.0\",\n    \"uuid\": \"^11.0.3\"\n  }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n  filename,\n  moduleText,\n}: {\n  filename: string;\n  moduleText: string;\n}) {\n  // TODO: Figure out how to remove all ts-ignores\n  // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n  // @ts-ignore\n  globalThis._evaluation = undefined;  // @ts-ignore\n  globalThis._set_global_evaluation = true;  // @ts-ignore\n\n  // it needs \"require\" to be passed in\n  new Function(\"require\", moduleText)(require);\n\n  // Return the modified _evals global variable\n  // @ts-ignore\n  return globalThis._evaluation;\n}\n\nasync function cli() {\n  const [, , ...args] = process.argv;\n\n  // Use argparse, which is the port of the python library\n  const parser = new ArgumentParser({\n    prog: \"lmnr\",\n    description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n  });\n\n  parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n  const subparsers = parser.add_subparsers({\n    title: \"subcommands\",\n    dest: \"subcommand\",\n  });\n\n  const parserEval = subparsers.add_parser(\"eval\", {\n    description: \"Run an evaluation\",\n    help: \"Run an evaluation\",\n  });\n\n  parserEval.add_argument(\"file\", {\n    help: \"A file containing the evaluation to run. If no file is provided, \" +\n      \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n    nargs: \"?\",\n  });\n\n  parserEval.add_argument(\"--fail-on-error\", {\n    help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n    action: \"store_true\",\n  });\n\n  parserEval.set_defaults({\n    func: async (args: any) => {\n      const files = args.file\n        ? [args.file]\n        : glob.sync('evals/**/*.eval.{ts,js}')\n\n      files.sort();\n\n      if (files.length === 0) {\n        console.error(\"No evaluation files found. Please provide a file or \" +\n          \"ensure there are files in the `evals` directory.\");\n        process.exit(1);\n      }\n\n      if (!args.file) {\n        console.log(`Located ${files.length} evaluation files in evals/`);\n      }\n\n      for (const file of files) {\n        console.log(`Loading ${file}...`);\n        // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n        const buildOptions = {\n          entryPoints: [file],\n          outfile: `tmp_out_${file}.js`,\n          write: false,  // will be loaded in memory as a temp file\n          platform: \"node\" as esbuild.Platform,\n          bundle: true,\n          external: [\"node_modules/*\"],\n        };\n\n        const result = await esbuild.build(buildOptions);\n\n        if (!result.outputFiles) {\n          console.error(\"Error when building: No output files found\");\n          if (args.fail_on_error) {\n            process.exit(1);\n          }\n          continue;\n        }\n\n        const outputFileText = result.outputFiles[0].text;\n\n        const evaluation = loadModule({\n          filename: args.file,\n          moduleText: outputFileText,\n        });\n\n        // @ts-ignore\n        if (!evaluation?.run) {\n          console.error(`Evaluation ${file} does not properly call evaluate()`);\n          if (args.fail_on_error) {\n            process.exit(1);\n          }\n          continue;\n        }\n\n        // @ts-ignore\n        await evaluation.run();\n      }\n    }\n  });\n\n  parser.set_defaults({\n    func: async (args: any) => {\n      parser.print_help();\n      process.exit(0);\n    }\n  });\n\n  const parsed = parser.parse_args(args);\n  await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,MAAQ;AAAA,QACR,KAAO;AAAA,QACP,YAAc;AAAA,MAChB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,QACR,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;AChFA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AACzB,YAAY,UAAU;AAEtB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,SAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOA,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,gBAAQ,MAAM,sGACsC;AACpD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACA,MAAK,MAAM;AACd,gBAAQ,IAAI,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAEA,iBAAW,QAAQ,OAAO;AACxB,gBAAQ,IAAI,WAAW,IAAI,KAAK;AAEhC,cAAM,eAAe;AAAA,UACnB,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,UACV,QAAQ;AAAA,UACR,UAAU,CAAC,gBAAgB;AAAA,QAC7B;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,kBAAQ,MAAM,4CAA4C;AAC1D,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,aAAa,WAAW;AAAA,UAC5B,UAAUA,MAAK;AAAA,UACf,YAAY;AAAA,QACd,CAAC;AAGD,YAAI,CAAC,YAAY,KAAK;AACpB,kBAAQ,MAAM,cAAc,IAAI,oCAAoC;AACpE,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAGA,cAAM,WAAW,IAAI;AAAA,MACvB;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,OAAOA,UAAc;AACzB,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI;","names":["args"]}

package/dist/index.d.mts

@@ -330,10 +330,8 @@ type SemanticSearchResponse = {
  * Trace types to categorize traces.
  * They are used as association properties passed to all spans in a trace.
  *
- * EVENT traces are traces created by the event runner.
- * They are used to mark traces created by the "online" evaluations for semantic events.
  */
-type TraceType = 'DEFAULT' | 'EVENT' | 'EVALUATION';
+type TraceType = 'DEFAULT' | 'EVALUATION';
 /**
  * Tracing levels to conditionally disable tracing.
  *
@@ -367,6 +365,7 @@ interface LaminarInitializeProps {
     grpcPort?: number;
     instrumentModules?: InitializeOptions["instrumentModules"];
     useExternalTracerProvider?: boolean;
+    _spanProcessor?: SpanProcessor;
 }
 declare class Laminar {
     private static baseHttpUrl;
@@ -612,7 +611,7 @@ declare class Laminar {
      *
      * See {@link startSpan} docs for a usage example
      */
-    static withSpan<T>(span: Span, fn: () => T, endOnExit?: boolean): T;
+    static withSpan<T>(span: Span, fn: () => T, endOnExit?: boolean): T | Promise<T>;
     static shutdown(): Promise<void>;
     static createEvaluation<D, T, O>({ groupId, name, data }: {
         groupId?: string;
@@ -630,7 +629,6 @@ declare class Laminar {
 interface ObserveOptions {
     name?: string;
     sessionId?: string;
-    userId?: string;
     traceType?: TraceType;
     spanType?: 'DEFAULT' | 'LLM';
     traceId?: string;
@@ -658,7 +656,7 @@ interface ObserveOptions {
  *    // Your code here
  * });
  */
-declare function observe<A extends unknown[], F extends (...args: A) => ReturnType<F>>({ name, sessionId, userId, traceType, spanType, traceId, }: ObserveOptions, fn: F, ...args: A): Promise<ReturnType<F>>;
+declare function observe<A extends unknown[], F extends (...args: A) => ReturnType<F>>({ name, sessionId, traceType, spanType, traceId, }: ObserveOptions, fn: F, ...args: A): Promise<ReturnType<F>>;
 /**
  * Sets the labels for any spans inside the function. This is useful for adding
  * labels to the spans created in the auto-instrumentations. Returns the result

package/dist/index.d.ts

@@ -330,10 +330,8 @@ type SemanticSearchResponse = {
  * Trace types to categorize traces.
  * They are used as association properties passed to all spans in a trace.
  *
- * EVENT traces are traces created by the event runner.
- * They are used to mark traces created by the "online" evaluations for semantic events.
  */
-type TraceType = 'DEFAULT' | 'EVENT' | 'EVALUATION';
+type TraceType = 'DEFAULT' | 'EVALUATION';
 /**
  * Tracing levels to conditionally disable tracing.
  *
@@ -367,6 +365,7 @@ interface LaminarInitializeProps {
     grpcPort?: number;
     instrumentModules?: InitializeOptions["instrumentModules"];
     useExternalTracerProvider?: boolean;
+    _spanProcessor?: SpanProcessor;
 }
 declare class Laminar {
     private static baseHttpUrl;
@@ -612,7 +611,7 @@ declare class Laminar {
      *
      * See {@link startSpan} docs for a usage example
      */
-    static withSpan<T>(span: Span, fn: () => T, endOnExit?: boolean): T;
+    static withSpan<T>(span: Span, fn: () => T, endOnExit?: boolean): T | Promise<T>;
     static shutdown(): Promise<void>;
     static createEvaluation<D, T, O>({ groupId, name, data }: {
         groupId?: string;
@@ -630,7 +629,6 @@ declare class Laminar {
 interface ObserveOptions {
     name?: string;
     sessionId?: string;
-    userId?: string;
     traceType?: TraceType;
     spanType?: 'DEFAULT' | 'LLM';
     traceId?: string;
@@ -658,7 +656,7 @@ interface ObserveOptions {
  *    // Your code here
  * });
  */
-declare function observe<A extends unknown[], F extends (...args: A) => ReturnType<F>>({ name, sessionId, userId, traceType, spanType, traceId, }: ObserveOptions, fn: F, ...args: A): Promise<ReturnType<F>>;
+declare function observe<A extends unknown[], F extends (...args: A) => ReturnType<F>>({ name, sessionId, traceType, spanType, traceId, }: ObserveOptions, fn: F, ...args: A): Promise<ReturnType<F>>;
 /**
  * Sets the labels for any spans inside the function. This is useful for adding
  * labels to the spans created in the auto-instrumentations. Returns the result