@lmnr-ai/lmnr 0.4.37 → 0.4.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. package/dist/cli.js +1 -1
  2. package/dist/cli.js.map +1 -1
  3. package/dist/cli.mjs +1 -1
  4. package/dist/cli.mjs.map +1 -1
  5. package/dist/index.d.mts +38 -19
  6. package/dist/index.d.ts +38 -19
  7. package/dist/index.js +198 -100
  8. package/dist/index.js.map +1 -1
  9. package/dist/index.mjs +198 -100
  10. package/dist/index.mjs.map +1 -1
  11. package/package.json +1 -1
  12. package/src/browser/playwright.ts +38 -20
  13. package/src/evaluations.ts +136 -99
  14. package/src/laminar.ts +49 -18
  15. package/src/sdk/interfaces/initialize-options.interface.ts +7 -0
  16. package/src/sdk/tracing/index.ts +3 -1
  17. package/src/types.ts +2 -1
  18. package/src/utils.ts +55 -0
package/dist/cli.js CHANGED
@@ -36,7 +36,7 @@ var require_package = __commonJS({
36
36
  "package.json"(exports2, module2) {
37
37
  module2.exports = {
38
38
  name: "@lmnr-ai/lmnr",
39
- version: "0.4.37",
39
+ version: "0.4.38",
40
40
  description: "TypeScript SDK for Laminar AI",
41
41
  main: "dist/index.js",
42
42
  types: "dist/index.d.ts",
package/dist/cli.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.37\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts --external puppeteer --external puppeteer-core --external playwright\",\n \"test\": \"tsx --test test/*.test.ts\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.33.1\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.716.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^1.10.1\",\n \"@google-cloud/aiplatform\": \"^3.34.0\",\n \"@google-cloud/vertexai\": \"^1.9.2\",\n \"@langchain/core\": \"^0.3.26\",\n \"@pinecone-database/pinecone\": \"^4.0.0\",\n \"@qdrant/js-client-rest\": \"^1.12.0\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.10.2\",\n \"@types/semver\": \"^7.5.8\",\n \"@types/uuid\": \"^10.0.0\",\n \"bufferutil\": \"^4.0.8\",\n \"chromadb\": \"^1.9.4\",\n \"cohere-ai\": \"^7.15.0\",\n \"langchain\": \"^0.3.8\",\n \"llamaindex\": \"^0.8.30\",\n \"openai\": \"^4.80.1\",\n \"playwright\": \"^1.50.0\",\n \"tsup\": \"^8.3.5\",\n \"tsx\": \"^4.19.2\",\n \"typescript\": \"^5.7.2\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.12.5\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^1.30.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.0\",\n \"@opentelemetry/instrumentation\": \"^0.57.0\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.0\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.0\",\n \"@opentelemetry/sdk-node\": \"^0.57.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.0\",\n \"@opentelemetry/semantic-conventions\": \"^1.28.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.6\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.6\",\n \"@traceloop/instrumentation-azure\": \"^0.11.6\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.6\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.6\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.6\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.6\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.6\",\n \"@traceloop/instrumentation-openai\": \"^0.11.6\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.6\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.6\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.6\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.24.2\",\n \"glob\": \"^11.0.0\",\n \"uuid\": \"^11.0.3\"\n }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\nimport { context, trace, propagation } from \"@opentelemetry/api\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // TODO: Figure out how to remove all ts-ignores\n // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n const files = args.file\n ? [args.file]\n : glob.sync('evals/**/*.eval.{ts,js}')\n\n files.sort();\n\n if (files.length === 0) {\n console.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are files in the `evals` directory.\");\n process.exit(1);\n }\n\n if (!args.file) {\n console.log(`Located ${files.length} evaluation files in evals/`);\n }\n\n for (const file of files) {\n console.log(`Loading ${file}...`);\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"node_modules/*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n if (!evaluation?.run) {\n console.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n // @ts-ignore\n await evaluation.run();\n\n // FIXME: Now every evaluation file creates a new tracer provider.\n // Attempt to re-initialize it in the same process breaks it.\n // For now, we disable all APIs after running each file, but ideally\n // we should keep a global tracer provider that is initialized once\n // here in the CLI.\n context.disable();\n trace.disable();\n propagation.disable();\n }\n }\n });\n\n parser.set_defaults({\n func: async (args: any) => {\n parser.print_help();\n process.exit(0);\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,iBAAAA,UAAAC,SAAA;AAAA,IAAAA,QAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,YAAc;AAAA,QACd,MAAQ;AAAA,QACR,KAAO;AAAA,QACP,YAAc;AAAA,MAChB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,qCAAqC;AAAA,QACrC,0CAA0C;AAAA,QAC1C,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,QACR,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;ACtFA;AAAA;AAAA;AAAA;AAAA;AAEA,sBAA+B;AAC/B,cAAyB;AACzB,WAAsB;AACtB,iBAA4C;AAE5C,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,OAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,gBAAQ,MAAM,sGACsC;AACpD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACA,MAAK,MAAM;AACd,gBAAQ,IAAI,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAEA,iBAAW,QAAQ,OAAO;AACxB,gBAAQ,IAAI,WAAW,IAAI,KAAK;AAEhC,cAAM,eAAe;AAAA,UACnB,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,UACV,QAAQ;AAAA,UACR,UAAU,CAAC,gBAAgB;AAAA,QAC7B;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,kBAAQ,MAAM,4CAA4C;AAC1D,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,aAAa,WAAW;AAAA,UAC5B,UAAUA,MAAK;AAAA,UACf,YAAY;AAAA,QACd,CAAC;AAGD,YAAI,CAAC,YAAY,KAAK;AACpB,kBAAQ,MAAM,cAAc,IAAI,oCAAoC;AACpE,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAGA,cAAM,WAAW,IAAI;AAOrB,2BAAQ,QAAQ;AAChB,yBAAM,QAAQ;AACd,+BAAY,QAAQ;AAAA,MACtB;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,OAAOA,UAAc;AACzB,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI;","names":["exports","module","args"]}
1
+ {"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.38\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts --external puppeteer --external puppeteer-core --external playwright\",\n \"test\": \"tsx --test test/*.test.ts\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.33.1\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.716.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^1.10.1\",\n \"@google-cloud/aiplatform\": \"^3.34.0\",\n \"@google-cloud/vertexai\": \"^1.9.2\",\n \"@langchain/core\": \"^0.3.26\",\n \"@pinecone-database/pinecone\": \"^4.0.0\",\n \"@qdrant/js-client-rest\": \"^1.12.0\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.10.2\",\n \"@types/semver\": \"^7.5.8\",\n \"@types/uuid\": \"^10.0.0\",\n \"bufferutil\": \"^4.0.8\",\n \"chromadb\": \"^1.9.4\",\n \"cohere-ai\": \"^7.15.0\",\n \"langchain\": \"^0.3.8\",\n \"llamaindex\": \"^0.8.30\",\n \"openai\": \"^4.80.1\",\n \"playwright\": \"^1.50.0\",\n \"tsup\": \"^8.3.5\",\n \"tsx\": \"^4.19.2\",\n \"typescript\": \"^5.7.2\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.12.5\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^1.30.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.0\",\n \"@opentelemetry/instrumentation\": \"^0.57.0\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.0\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.0\",\n \"@opentelemetry/sdk-node\": \"^0.57.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.0\",\n \"@opentelemetry/semantic-conventions\": \"^1.28.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.6\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.6\",\n \"@traceloop/instrumentation-azure\": \"^0.11.6\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.6\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.6\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.6\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.6\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.6\",\n \"@traceloop/instrumentation-openai\": \"^0.11.6\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.6\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.6\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.6\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.24.2\",\n \"glob\": \"^11.0.0\",\n \"uuid\": \"^11.0.3\"\n }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\nimport { context, trace, propagation } from \"@opentelemetry/api\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // TODO: Figure out how to remove all ts-ignores\n // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n const files = args.file\n ? [args.file]\n : glob.sync('evals/**/*.eval.{ts,js}')\n\n files.sort();\n\n if (files.length === 0) {\n console.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are files in the `evals` directory.\");\n process.exit(1);\n }\n\n if (!args.file) {\n console.log(`Located ${files.length} evaluation files in evals/`);\n }\n\n for (const file of files) {\n console.log(`Loading ${file}...`);\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"node_modules/*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n if (!evaluation?.run) {\n console.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n // @ts-ignore\n await evaluation.run();\n\n // FIXME: Now every evaluation file creates a new tracer provider.\n // Attempt to re-initialize it in the same process breaks it.\n // For now, we disable all APIs after running each file, but ideally\n // we should keep a global tracer provider that is initialized once\n // here in the CLI.\n context.disable();\n trace.disable();\n propagation.disable();\n }\n }\n });\n\n parser.set_defaults({\n func: async (args: any) => {\n parser.print_help();\n process.exit(0);\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,iBAAAA,UAAAC,SAAA;AAAA,IAAAA,QAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,YAAc;AAAA,QACd,MAAQ;AAAA,QACR,KAAO;AAAA,QACP,YAAc;AAAA,MAChB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,qCAAqC;AAAA,QACrC,0CAA0C;AAAA,QAC1C,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,QACR,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;ACtFA;AAAA;AAAA;AAAA;AAAA;AAEA,sBAA+B;AAC/B,cAAyB;AACzB,WAAsB;AACtB,iBAA4C;AAE5C,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,OAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,gBAAQ,MAAM,sGACsC;AACpD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACA,MAAK,MAAM;AACd,gBAAQ,IAAI,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAEA,iBAAW,QAAQ,OAAO;AACxB,gBAAQ,IAAI,WAAW,IAAI,KAAK;AAEhC,cAAM,eAAe;AAAA,UACnB,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,UACV,QAAQ;AAAA,UACR,UAAU,CAAC,gBAAgB;AAAA,QAC7B;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,kBAAQ,MAAM,4CAA4C;AAC1D,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,aAAa,WAAW;AAAA,UAC5B,UAAUA,MAAK;AAAA,UACf,YAAY;AAAA,QACd,CAAC;AAGD,YAAI,CAAC,YAAY,KAAK;AACpB,kBAAQ,MAAM,cAAc,IAAI,oCAAoC;AACpE,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAGA,cAAM,WAAW,IAAI;AAOrB,2BAAQ,QAAQ;AAChB,yBAAM,QAAQ;AACd,+BAAY,QAAQ;AAAA,MACtB;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,OAAOA,UAAc;AACzB,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI;","names":["exports","module","args"]}
package/dist/cli.mjs CHANGED
@@ -15,7 +15,7 @@ var require_package = __commonJS({
15
15
  "package.json"(exports, module) {
16
16
  module.exports = {
17
17
  name: "@lmnr-ai/lmnr",
18
- version: "0.4.37",
18
+ version: "0.4.38",
19
19
  description: "TypeScript SDK for Laminar AI",
20
20
  main: "dist/index.js",
21
21
  types: "dist/index.d.ts",
package/dist/cli.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.37\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts --external puppeteer --external puppeteer-core --external playwright\",\n \"test\": \"tsx --test test/*.test.ts\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.33.1\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.716.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^1.10.1\",\n \"@google-cloud/aiplatform\": \"^3.34.0\",\n \"@google-cloud/vertexai\": \"^1.9.2\",\n \"@langchain/core\": \"^0.3.26\",\n \"@pinecone-database/pinecone\": \"^4.0.0\",\n \"@qdrant/js-client-rest\": \"^1.12.0\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.10.2\",\n \"@types/semver\": \"^7.5.8\",\n \"@types/uuid\": \"^10.0.0\",\n \"bufferutil\": \"^4.0.8\",\n \"chromadb\": \"^1.9.4\",\n \"cohere-ai\": \"^7.15.0\",\n \"langchain\": \"^0.3.8\",\n \"llamaindex\": \"^0.8.30\",\n \"openai\": \"^4.80.1\",\n \"playwright\": \"^1.50.0\",\n \"tsup\": \"^8.3.5\",\n \"tsx\": \"^4.19.2\",\n \"typescript\": \"^5.7.2\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.12.5\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^1.30.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.0\",\n \"@opentelemetry/instrumentation\": \"^0.57.0\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.0\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.0\",\n \"@opentelemetry/sdk-node\": \"^0.57.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.0\",\n \"@opentelemetry/semantic-conventions\": \"^1.28.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.6\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.6\",\n \"@traceloop/instrumentation-azure\": \"^0.11.6\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.6\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.6\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.6\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.6\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.6\",\n \"@traceloop/instrumentation-openai\": \"^0.11.6\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.6\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.6\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.6\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.24.2\",\n \"glob\": \"^11.0.0\",\n \"uuid\": \"^11.0.3\"\n }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\nimport { context, trace, propagation } from \"@opentelemetry/api\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // TODO: Figure out how to remove all ts-ignores\n // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n const files = args.file\n ? [args.file]\n : glob.sync('evals/**/*.eval.{ts,js}')\n\n files.sort();\n\n if (files.length === 0) {\n console.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are files in the `evals` directory.\");\n process.exit(1);\n }\n\n if (!args.file) {\n console.log(`Located ${files.length} evaluation files in evals/`);\n }\n\n for (const file of files) {\n console.log(`Loading ${file}...`);\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"node_modules/*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n if (!evaluation?.run) {\n console.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n // @ts-ignore\n await evaluation.run();\n\n // FIXME: Now every evaluation file creates a new tracer provider.\n // Attempt to re-initialize it in the same process breaks it.\n // For now, we disable all APIs after running each file, but ideally\n // we should keep a global tracer provider that is initialized once\n // here in the CLI.\n context.disable();\n trace.disable();\n propagation.disable();\n }\n }\n });\n\n parser.set_defaults({\n func: async (args: any) => {\n parser.print_help();\n process.exit(0);\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,YAAc;AAAA,QACd,MAAQ;AAAA,QACR,KAAO;AAAA,QACP,YAAc;AAAA,MAChB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,qCAAqC;AAAA,QACrC,0CAA0C;AAAA,QAC1C,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,QACR,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;ACpFA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AACzB,YAAY,UAAU;AACtB,SAAS,SAAS,OAAO,mBAAmB;AAE5C,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,SAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOA,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,gBAAQ,MAAM,sGACsC;AACpD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACA,MAAK,MAAM;AACd,gBAAQ,IAAI,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAEA,iBAAW,QAAQ,OAAO;AACxB,gBAAQ,IAAI,WAAW,IAAI,KAAK;AAEhC,cAAM,eAAe;AAAA,UACnB,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,UACV,QAAQ;AAAA,UACR,UAAU,CAAC,gBAAgB;AAAA,QAC7B;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,kBAAQ,MAAM,4CAA4C;AAC1D,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,aAAa,WAAW;AAAA,UAC5B,UAAUA,MAAK;AAAA,UACf,YAAY;AAAA,QACd,CAAC;AAGD,YAAI,CAAC,YAAY,KAAK;AACpB,kBAAQ,MAAM,cAAc,IAAI,oCAAoC;AACpE,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAGA,cAAM,WAAW,IAAI;AAOrB,gBAAQ,QAAQ;AAChB,cAAM,QAAQ;AACd,oBAAY,QAAQ;AAAA,MACtB;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,OAAOA,UAAc;AACzB,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI;","names":["args"]}
1
+ {"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.38\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts --external puppeteer --external puppeteer-core --external playwright\",\n \"test\": \"tsx --test test/*.test.ts\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.33.1\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.716.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^1.10.1\",\n \"@google-cloud/aiplatform\": \"^3.34.0\",\n \"@google-cloud/vertexai\": \"^1.9.2\",\n \"@langchain/core\": \"^0.3.26\",\n \"@pinecone-database/pinecone\": \"^4.0.0\",\n \"@qdrant/js-client-rest\": \"^1.12.0\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.10.2\",\n \"@types/semver\": \"^7.5.8\",\n \"@types/uuid\": \"^10.0.0\",\n \"bufferutil\": \"^4.0.8\",\n \"chromadb\": \"^1.9.4\",\n \"cohere-ai\": \"^7.15.0\",\n \"langchain\": \"^0.3.8\",\n \"llamaindex\": \"^0.8.30\",\n \"openai\": \"^4.80.1\",\n \"playwright\": \"^1.50.0\",\n \"tsup\": \"^8.3.5\",\n \"tsx\": \"^4.19.2\",\n \"typescript\": \"^5.7.2\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.12.5\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^1.30.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.0\",\n \"@opentelemetry/instrumentation\": \"^0.57.0\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.0\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.0\",\n \"@opentelemetry/sdk-node\": \"^0.57.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.0\",\n \"@opentelemetry/semantic-conventions\": \"^1.28.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.6\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.6\",\n \"@traceloop/instrumentation-azure\": \"^0.11.6\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.6\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.6\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.6\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.6\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.6\",\n \"@traceloop/instrumentation-openai\": \"^0.11.6\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.6\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.6\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.6\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.24.2\",\n \"glob\": \"^11.0.0\",\n \"uuid\": \"^11.0.3\"\n }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\nimport { context, trace, propagation } from \"@opentelemetry/api\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // TODO: Figure out how to remove all ts-ignores\n // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n const files = args.file\n ? [args.file]\n : glob.sync('evals/**/*.eval.{ts,js}')\n\n files.sort();\n\n if (files.length === 0) {\n console.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are files in the `evals` directory.\");\n process.exit(1);\n }\n\n if (!args.file) {\n console.log(`Located ${files.length} evaluation files in evals/`);\n }\n\n for (const file of files) {\n console.log(`Loading ${file}...`);\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"node_modules/*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n if (!evaluation?.run) {\n console.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n // @ts-ignore\n await evaluation.run();\n\n // FIXME: Now every evaluation file creates a new tracer provider.\n // Attempt to re-initialize it in the same process breaks it.\n // For now, we disable all APIs after running each file, but ideally\n // we should keep a global tracer provider that is initialized once\n // here in the CLI.\n context.disable();\n trace.disable();\n propagation.disable();\n }\n }\n });\n\n parser.set_defaults({\n func: async (args: any) => {\n parser.print_help();\n process.exit(0);\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,YAAc;AAAA,QACd,MAAQ;AAAA,QACR,KAAO;AAAA,QACP,YAAc;AAAA,MAChB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,qCAAqC;AAAA,QACrC,0CAA0C;AAAA,QAC1C,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,QACR,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;ACpFA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AACzB,YAAY,UAAU;AACtB,SAAS,SAAS,OAAO,mBAAmB;AAE5C,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,SAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOA,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,gBAAQ,MAAM,sGACsC;AACpD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACA,MAAK,MAAM;AACd,gBAAQ,IAAI,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAEA,iBAAW,QAAQ,OAAO;AACxB,gBAAQ,IAAI,WAAW,IAAI,KAAK;AAEhC,cAAM,eAAe;AAAA,UACnB,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,UACV,QAAQ;AAAA,UACR,UAAU,CAAC,gBAAgB;AAAA,QAC7B;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,kBAAQ,MAAM,4CAA4C;AAC1D,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,aAAa,WAAW;AAAA,UAC5B,UAAUA,MAAK;AAAA,UACf,YAAY;AAAA,QACd,CAAC;AAGD,YAAI,CAAC,YAAY,KAAK;AACpB,kBAAQ,MAAM,cAAc,IAAI,oCAAoC;AACpE,cAAIA,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAGA,cAAM,WAAW,IAAI;AAOrB,gBAAQ,QAAQ;AAChB,cAAM,QAAQ;AACd,oBAAY,QAAQ;AAAA,MACtB;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,OAAOA,UAAc;AACzB,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI;","names":["args"]}
package/dist/index.d.mts CHANGED
@@ -35,6 +35,8 @@ declare class LaminarDataset<D, T> extends EvaluationDataset<D, T> {
35
35
  get(index: number): Promise<Datapoint<D, T>>;
36
36
  }
37
37
 
38
+ type StringUUID = `${string}-${string}-${string}-${string}-${string}`;
39
+
38
40
  /**
39
41
  * Options for initializing the Traceloop SDK.
40
42
  */
@@ -135,6 +137,12 @@ interface InitializeOptions {
135
137
  * Defaults to false.
136
138
  */
137
139
  _resetConfiguration?: boolean;
140
+ /**
141
+ * The maximum number of spans to export at a time. Optional.
142
+ * Defaults to the default OTLP span processor batch size.
143
+ * (512 at the time of writing)
144
+ */
145
+ maxExportBatchSize?: number;
138
146
  }
139
147
 
140
148
  declare global {
@@ -146,10 +154,9 @@ declare global {
146
154
  */
147
155
  interface EvaluatorConfig {
148
156
  /**
149
- * The number of data points to evaluate in one batch. This many
150
- * data points will be evaluated in parallel. Defaults to 5.
157
+ * The number of data points to evaluate in parallel at a time. Defaults to 5.
151
158
  */
152
- batchSize?: number;
159
+ concurrencyLimit?: number;
153
160
  /**
154
161
  * The project API key to use for the evaluation. If not provided,
155
162
  * the API key from the environment variable `LMNR_PROJECT_API_KEY` will be used.
@@ -188,6 +195,10 @@ interface EvaluatorConfig {
188
195
  * Defines default log level for SDK and all instrumentations.
189
196
  */
190
197
  logLevel?: "debug" | "info" | "warn" | "error";
198
+ /**
199
+ * Maximum number of spans to export at a time. Defaults to 64.
200
+ */
201
+ traceExportBatchSize?: number;
191
202
  }
192
203
  /**
193
204
  * Datapoint is a single data point in the evaluation. `D` is the type of the input data,
@@ -253,6 +264,10 @@ interface EvaluationConstructorProps<D, T, O> {
253
264
  * Optional group id of the evaluation. Only evaluations within the same
254
265
  * group_id can be visually compared. Defaults to "default".
255
266
  */
267
+ groupName?: string;
268
+ /**
269
+ * Deprecated. Use `groupName` instead.
270
+ */
256
271
  groupId?: string;
257
272
  /**
258
273
  * Optional override configurations for the evaluator.
@@ -266,15 +281,17 @@ declare class Evaluation<D, T, O> {
266
281
  private executor;
267
282
  private evaluators;
268
283
  private humanEvaluators?;
269
- private groupId?;
284
+ private groupName?;
270
285
  private name?;
271
- private batchSize;
286
+ private concurrencyLimit;
272
287
  private traceDisableBatch;
273
288
  private traceExportTimeoutMillis?;
274
- constructor({ data, executor, evaluators, humanEvaluators, groupId, name, config }: EvaluationConstructorProps<D, T, O>);
289
+ private traceExportBatchSize;
290
+ private uploadPromises;
291
+ constructor({ data, executor, evaluators, humanEvaluators, groupName, name, config }: EvaluationConstructorProps<D, T, O>);
275
292
  run(): Promise<void>;
276
- evaluateInBatches(): Promise<EvaluationDatapoint<D, T, O>[]>;
277
- private evaluateBatch;
293
+ evaluateInBatches(evalId: StringUUID): Promise<EvaluationDatapoint<D, T, O>[]>;
294
+ private evaluateDatapoint;
278
295
  private getLength;
279
296
  }
280
297
  /**
@@ -292,15 +309,15 @@ declare class Evaluation<D, T, O> {
292
309
  * returns.
293
310
  * @param props.humanEvaluators [Beta] Array of instances of {@link HumanEvaluator}.
294
311
  * For now, HumanEvaluator only holds the queue name.
295
- * @param props.groupId Group name which is same as the feature you are evaluating
296
- * in your project or application. Defaults to "default".
312
+ * @param props.groupId Deprecated. Use `groupName` instead. Group name,
313
+ * same as the feature you are evaluating in your project or application.
314
+ * Evaluations within the same group can be visually compared.
315
+ * Defaults to "default".
297
316
  * @param props.name Optional name of the evaluation. Used to easily identify
298
317
  * the evaluation in the group.
299
318
  * @param props.config Optional override configurations for the evaluator.
300
319
  */
301
- declare function evaluate<D, T, O>({ data, executor, evaluators, humanEvaluators, groupId, name, config }: EvaluationConstructorProps<D, T, O>): Promise<void>;
302
-
303
- type StringUUID = `${string}-${string}-${string}-${string}-${string}`;
320
+ declare function evaluate<D, T, O>({ data, executor, evaluators, humanEvaluators, groupName, groupId, name, config }: EvaluationConstructorProps<D, T, O>): Promise<void>;
304
321
 
305
322
  type ChatMessage = {
306
323
  role: 'user' | 'assistant' | 'system';
@@ -326,7 +343,7 @@ type Event = {
326
343
  spanId: StringUUID;
327
344
  value: number | string | null;
328
345
  };
329
- type CreateEvaluationResponse = {
346
+ type InitEvaluationResponse = {
330
347
  id: StringUUID;
331
348
  createdAt: Date;
332
349
  groupId: string;
@@ -339,6 +356,7 @@ type EvaluationDatapoint<D, T, O> = {
339
356
  executorOutput?: O;
340
357
  scores: Record<string, number>;
341
358
  traceId: string;
359
+ index: number;
342
360
  humanEvaluators?: HumanEvaluator[];
343
361
  executorSpanId?: string;
344
362
  };
@@ -399,6 +417,7 @@ interface LaminarInitializeProps {
399
417
  traceExportTimeoutMillis?: number;
400
418
  logLevel?: "debug" | "info" | "warn" | "error";
401
419
  useExternalTracerProvider?: boolean;
420
+ maxExportBatchSize?: number;
402
421
  }
403
422
  declare class Laminar {
404
423
  private static baseHttpUrl;
@@ -460,7 +479,7 @@ declare class Laminar {
460
479
  *
461
480
  * @throws {Error} - If project API key is not set
462
481
  */
463
- static initialize({ projectApiKey, env, baseUrl, httpPort, grpcPort, instrumentModules, useExternalTracerProvider, preserveNextJsSpans, disableBatch, traceExportTimeoutMillis, logLevel, }: LaminarInitializeProps): void;
482
+ static initialize({ projectApiKey, env, baseUrl, httpPort, grpcPort, instrumentModules, useExternalTracerProvider, preserveNextJsSpans, disableBatch, traceExportTimeoutMillis, logLevel, maxExportBatchSize, }: LaminarInitializeProps): void;
464
483
  /**
465
484
  * Check if Laminar has been initialized. Utility to make sure other methods
466
485
  * are called after initialization.
@@ -653,11 +672,11 @@ declare class Laminar {
653
672
  */
654
673
  static withSpan<T>(span: Span, fn: () => T, endOnExit?: boolean): T | Promise<T>;
655
674
  static shutdown(): Promise<void>;
656
- static createEvaluation<D, T, O>({ groupId, name, data }: {
657
- groupId?: string;
675
+ static initEvaluation<D, T, O>({ groupName, name, }: {
676
+ groupName?: string;
658
677
  name?: string;
659
- data: EvaluationDatapoint<D, T, O>[];
660
- }): Promise<CreateEvaluationResponse>;
678
+ }): Promise<InitEvaluationResponse>;
679
+ static saveEvalDatapoints<D, T, O>(evalId: StringUUID, datapoints: EvaluationDatapoint<D, T, O>[], groupName?: string): Promise<void>;
661
680
  static getDatapoints<D, T>({ datasetName, offset, limit, }: {
662
681
  datasetName: string;
663
682
  offset: number;
package/dist/index.d.ts CHANGED
@@ -35,6 +35,8 @@ declare class LaminarDataset<D, T> extends EvaluationDataset<D, T> {
35
35
  get(index: number): Promise<Datapoint<D, T>>;
36
36
  }
37
37
 
38
+ type StringUUID = `${string}-${string}-${string}-${string}-${string}`;
39
+
38
40
  /**
39
41
  * Options for initializing the Traceloop SDK.
40
42
  */
@@ -135,6 +137,12 @@ interface InitializeOptions {
135
137
  * Defaults to false.
136
138
  */
137
139
  _resetConfiguration?: boolean;
140
+ /**
141
+ * The maximum number of spans to export at a time. Optional.
142
+ * Defaults to the default OTLP span processor batch size.
143
+ * (512 at the time of writing)
144
+ */
145
+ maxExportBatchSize?: number;
138
146
  }
139
147
 
140
148
  declare global {
@@ -146,10 +154,9 @@ declare global {
146
154
  */
147
155
  interface EvaluatorConfig {
148
156
  /**
149
- * The number of data points to evaluate in one batch. This many
150
- * data points will be evaluated in parallel. Defaults to 5.
157
+ * The number of data points to evaluate in parallel at a time. Defaults to 5.
151
158
  */
152
- batchSize?: number;
159
+ concurrencyLimit?: number;
153
160
  /**
154
161
  * The project API key to use for the evaluation. If not provided,
155
162
  * the API key from the environment variable `LMNR_PROJECT_API_KEY` will be used.
@@ -188,6 +195,10 @@ interface EvaluatorConfig {
188
195
  * Defines default log level for SDK and all instrumentations.
189
196
  */
190
197
  logLevel?: "debug" | "info" | "warn" | "error";
198
+ /**
199
+ * Maximum number of spans to export at a time. Defaults to 64.
200
+ */
201
+ traceExportBatchSize?: number;
191
202
  }
192
203
  /**
193
204
  * Datapoint is a single data point in the evaluation. `D` is the type of the input data,
@@ -253,6 +264,10 @@ interface EvaluationConstructorProps<D, T, O> {
253
264
  * Optional group id of the evaluation. Only evaluations within the same
254
265
  * group_id can be visually compared. Defaults to "default".
255
266
  */
267
+ groupName?: string;
268
+ /**
269
+ * Deprecated. Use `groupName` instead.
270
+ */
256
271
  groupId?: string;
257
272
  /**
258
273
  * Optional override configurations for the evaluator.
@@ -266,15 +281,17 @@ declare class Evaluation<D, T, O> {
266
281
  private executor;
267
282
  private evaluators;
268
283
  private humanEvaluators?;
269
- private groupId?;
284
+ private groupName?;
270
285
  private name?;
271
- private batchSize;
286
+ private concurrencyLimit;
272
287
  private traceDisableBatch;
273
288
  private traceExportTimeoutMillis?;
274
- constructor({ data, executor, evaluators, humanEvaluators, groupId, name, config }: EvaluationConstructorProps<D, T, O>);
289
+ private traceExportBatchSize;
290
+ private uploadPromises;
291
+ constructor({ data, executor, evaluators, humanEvaluators, groupName, name, config }: EvaluationConstructorProps<D, T, O>);
275
292
  run(): Promise<void>;
276
- evaluateInBatches(): Promise<EvaluationDatapoint<D, T, O>[]>;
277
- private evaluateBatch;
293
+ evaluateInBatches(evalId: StringUUID): Promise<EvaluationDatapoint<D, T, O>[]>;
294
+ private evaluateDatapoint;
278
295
  private getLength;
279
296
  }
280
297
  /**
@@ -292,15 +309,15 @@ declare class Evaluation<D, T, O> {
292
309
  * returns.
293
310
  * @param props.humanEvaluators [Beta] Array of instances of {@link HumanEvaluator}.
294
311
  * For now, HumanEvaluator only holds the queue name.
295
- * @param props.groupId Group name which is same as the feature you are evaluating
296
- * in your project or application. Defaults to "default".
312
+ * @param props.groupId Deprecated. Use `groupName` instead. Group name,
313
+ * same as the feature you are evaluating in your project or application.
314
+ * Evaluations within the same group can be visually compared.
315
+ * Defaults to "default".
297
316
  * @param props.name Optional name of the evaluation. Used to easily identify
298
317
  * the evaluation in the group.
299
318
  * @param props.config Optional override configurations for the evaluator.
300
319
  */
301
- declare function evaluate<D, T, O>({ data, executor, evaluators, humanEvaluators, groupId, name, config }: EvaluationConstructorProps<D, T, O>): Promise<void>;
302
-
303
- type StringUUID = `${string}-${string}-${string}-${string}-${string}`;
320
+ declare function evaluate<D, T, O>({ data, executor, evaluators, humanEvaluators, groupName, groupId, name, config }: EvaluationConstructorProps<D, T, O>): Promise<void>;
304
321
 
305
322
  type ChatMessage = {
306
323
  role: 'user' | 'assistant' | 'system';
@@ -326,7 +343,7 @@ type Event = {
326
343
  spanId: StringUUID;
327
344
  value: number | string | null;
328
345
  };
329
- type CreateEvaluationResponse = {
346
+ type InitEvaluationResponse = {
330
347
  id: StringUUID;
331
348
  createdAt: Date;
332
349
  groupId: string;
@@ -339,6 +356,7 @@ type EvaluationDatapoint<D, T, O> = {
339
356
  executorOutput?: O;
340
357
  scores: Record<string, number>;
341
358
  traceId: string;
359
+ index: number;
342
360
  humanEvaluators?: HumanEvaluator[];
343
361
  executorSpanId?: string;
344
362
  };
@@ -399,6 +417,7 @@ interface LaminarInitializeProps {
399
417
  traceExportTimeoutMillis?: number;
400
418
  logLevel?: "debug" | "info" | "warn" | "error";
401
419
  useExternalTracerProvider?: boolean;
420
+ maxExportBatchSize?: number;
402
421
  }
403
422
  declare class Laminar {
404
423
  private static baseHttpUrl;
@@ -460,7 +479,7 @@ declare class Laminar {
460
479
  *
461
480
  * @throws {Error} - If project API key is not set
462
481
  */
463
- static initialize({ projectApiKey, env, baseUrl, httpPort, grpcPort, instrumentModules, useExternalTracerProvider, preserveNextJsSpans, disableBatch, traceExportTimeoutMillis, logLevel, }: LaminarInitializeProps): void;
482
+ static initialize({ projectApiKey, env, baseUrl, httpPort, grpcPort, instrumentModules, useExternalTracerProvider, preserveNextJsSpans, disableBatch, traceExportTimeoutMillis, logLevel, maxExportBatchSize, }: LaminarInitializeProps): void;
464
483
  /**
465
484
  * Check if Laminar has been initialized. Utility to make sure other methods
466
485
  * are called after initialization.
@@ -653,11 +672,11 @@ declare class Laminar {
653
672
  */
654
673
  static withSpan<T>(span: Span, fn: () => T, endOnExit?: boolean): T | Promise<T>;
655
674
  static shutdown(): Promise<void>;
656
- static createEvaluation<D, T, O>({ groupId, name, data }: {
657
- groupId?: string;
675
+ static initEvaluation<D, T, O>({ groupName, name, }: {
676
+ groupName?: string;
658
677
  name?: string;
659
- data: EvaluationDatapoint<D, T, O>[];
660
- }): Promise<CreateEvaluationResponse>;
678
+ }): Promise<InitEvaluationResponse>;
679
+ static saveEvalDatapoints<D, T, O>(evalId: StringUUID, datapoints: EvaluationDatapoint<D, T, O>[], groupName?: string): Promise<void>;
661
680
  static getDatapoints<D, T>({ datasetName, offset, limit, }: {
662
681
  datasetName: string;
663
682
  offset: number;