@lmnr-ai/lmnr 0.4.15 → 0.4.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli.js +1 -2
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +1 -2
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +33 -8
- package/dist/index.d.ts +33 -8
- package/dist/index.js +38 -235
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +37 -236
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -2
- package/src/datasets.ts +2 -2
- package/src/evaluations.ts +33 -8
- package/src/index.ts +1 -1
- package/src/sdk/tracing/attributes.ts +1 -0
- package/src/sdk/tracing/decorators.ts +3 -4
- package/src/sdk/tracing/index.ts +25 -60
- package/src/sdk/telemetry/telemetry.ts +0 -86
package/README.md
CHANGED
|
@@ -41,7 +41,6 @@ This can be useful if you want to trace a request handler or a function which co
|
|
|
41
41
|
### Example
|
|
42
42
|
|
|
43
43
|
```javascript
|
|
44
|
-
const { Configuration, OpenAIApi } = require("openai");
|
|
45
44
|
import { OpenAI } from 'openai';
|
|
46
45
|
import { Laminar as L, observe } from '@lmnr-ai/lmnr';
|
|
47
46
|
|
|
@@ -96,7 +95,7 @@ npm install @lmnr-ai/lmnr
|
|
|
96
95
|
|
|
97
96
|
Create a file named `my-first-eval.ts` with the following code:
|
|
98
97
|
|
|
99
|
-
```
|
|
98
|
+
```javascript
|
|
100
99
|
import { evaluate } from '@lmnr-ai/lmnr';
|
|
101
100
|
|
|
102
101
|
const writePoem = ({topic}: {topic: string}) => {
|
|
@@ -136,6 +135,7 @@ You can run evaluations locally by providing executor (part of the logic used in
|
|
|
136
135
|
- `executor` – the logic you want to evaluate. This function must take `data` as the first argument, and produce any output.
|
|
137
136
|
- `evaluators` – Object which maps evaluator names to evaluators. Each evaluator is a function that takes output of executor as the first argument, `target` as the second argument and produces numeric scores. Each function can produce either a single number or `Record<string, number>` of scores.
|
|
138
137
|
- `name` – optional name for the evaluation. Automatically generated if not provided.
|
|
138
|
+
- `groupId` – optional group name for evaluation. Evaluations within the same group can be compared visually side by side.
|
|
139
139
|
- `config` – optional additional override parameters.
|
|
140
140
|
|
|
141
141
|
\* If you already have the outputs of executors you want to evaluate, you can specify the executor as an identity function, that takes in `data` and returns only needed value(s) from it.
|
package/dist/cli.js
CHANGED
|
@@ -36,7 +36,7 @@ var require_package = __commonJS({
|
|
|
36
36
|
"package.json"(exports2, module2) {
|
|
37
37
|
module2.exports = {
|
|
38
38
|
name: "@lmnr-ai/lmnr",
|
|
39
|
-
version: "0.4.
|
|
39
|
+
version: "0.4.17",
|
|
40
40
|
description: "TypeScript SDK for Laminar AI",
|
|
41
41
|
main: "dist/index.js",
|
|
42
42
|
types: "dist/index.d.ts",
|
|
@@ -114,7 +114,6 @@ var require_package = __commonJS({
|
|
|
114
114
|
argparse: ">=2.0",
|
|
115
115
|
"cli-progress": ">=3.12",
|
|
116
116
|
esbuild: ">=0.23",
|
|
117
|
-
"posthog-node": ">=4.0",
|
|
118
117
|
uuid: ">=10.0.0"
|
|
119
118
|
}
|
|
120
119
|
};
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.
|
|
1
|
+
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.17\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.27.3\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.679.0\",\n \"@azure/openai\": \"1.0.0-beta.13\",\n \"@google-cloud/aiplatform\": \"^3.29.0\",\n \"@google-cloud/vertexai\": \"^1.7.0\",\n \"@langchain/core\": \"^0.3.2\",\n \"@pinecone-database/pinecone\": \"^3.0.3\",\n \"@qdrant/js-client-rest\": \"^1.11.0\",\n \"@types/argparse\": \"^2.0.16\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.5.5\",\n \"@types/semver\": \"^7.5.8\",\n \"@types/uuid\": \"^10.0.0\",\n \"bufferutil\": \"^4.0.8\",\n \"chromadb\": \"^1.9.2\",\n \"cohere-ai\": \"^7.14.0\",\n \"langchain\": \"^0.3.2\",\n \"llamaindex\": \"^0.7.10\",\n \"openai\": \"^4.62.1\",\n \"runnables\": \"link:@langchain/core/runnables\",\n \"tsup\": \"^8.3.0\",\n \"typescript\": \"^5.6.2\",\n \"vectorstores\": \"link:@langchain/core/vectorstores\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.11.3\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^1.26.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.53.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.53.0\",\n \"@opentelemetry/instrumentation\": \"^0.53.0\",\n \"@opentelemetry/sdk-node\": \"^0.53.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.26.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.26.0\",\n \"@opentelemetry/semantic-conventions\": \"^1.27.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.3\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.1\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n \"@traceloop/instrumentation-openai\": \"^0.11.3\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.1\",\n \"argparse\": \">=2.0\",\n \"cli-progress\": \">=3.12\",\n \"esbuild\": \">=0.23\",\n \"uuid\": \">=10.0.0\"\n }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // TODO: Figure out how to remove all ts-ignores\n // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parser_eval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n });\n\n parser_eval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run\",\n });\n\n parser_eval.set_defaults({\n func: async (args: any) => {\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [args.file],\n outfile: \"tmp_out.js\",\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"node_modules/*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n process.exit(1);\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n await evaluation.run();\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,iBAAAA,UAAAC,SAAA;AAAA,IAAAA,QAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,WAAa;AAAA,QACb,MAAQ;AAAA,QACR,YAAc;AAAA,QACd,cAAgB;AAAA,MAClB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;AClFA;AAAA;AAAA;AAAA;AAAA;AAEA,sBAA+B;AAC/B,cAAyB;AAEzB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,OAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACjB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAC9B,MAAM;AAAA,IACN,aAAa;AAAA,EACjB,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACrC,OAAO;AAAA,IACP,MAAM;AAAA,EACV,CAAC;AAED,QAAM,cAAc,WAAW,WAAW,QAAQ;AAAA,IAC9C,aAAa;AAAA,EACjB,CAAC;AAED,cAAY,aAAa,QAAQ;AAAA,IAC7B,MAAM;AAAA,EACV,CAAC;AAED,cAAY,aAAa;AAAA,IACrB,MAAM,OAAOC,UAAc;AAEvB,YAAM,eAAe;AAAA,QACnB,aAAa,CAACA,MAAK,IAAI;AAAA,QACvB,SAAS;AAAA,QACT,OAAO;AAAA;AAAA,QACP,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU,CAAC,gBAAgB;AAAA,MAC7B;AAEA,YAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,UAAI,CAAC,OAAO,aAAa;AACrB,gBAAQ,MAAM,4CAA4C;AAC1D,gBAAQ,KAAK,CAAC;AAAA,MAClB;AAEA,YAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,YAAM,aAAa,WAAW;AAAA,QAC5B,UAAUA,MAAK;AAAA,QACf,YAAY;AAAA,MACd,CAAC;AAGD,YAAM,WAAW,IAAI;AAAA,IACzB;AAAA,EACJ,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC5B;AAEA,IAAI;","names":["exports","module","args"]}
|
package/dist/cli.mjs
CHANGED
|
@@ -15,7 +15,7 @@ var require_package = __commonJS({
|
|
|
15
15
|
"package.json"(exports, module) {
|
|
16
16
|
module.exports = {
|
|
17
17
|
name: "@lmnr-ai/lmnr",
|
|
18
|
-
version: "0.4.
|
|
18
|
+
version: "0.4.17",
|
|
19
19
|
description: "TypeScript SDK for Laminar AI",
|
|
20
20
|
main: "dist/index.js",
|
|
21
21
|
types: "dist/index.d.ts",
|
|
@@ -93,7 +93,6 @@ var require_package = __commonJS({
|
|
|
93
93
|
argparse: ">=2.0",
|
|
94
94
|
"cli-progress": ">=3.12",
|
|
95
95
|
esbuild: ">=0.23",
|
|
96
|
-
"posthog-node": ">=4.0",
|
|
97
96
|
uuid: ">=10.0.0"
|
|
98
97
|
}
|
|
99
98
|
};
|
package/dist/cli.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.
|
|
1
|
+
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.17\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.27.3\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.679.0\",\n \"@azure/openai\": \"1.0.0-beta.13\",\n \"@google-cloud/aiplatform\": \"^3.29.0\",\n \"@google-cloud/vertexai\": \"^1.7.0\",\n \"@langchain/core\": \"^0.3.2\",\n \"@pinecone-database/pinecone\": \"^3.0.3\",\n \"@qdrant/js-client-rest\": \"^1.11.0\",\n \"@types/argparse\": \"^2.0.16\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.5.5\",\n \"@types/semver\": \"^7.5.8\",\n \"@types/uuid\": \"^10.0.0\",\n \"bufferutil\": \"^4.0.8\",\n \"chromadb\": \"^1.9.2\",\n \"cohere-ai\": \"^7.14.0\",\n \"langchain\": \"^0.3.2\",\n \"llamaindex\": \"^0.7.10\",\n \"openai\": \"^4.62.1\",\n \"runnables\": \"link:@langchain/core/runnables\",\n \"tsup\": \"^8.3.0\",\n \"typescript\": \"^5.6.2\",\n \"vectorstores\": \"link:@langchain/core/vectorstores\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.11.3\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^1.26.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.53.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.53.0\",\n \"@opentelemetry/instrumentation\": \"^0.53.0\",\n \"@opentelemetry/sdk-node\": \"^0.53.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.26.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.26.0\",\n \"@opentelemetry/semantic-conventions\": \"^1.27.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.3\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.1\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n \"@traceloop/instrumentation-openai\": \"^0.11.3\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.1\",\n \"argparse\": \">=2.0\",\n \"cli-progress\": \">=3.12\",\n \"esbuild\": \">=0.23\",\n \"uuid\": \">=10.0.0\"\n }\n}\n","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // TODO: Figure out how to remove all ts-ignores\n // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parser_eval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n });\n\n parser_eval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run\",\n });\n\n parser_eval.set_defaults({\n func: async (args: any) => {\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [args.file],\n outfile: \"tmp_out.js\",\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"node_modules/*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n process.exit(1);\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n await evaluation.run();\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,WAAa;AAAA,QACb,MAAQ;AAAA,QACR,YAAc;AAAA,QACd,cAAgB;AAAA,MAClB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;AChFA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AAEzB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,SAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACjB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAC9B,MAAM;AAAA,IACN,aAAa;AAAA,EACjB,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACrC,OAAO;AAAA,IACP,MAAM;AAAA,EACV,CAAC;AAED,QAAM,cAAc,WAAW,WAAW,QAAQ;AAAA,IAC9C,aAAa;AAAA,EACjB,CAAC;AAED,cAAY,aAAa,QAAQ;AAAA,IAC7B,MAAM;AAAA,EACV,CAAC;AAED,cAAY,aAAa;AAAA,IACrB,MAAM,OAAOA,UAAc;AAEvB,YAAM,eAAe;AAAA,QACnB,aAAa,CAACA,MAAK,IAAI;AAAA,QACvB,SAAS;AAAA,QACT,OAAO;AAAA;AAAA,QACP,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU,CAAC,gBAAgB;AAAA,MAC7B;AAEA,YAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,UAAI,CAAC,OAAO,aAAa;AACrB,gBAAQ,MAAM,4CAA4C;AAC1D,gBAAQ,KAAK,CAAC;AAAA,MAClB;AAEA,YAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,YAAM,aAAa,WAAW;AAAA,QAC5B,UAAUA,MAAK;AAAA,QACf,YAAY;AAAA,MACd,CAAC;AAGD,YAAM,WAAW,IAAI;AAAA,IACzB;AAAA,EACJ,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC5B;AAEA,IAAI;","names":["args"]}
|
package/dist/index.d.mts
CHANGED
|
@@ -18,12 +18,12 @@ import * as llamaindex from 'llamaindex';
|
|
|
18
18
|
import * as chromadb from 'chromadb';
|
|
19
19
|
import * as qdrant from '@qdrant/js-client-rest';
|
|
20
20
|
|
|
21
|
-
declare abstract class
|
|
21
|
+
declare abstract class EvaluationDataset<D, T> {
|
|
22
22
|
slice(start: number, end: number): Promise<Datapoint<D, T>[]>;
|
|
23
23
|
abstract size(): Promise<number> | number;
|
|
24
24
|
abstract get(index: number): Promise<Datapoint<D, T>> | Datapoint<D, T>;
|
|
25
25
|
}
|
|
26
|
-
declare class LaminarDataset<D, T> extends
|
|
26
|
+
declare class LaminarDataset<D, T> extends EvaluationDataset<D, T> {
|
|
27
27
|
private fetchedItems;
|
|
28
28
|
private len;
|
|
29
29
|
private offset;
|
|
@@ -124,11 +124,35 @@ declare global {
|
|
|
124
124
|
* Configuration for the Evaluator
|
|
125
125
|
*/
|
|
126
126
|
interface EvaluatorConfig {
|
|
127
|
+
/**
|
|
128
|
+
* The number of data points to evaluate in one batch. This many
|
|
129
|
+
* data points will be evaluated in parallel. Defaults to 5.
|
|
130
|
+
*/
|
|
127
131
|
batchSize?: number;
|
|
132
|
+
/**
|
|
133
|
+
* The project API key to use for the evaluation. If not provided,
|
|
134
|
+
* the API key from the environment variable `LMNR_PROJECT_API_KEY` will be used.
|
|
135
|
+
*/
|
|
128
136
|
projectApiKey?: string;
|
|
137
|
+
/**
|
|
138
|
+
* The base URL of the Laminar API. If not provided, the default is
|
|
139
|
+
* `https://api.lmnr.ai`. Useful with self-hosted Laminar instances.
|
|
140
|
+
* Do NOT include the port in the URL, use `httpPort` and `grpcPort` instead.
|
|
141
|
+
*/
|
|
129
142
|
baseUrl?: string;
|
|
143
|
+
/**
|
|
144
|
+
* The HTTP port of the Laminar API. If not provided, the default is 443.
|
|
145
|
+
*/
|
|
130
146
|
httpPort?: number;
|
|
147
|
+
/**
|
|
148
|
+
* The gRPC port of the Laminar API. If not provided, the default is 8443.
|
|
149
|
+
*/
|
|
131
150
|
grpcPort?: number;
|
|
151
|
+
/**
|
|
152
|
+
* Object with modules to instrument. If not provided, all
|
|
153
|
+
* available modules are instrumented.
|
|
154
|
+
* See {@link https://docs.lmnr.ai/tracing/automatic-instrumentation}
|
|
155
|
+
*/
|
|
132
156
|
instrumentModules?: InitializeOptions['instrumentModules'];
|
|
133
157
|
}
|
|
134
158
|
/**
|
|
@@ -159,7 +183,7 @@ interface EvaluationConstructorProps<D, T, O> {
|
|
|
159
183
|
* List of data points to evaluate. `data` is the input to the executor function,
|
|
160
184
|
* `target` is the input to the evaluator function.
|
|
161
185
|
*/
|
|
162
|
-
data: (Datapoint<D, T>[]) |
|
|
186
|
+
data: (Datapoint<D, T>[]) | EvaluationDataset<D, T>;
|
|
163
187
|
/**
|
|
164
188
|
* The executor function. Takes the data point + any additional arguments
|
|
165
189
|
* and returns the output to evaluate.
|
|
@@ -175,13 +199,14 @@ interface EvaluationConstructorProps<D, T, O> {
|
|
|
175
199
|
*/
|
|
176
200
|
evaluators: Record<string, EvaluatorFunction<O, T>>;
|
|
177
201
|
/**
|
|
178
|
-
*
|
|
202
|
+
* Name of the evaluation. If not provided, a random name will be assigned.
|
|
179
203
|
*/
|
|
180
|
-
|
|
204
|
+
name?: string;
|
|
181
205
|
/**
|
|
182
|
-
*
|
|
206
|
+
* Optional group id of the evaluation. Only evaluations within the same
|
|
207
|
+
* group_id can be visually compared. Defaults to "default".
|
|
183
208
|
*/
|
|
184
|
-
|
|
209
|
+
groupId?: string;
|
|
185
210
|
/**
|
|
186
211
|
* Optional override configurations for the evaluator.
|
|
187
212
|
*/
|
|
@@ -482,4 +507,4 @@ interface ObserveOptions {
|
|
|
482
507
|
*/
|
|
483
508
|
declare function observe<A extends unknown[], F extends (...args: A) => ReturnType<F>>({ name, sessionId, userId, traceType, spanType, }: ObserveOptions, fn: F, ...args: A): Promise<ReturnType<F>>;
|
|
484
509
|
|
|
485
|
-
export { type ChatMessage, type Datapoint, Dataset, type EvaluateEvent, type Event, Laminar, LaminarAttributes, LaminarDataset, type NodeInput, type PipelineRunRequest, type PipelineRunResponse, evaluate, observe };
|
|
510
|
+
export { type ChatMessage, type Datapoint, EvaluationDataset as Dataset, type EvaluateEvent, type Event, Laminar, LaminarAttributes, LaminarDataset, type NodeInput, type PipelineRunRequest, type PipelineRunResponse, evaluate, observe };
|
package/dist/index.d.ts
CHANGED
|
@@ -18,12 +18,12 @@ import * as llamaindex from 'llamaindex';
|
|
|
18
18
|
import * as chromadb from 'chromadb';
|
|
19
19
|
import * as qdrant from '@qdrant/js-client-rest';
|
|
20
20
|
|
|
21
|
-
declare abstract class
|
|
21
|
+
declare abstract class EvaluationDataset<D, T> {
|
|
22
22
|
slice(start: number, end: number): Promise<Datapoint<D, T>[]>;
|
|
23
23
|
abstract size(): Promise<number> | number;
|
|
24
24
|
abstract get(index: number): Promise<Datapoint<D, T>> | Datapoint<D, T>;
|
|
25
25
|
}
|
|
26
|
-
declare class LaminarDataset<D, T> extends
|
|
26
|
+
declare class LaminarDataset<D, T> extends EvaluationDataset<D, T> {
|
|
27
27
|
private fetchedItems;
|
|
28
28
|
private len;
|
|
29
29
|
private offset;
|
|
@@ -124,11 +124,35 @@ declare global {
|
|
|
124
124
|
* Configuration for the Evaluator
|
|
125
125
|
*/
|
|
126
126
|
interface EvaluatorConfig {
|
|
127
|
+
/**
|
|
128
|
+
* The number of data points to evaluate in one batch. This many
|
|
129
|
+
* data points will be evaluated in parallel. Defaults to 5.
|
|
130
|
+
*/
|
|
127
131
|
batchSize?: number;
|
|
132
|
+
/**
|
|
133
|
+
* The project API key to use for the evaluation. If not provided,
|
|
134
|
+
* the API key from the environment variable `LMNR_PROJECT_API_KEY` will be used.
|
|
135
|
+
*/
|
|
128
136
|
projectApiKey?: string;
|
|
137
|
+
/**
|
|
138
|
+
* The base URL of the Laminar API. If not provided, the default is
|
|
139
|
+
* `https://api.lmnr.ai`. Useful with self-hosted Laminar instances.
|
|
140
|
+
* Do NOT include the port in the URL, use `httpPort` and `grpcPort` instead.
|
|
141
|
+
*/
|
|
129
142
|
baseUrl?: string;
|
|
143
|
+
/**
|
|
144
|
+
* The HTTP port of the Laminar API. If not provided, the default is 443.
|
|
145
|
+
*/
|
|
130
146
|
httpPort?: number;
|
|
147
|
+
/**
|
|
148
|
+
* The gRPC port of the Laminar API. If not provided, the default is 8443.
|
|
149
|
+
*/
|
|
131
150
|
grpcPort?: number;
|
|
151
|
+
/**
|
|
152
|
+
* Object with modules to instrument. If not provided, all
|
|
153
|
+
* available modules are instrumented.
|
|
154
|
+
* See {@link https://docs.lmnr.ai/tracing/automatic-instrumentation}
|
|
155
|
+
*/
|
|
132
156
|
instrumentModules?: InitializeOptions['instrumentModules'];
|
|
133
157
|
}
|
|
134
158
|
/**
|
|
@@ -159,7 +183,7 @@ interface EvaluationConstructorProps<D, T, O> {
|
|
|
159
183
|
* List of data points to evaluate. `data` is the input to the executor function,
|
|
160
184
|
* `target` is the input to the evaluator function.
|
|
161
185
|
*/
|
|
162
|
-
data: (Datapoint<D, T>[]) |
|
|
186
|
+
data: (Datapoint<D, T>[]) | EvaluationDataset<D, T>;
|
|
163
187
|
/**
|
|
164
188
|
* The executor function. Takes the data point + any additional arguments
|
|
165
189
|
* and returns the output to evaluate.
|
|
@@ -175,13 +199,14 @@ interface EvaluationConstructorProps<D, T, O> {
|
|
|
175
199
|
*/
|
|
176
200
|
evaluators: Record<string, EvaluatorFunction<O, T>>;
|
|
177
201
|
/**
|
|
178
|
-
*
|
|
202
|
+
* Name of the evaluation. If not provided, a random name will be assigned.
|
|
179
203
|
*/
|
|
180
|
-
|
|
204
|
+
name?: string;
|
|
181
205
|
/**
|
|
182
|
-
*
|
|
206
|
+
* Optional group id of the evaluation. Only evaluations within the same
|
|
207
|
+
* group_id can be visually compared. Defaults to "default".
|
|
183
208
|
*/
|
|
184
|
-
|
|
209
|
+
groupId?: string;
|
|
185
210
|
/**
|
|
186
211
|
* Optional override configurations for the evaluator.
|
|
187
212
|
*/
|
|
@@ -482,4 +507,4 @@ interface ObserveOptions {
|
|
|
482
507
|
*/
|
|
483
508
|
declare function observe<A extends unknown[], F extends (...args: A) => ReturnType<F>>({ name, sessionId, userId, traceType, spanType, }: ObserveOptions, fn: F, ...args: A): Promise<ReturnType<F>>;
|
|
484
509
|
|
|
485
|
-
export { type ChatMessage, type Datapoint, Dataset, type EvaluateEvent, type Event, Laminar, LaminarAttributes, LaminarDataset, type NodeInput, type PipelineRunRequest, type PipelineRunResponse, evaluate, observe };
|
|
510
|
+
export { type ChatMessage, type Datapoint, EvaluationDataset as Dataset, type EvaluateEvent, type Event, Laminar, LaminarAttributes, LaminarDataset, type NodeInput, type PipelineRunRequest, type PipelineRunResponse, evaluate, observe };
|