@lmnr-ai/lmnr 0.4.12 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -12
- package/dist/cli.js +35 -35
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +35 -35
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +21 -52
- package/dist/index.d.ts +21 -52
- package/dist/index.js +96 -154
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +96 -154
- package/dist/index.mjs.map +1 -1
- package/package.json +35 -35
- package/src/evaluations.ts +61 -51
- package/src/laminar.ts +14 -96
- package/src/types.ts +1 -12
package/README.md
CHANGED
|
@@ -18,6 +18,9 @@ npm install @lmnr-ai/lmnr
|
|
|
18
18
|
And then in the code
|
|
19
19
|
|
|
20
20
|
```typescript
|
|
21
|
+
# Only if you are using Next.js
|
|
22
|
+
export NEXT_OTEL_FETCH_DISABLED=1
|
|
23
|
+
|
|
21
24
|
import { Laminar as L } from '@lmnr-ai/lmnr'
|
|
22
25
|
|
|
23
26
|
L.initialize({ projectApiKey: '<PROJECT_API_KEY>' })
|
|
@@ -66,13 +69,9 @@ await observe({name: 'poemWriter'}, async () => await poemWriter('laminar flow')
|
|
|
66
69
|
|
|
67
70
|
### Sending events
|
|
68
71
|
|
|
69
|
-
You can send events
|
|
70
|
-
- `.event(name, value)` – for a pre-defined event with one of possible values.
|
|
71
|
-
- `.evaluate_event(name, evaluator, data, env)` – for an event that is evaluated by evaluator pipeline based on the data.
|
|
72
|
-
|
|
73
|
-
Note that to run an evaluate event, you need to create an evaluator pipeline and create a target version for it.
|
|
72
|
+
You can send laminar events using `L.event(name, value)`.
|
|
74
73
|
|
|
75
|
-
Read our [docs](https://docs.lmnr.ai) to learn more about
|
|
74
|
+
Read our [docs](https://docs.lmnr.ai) to learn more about events and examples.
|
|
76
75
|
|
|
77
76
|
### Example
|
|
78
77
|
|
|
@@ -83,11 +82,6 @@ const poem = response.choices[0].message.content;
|
|
|
83
82
|
|
|
84
83
|
// this will register True or False value with Laminar
|
|
85
84
|
L.event('topic alignment', poem.includes(topic))
|
|
86
|
-
|
|
87
|
-
// this will run the pipeline `checkWordy` with `poem` set as the value
|
|
88
|
-
// of `textInput` node, and write the result as an event with name
|
|
89
|
-
// "excessiveWordiness"
|
|
90
|
-
L.evaluateEvent('excessiveWordiness', 'checkWordy', {'textInput': 'poem'})
|
|
91
85
|
```
|
|
92
86
|
|
|
93
87
|
## Evaluations
|
|
@@ -117,7 +111,8 @@ evaluate({
|
|
|
117
111
|
executor: (data) => writePoem(data),
|
|
118
112
|
evaluators: {
|
|
119
113
|
containsPoem: (output, target) => target.poem.includes(output) ? 1 : 0
|
|
120
|
-
}
|
|
114
|
+
},
|
|
115
|
+
groupId: 'my_first_feature'
|
|
121
116
|
})
|
|
122
117
|
```
|
|
123
118
|
|
package/dist/cli.js
CHANGED
|
@@ -36,7 +36,7 @@ var require_package = __commonJS({
|
|
|
36
36
|
"package.json"(exports2, module2) {
|
|
37
37
|
module2.exports = {
|
|
38
38
|
name: "@lmnr-ai/lmnr",
|
|
39
|
-
version: "0.4.
|
|
39
|
+
version: "0.4.14",
|
|
40
40
|
description: "TypeScript SDK for Laminar AI",
|
|
41
41
|
main: "dist/index.js",
|
|
42
42
|
types: "dist/index.d.ts",
|
|
@@ -64,41 +64,41 @@ var require_package = __commonJS({
|
|
|
64
64
|
},
|
|
65
65
|
homepage: "https://github.com/lmnr-ai/lmnr-ts#README",
|
|
66
66
|
devDependencies: {
|
|
67
|
-
"@anthropic-ai/sdk": "
|
|
68
|
-
"@aws-sdk/client-bedrock-runtime": "
|
|
69
|
-
"@azure/openai": "
|
|
70
|
-
"@google-cloud/aiplatform": "
|
|
71
|
-
"@google-cloud/vertexai": "
|
|
72
|
-
"@langchain/core": "
|
|
73
|
-
"@pinecone-database/pinecone": "
|
|
74
|
-
"@qdrant/js-client-rest": "
|
|
75
|
-
"@types/argparse": "
|
|
76
|
-
"@types/cli-progress": "
|
|
77
|
-
"@types/node": "
|
|
78
|
-
"@types/semver": "
|
|
79
|
-
"@types/uuid": "
|
|
80
|
-
bufferutil: "
|
|
81
|
-
chromadb: "
|
|
82
|
-
"cohere-ai": "
|
|
83
|
-
langchain: "
|
|
84
|
-
llamaindex: "
|
|
85
|
-
openai: "
|
|
67
|
+
"@anthropic-ai/sdk": "^0.27.3",
|
|
68
|
+
"@aws-sdk/client-bedrock-runtime": "^3.654.0",
|
|
69
|
+
"@azure/openai": "1.0.0-beta.13",
|
|
70
|
+
"@google-cloud/aiplatform": "^3.29.0",
|
|
71
|
+
"@google-cloud/vertexai": "^1.7.0",
|
|
72
|
+
"@langchain/core": "^0.3.2",
|
|
73
|
+
"@pinecone-database/pinecone": "^3.0.3",
|
|
74
|
+
"@qdrant/js-client-rest": "^1.11.0",
|
|
75
|
+
"@types/argparse": "^2.0.16",
|
|
76
|
+
"@types/cli-progress": "^3.11.6",
|
|
77
|
+
"@types/node": "^22.5.5",
|
|
78
|
+
"@types/semver": "^7.5.8",
|
|
79
|
+
"@types/uuid": "^10.0.0",
|
|
80
|
+
bufferutil: "^4.0.8",
|
|
81
|
+
chromadb: "^1.8.1",
|
|
82
|
+
"cohere-ai": "^7.13.1",
|
|
83
|
+
langchain: "^0.3.2",
|
|
84
|
+
llamaindex: "^0.6.2",
|
|
85
|
+
openai: "^4.62.1",
|
|
86
86
|
runnables: "link:@langchain/core/runnables",
|
|
87
|
-
tsup: "
|
|
88
|
-
typescript: "
|
|
87
|
+
tsup: "^8.3.0",
|
|
88
|
+
typescript: "^5.6.2",
|
|
89
89
|
vectorstores: "link:@langchain/core/vectorstores"
|
|
90
90
|
},
|
|
91
91
|
dependencies: {
|
|
92
|
-
"@grpc/grpc-js": "
|
|
93
|
-
"@opentelemetry/api": "
|
|
94
|
-
"@opentelemetry/core": "
|
|
95
|
-
"@opentelemetry/exporter-trace-otlp-grpc": "
|
|
96
|
-
"@opentelemetry/exporter-trace-otlp-proto": "
|
|
97
|
-
"@opentelemetry/instrumentation": "
|
|
98
|
-
"@opentelemetry/sdk-node": "
|
|
92
|
+
"@grpc/grpc-js": "^1.11.3",
|
|
93
|
+
"@opentelemetry/api": "^1.9.0",
|
|
94
|
+
"@opentelemetry/core": "^1.26.0",
|
|
95
|
+
"@opentelemetry/exporter-trace-otlp-grpc": "^0.53.0",
|
|
96
|
+
"@opentelemetry/exporter-trace-otlp-proto": "^0.53.0",
|
|
97
|
+
"@opentelemetry/instrumentation": "^0.53.0",
|
|
98
|
+
"@opentelemetry/sdk-node": "^0.53.0",
|
|
99
99
|
"@opentelemetry/sdk-trace-base": "^1.26.0",
|
|
100
100
|
"@opentelemetry/sdk-trace-node": "^1.26.0",
|
|
101
|
-
"@opentelemetry/semantic-conventions": "
|
|
101
|
+
"@opentelemetry/semantic-conventions": "^1.27.0",
|
|
102
102
|
"@traceloop/ai-semantic-conventions": "^0.11.0",
|
|
103
103
|
"@traceloop/instrumentation-anthropic": "^0.11.1",
|
|
104
104
|
"@traceloop/instrumentation-azure": "^0.11.1",
|
|
@@ -111,11 +111,11 @@ var require_package = __commonJS({
|
|
|
111
111
|
"@traceloop/instrumentation-pinecone": "^0.11.1",
|
|
112
112
|
"@traceloop/instrumentation-qdrant": "^0.11.1",
|
|
113
113
|
"@traceloop/instrumentation-vertexai": "^0.11.1",
|
|
114
|
-
argparse: ">=2.0
|
|
115
|
-
"cli-progress": ">=3.
|
|
116
|
-
esbuild: ">=0.
|
|
117
|
-
"posthog-node": ">=
|
|
118
|
-
uuid: ">=
|
|
114
|
+
argparse: ">=2.0",
|
|
115
|
+
"cli-progress": ">=3.12",
|
|
116
|
+
esbuild: ">=0.23",
|
|
117
|
+
"posthog-node": ">=4.0",
|
|
118
|
+
uuid: ">=10.0.0"
|
|
119
119
|
}
|
|
120
120
|
};
|
|
121
121
|
}
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.
|
|
1
|
+
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.14\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.27.3\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.654.0\",\n \"@azure/openai\": \"1.0.0-beta.13\",\n \"@google-cloud/aiplatform\": \"^3.29.0\",\n \"@google-cloud/vertexai\": \"^1.7.0\",\n \"@langchain/core\": \"^0.3.2\",\n \"@pinecone-database/pinecone\": \"^3.0.3\",\n \"@qdrant/js-client-rest\": \"^1.11.0\",\n \"@types/argparse\": \"^2.0.16\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.5.5\",\n \"@types/semver\": \"^7.5.8\",\n \"@types/uuid\": \"^10.0.0\",\n \"bufferutil\": \"^4.0.8\",\n \"chromadb\": \"^1.8.1\",\n \"cohere-ai\": \"^7.13.1\",\n \"langchain\": \"^0.3.2\",\n \"llamaindex\": \"^0.6.2\",\n \"openai\": \"^4.62.1\",\n \"runnables\": \"link:@langchain/core/runnables\",\n \"tsup\": \"^8.3.0\",\n \"typescript\": \"^5.6.2\",\n \"vectorstores\": \"link:@langchain/core/vectorstores\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.11.3\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^1.26.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.53.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.53.0\",\n \"@opentelemetry/instrumentation\": \"^0.53.0\",\n \"@opentelemetry/sdk-node\": \"^0.53.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.26.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.26.0\",\n \"@opentelemetry/semantic-conventions\": \"^1.27.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.1\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.1\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n \"@traceloop/instrumentation-openai\": \"^0.11.1\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.1\",\n \"argparse\": \">=2.0\",\n \"cli-progress\": \">=3.12\",\n \"esbuild\": \">=0.23\",\n \"posthog-node\": \">=4.0\",\n \"uuid\": \">=10.0.0\"\n }\n}","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // TODO: Figure out how to remove all ts-ignores\n // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parser_eval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n });\n\n parser_eval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run\",\n });\n\n parser_eval.set_defaults({\n func: async (args: any) => {\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [args.file],\n outfile: \"tmp_out.js\",\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"node_modules/*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n process.exit(1);\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n await evaluation.run();\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,iBAAAA,UAAAC,SAAA;AAAA,IAAAA,QAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,WAAa;AAAA,QACb,MAAQ;AAAA,QACR,YAAc;AAAA,QACd,cAAgB;AAAA,MAClB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,gBAAgB;AAAA,QAChB,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;ACnFA;AAAA;AAAA;AAAA;AAAA;AAEA,sBAA+B;AAC/B,cAAyB;AAEzB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,OAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACjB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAC9B,MAAM;AAAA,IACN,aAAa;AAAA,EACjB,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACrC,OAAO;AAAA,IACP,MAAM;AAAA,EACV,CAAC;AAED,QAAM,cAAc,WAAW,WAAW,QAAQ;AAAA,IAC9C,aAAa;AAAA,EACjB,CAAC;AAED,cAAY,aAAa,QAAQ;AAAA,IAC7B,MAAM;AAAA,EACV,CAAC;AAED,cAAY,aAAa;AAAA,IACrB,MAAM,OAAOC,UAAc;AAEvB,YAAM,eAAe;AAAA,QACnB,aAAa,CAACA,MAAK,IAAI;AAAA,QACvB,SAAS;AAAA,QACT,OAAO;AAAA;AAAA,QACP,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU,CAAC,gBAAgB;AAAA,MAC7B;AAEA,YAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,UAAI,CAAC,OAAO,aAAa;AACrB,gBAAQ,MAAM,4CAA4C;AAC1D,gBAAQ,KAAK,CAAC;AAAA,MAClB;AAEA,YAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,YAAM,aAAa,WAAW;AAAA,QAC5B,UAAUA,MAAK;AAAA,QACf,YAAY;AAAA,MACd,CAAC;AAGD,YAAM,WAAW,IAAI;AAAA,IACzB;AAAA,EACJ,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC5B;AAEA,IAAI;","names":["exports","module","args"]}
|
package/dist/cli.mjs
CHANGED
|
@@ -15,7 +15,7 @@ var require_package = __commonJS({
|
|
|
15
15
|
"package.json"(exports, module) {
|
|
16
16
|
module.exports = {
|
|
17
17
|
name: "@lmnr-ai/lmnr",
|
|
18
|
-
version: "0.4.
|
|
18
|
+
version: "0.4.14",
|
|
19
19
|
description: "TypeScript SDK for Laminar AI",
|
|
20
20
|
main: "dist/index.js",
|
|
21
21
|
types: "dist/index.d.ts",
|
|
@@ -43,41 +43,41 @@ var require_package = __commonJS({
|
|
|
43
43
|
},
|
|
44
44
|
homepage: "https://github.com/lmnr-ai/lmnr-ts#README",
|
|
45
45
|
devDependencies: {
|
|
46
|
-
"@anthropic-ai/sdk": "
|
|
47
|
-
"@aws-sdk/client-bedrock-runtime": "
|
|
48
|
-
"@azure/openai": "
|
|
49
|
-
"@google-cloud/aiplatform": "
|
|
50
|
-
"@google-cloud/vertexai": "
|
|
51
|
-
"@langchain/core": "
|
|
52
|
-
"@pinecone-database/pinecone": "
|
|
53
|
-
"@qdrant/js-client-rest": "
|
|
54
|
-
"@types/argparse": "
|
|
55
|
-
"@types/cli-progress": "
|
|
56
|
-
"@types/node": "
|
|
57
|
-
"@types/semver": "
|
|
58
|
-
"@types/uuid": "
|
|
59
|
-
bufferutil: "
|
|
60
|
-
chromadb: "
|
|
61
|
-
"cohere-ai": "
|
|
62
|
-
langchain: "
|
|
63
|
-
llamaindex: "
|
|
64
|
-
openai: "
|
|
46
|
+
"@anthropic-ai/sdk": "^0.27.3",
|
|
47
|
+
"@aws-sdk/client-bedrock-runtime": "^3.654.0",
|
|
48
|
+
"@azure/openai": "1.0.0-beta.13",
|
|
49
|
+
"@google-cloud/aiplatform": "^3.29.0",
|
|
50
|
+
"@google-cloud/vertexai": "^1.7.0",
|
|
51
|
+
"@langchain/core": "^0.3.2",
|
|
52
|
+
"@pinecone-database/pinecone": "^3.0.3",
|
|
53
|
+
"@qdrant/js-client-rest": "^1.11.0",
|
|
54
|
+
"@types/argparse": "^2.0.16",
|
|
55
|
+
"@types/cli-progress": "^3.11.6",
|
|
56
|
+
"@types/node": "^22.5.5",
|
|
57
|
+
"@types/semver": "^7.5.8",
|
|
58
|
+
"@types/uuid": "^10.0.0",
|
|
59
|
+
bufferutil: "^4.0.8",
|
|
60
|
+
chromadb: "^1.8.1",
|
|
61
|
+
"cohere-ai": "^7.13.1",
|
|
62
|
+
langchain: "^0.3.2",
|
|
63
|
+
llamaindex: "^0.6.2",
|
|
64
|
+
openai: "^4.62.1",
|
|
65
65
|
runnables: "link:@langchain/core/runnables",
|
|
66
|
-
tsup: "
|
|
67
|
-
typescript: "
|
|
66
|
+
tsup: "^8.3.0",
|
|
67
|
+
typescript: "^5.6.2",
|
|
68
68
|
vectorstores: "link:@langchain/core/vectorstores"
|
|
69
69
|
},
|
|
70
70
|
dependencies: {
|
|
71
|
-
"@grpc/grpc-js": "
|
|
72
|
-
"@opentelemetry/api": "
|
|
73
|
-
"@opentelemetry/core": "
|
|
74
|
-
"@opentelemetry/exporter-trace-otlp-grpc": "
|
|
75
|
-
"@opentelemetry/exporter-trace-otlp-proto": "
|
|
76
|
-
"@opentelemetry/instrumentation": "
|
|
77
|
-
"@opentelemetry/sdk-node": "
|
|
71
|
+
"@grpc/grpc-js": "^1.11.3",
|
|
72
|
+
"@opentelemetry/api": "^1.9.0",
|
|
73
|
+
"@opentelemetry/core": "^1.26.0",
|
|
74
|
+
"@opentelemetry/exporter-trace-otlp-grpc": "^0.53.0",
|
|
75
|
+
"@opentelemetry/exporter-trace-otlp-proto": "^0.53.0",
|
|
76
|
+
"@opentelemetry/instrumentation": "^0.53.0",
|
|
77
|
+
"@opentelemetry/sdk-node": "^0.53.0",
|
|
78
78
|
"@opentelemetry/sdk-trace-base": "^1.26.0",
|
|
79
79
|
"@opentelemetry/sdk-trace-node": "^1.26.0",
|
|
80
|
-
"@opentelemetry/semantic-conventions": "
|
|
80
|
+
"@opentelemetry/semantic-conventions": "^1.27.0",
|
|
81
81
|
"@traceloop/ai-semantic-conventions": "^0.11.0",
|
|
82
82
|
"@traceloop/instrumentation-anthropic": "^0.11.1",
|
|
83
83
|
"@traceloop/instrumentation-azure": "^0.11.1",
|
|
@@ -90,11 +90,11 @@ var require_package = __commonJS({
|
|
|
90
90
|
"@traceloop/instrumentation-pinecone": "^0.11.1",
|
|
91
91
|
"@traceloop/instrumentation-qdrant": "^0.11.1",
|
|
92
92
|
"@traceloop/instrumentation-vertexai": "^0.11.1",
|
|
93
|
-
argparse: ">=2.0
|
|
94
|
-
"cli-progress": ">=3.
|
|
95
|
-
esbuild: ">=0.
|
|
96
|
-
"posthog-node": ">=
|
|
97
|
-
uuid: ">=
|
|
93
|
+
argparse: ">=2.0",
|
|
94
|
+
"cli-progress": ">=3.12",
|
|
95
|
+
esbuild: ">=0.23",
|
|
96
|
+
"posthog-node": ">=4.0",
|
|
97
|
+
uuid: ">=10.0.0"
|
|
98
98
|
}
|
|
99
99
|
};
|
|
100
100
|
}
|
package/dist/cli.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.
|
|
1
|
+
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.14\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.27.3\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.654.0\",\n \"@azure/openai\": \"1.0.0-beta.13\",\n \"@google-cloud/aiplatform\": \"^3.29.0\",\n \"@google-cloud/vertexai\": \"^1.7.0\",\n \"@langchain/core\": \"^0.3.2\",\n \"@pinecone-database/pinecone\": \"^3.0.3\",\n \"@qdrant/js-client-rest\": \"^1.11.0\",\n \"@types/argparse\": \"^2.0.16\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.5.5\",\n \"@types/semver\": \"^7.5.8\",\n \"@types/uuid\": \"^10.0.0\",\n \"bufferutil\": \"^4.0.8\",\n \"chromadb\": \"^1.8.1\",\n \"cohere-ai\": \"^7.13.1\",\n \"langchain\": \"^0.3.2\",\n \"llamaindex\": \"^0.6.2\",\n \"openai\": \"^4.62.1\",\n \"runnables\": \"link:@langchain/core/runnables\",\n \"tsup\": \"^8.3.0\",\n \"typescript\": \"^5.6.2\",\n \"vectorstores\": \"link:@langchain/core/vectorstores\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.11.3\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/core\": \"^1.26.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.53.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.53.0\",\n \"@opentelemetry/instrumentation\": \"^0.53.0\",\n \"@opentelemetry/sdk-node\": \"^0.53.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.26.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.26.0\",\n \"@opentelemetry/semantic-conventions\": \"^1.27.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.1\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.1\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n \"@traceloop/instrumentation-openai\": \"^0.11.1\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.1\",\n \"argparse\": \">=2.0\",\n \"cli-progress\": \">=3.12\",\n \"esbuild\": \">=0.23\",\n \"posthog-node\": \">=4.0\",\n \"uuid\": \">=10.0.0\"\n }\n}","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // TODO: Figure out how to remove all ts-ignores\n // TODO: Cleanup by setting the original values of _evaluation and _set_global_evaluation back\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parser_eval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n });\n\n parser_eval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run\",\n });\n\n parser_eval.set_defaults({\n func: async (args: any) => {\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [args.file],\n outfile: \"tmp_out.js\",\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"node_modules/*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n process.exit(1);\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n await evaluation.run();\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,WAAa;AAAA,QACb,MAAQ;AAAA,QACR,YAAc;AAAA,QACd,cAAgB;AAAA,MAClB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,gBAAgB;AAAA,QAChB,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;ACjFA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AAEzB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAID,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,SAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACjB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAC9B,MAAM;AAAA,IACN,aAAa;AAAA,EACjB,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACrC,OAAO;AAAA,IACP,MAAM;AAAA,EACV,CAAC;AAED,QAAM,cAAc,WAAW,WAAW,QAAQ;AAAA,IAC9C,aAAa;AAAA,EACjB,CAAC;AAED,cAAY,aAAa,QAAQ;AAAA,IAC7B,MAAM;AAAA,EACV,CAAC;AAED,cAAY,aAAa;AAAA,IACrB,MAAM,OAAOA,UAAc;AAEvB,YAAM,eAAe;AAAA,QACnB,aAAa,CAACA,MAAK,IAAI;AAAA,QACvB,SAAS;AAAA,QACT,OAAO;AAAA;AAAA,QACP,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU,CAAC,gBAAgB;AAAA,MAC7B;AAEA,YAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,UAAI,CAAC,OAAO,aAAa;AACrB,gBAAQ,MAAM,4CAA4C;AAC1D,gBAAQ,KAAK,CAAC;AAAA,MAClB;AAEA,YAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,YAAM,aAAa,WAAW;AAAA,QAC5B,UAAUA,MAAK;AAAA,QACf,YAAY;AAAA,MACd,CAAC;AAGD,YAAM,WAAW,IAAI;AAAA,IACzB;AAAA,EACJ,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC5B;AAEA,IAAI;","names":["args"]}
|
package/dist/index.d.mts
CHANGED
|
@@ -54,16 +54,9 @@ type Event = {
|
|
|
54
54
|
type CreateEvaluationResponse = {
|
|
55
55
|
id: StringUUID;
|
|
56
56
|
createdAt: Date;
|
|
57
|
+
groupId: string;
|
|
57
58
|
name: string;
|
|
58
|
-
status: EvaluationStatus;
|
|
59
59
|
projectId: StringUUID;
|
|
60
|
-
metadata: Record<string, any> | null;
|
|
61
|
-
};
|
|
62
|
-
type EvaluationStats = {
|
|
63
|
-
averageScores: Record<string, number>;
|
|
64
|
-
};
|
|
65
|
-
type UpdateEvaluationResponse = {
|
|
66
|
-
stats: EvaluationStats;
|
|
67
60
|
};
|
|
68
61
|
type EvaluationDatapoint<D, T, O> = {
|
|
69
62
|
data: Record<string, any> & D;
|
|
@@ -72,7 +65,6 @@ type EvaluationDatapoint<D, T, O> = {
|
|
|
72
65
|
scores: Record<string, number>;
|
|
73
66
|
traceId: string;
|
|
74
67
|
};
|
|
75
|
-
type EvaluationStatus = 'Started' | 'Finished' | 'Error';
|
|
76
68
|
/**
|
|
77
69
|
* Trace types to categorize traces.
|
|
78
70
|
* They are used as association properties passed to all spans in a trace.
|
|
@@ -184,9 +176,10 @@ declare class Laminar {
|
|
|
184
176
|
* @param project_api_key - Laminar project api key. You can generate one by going
|
|
185
177
|
* to the projects settings page on the Laminar dashboard.
|
|
186
178
|
* If not specified, it will try to read from the LMNR_PROJECT_API_KEY environment variable.
|
|
187
|
-
* @param env - Default environment passed to `run`
|
|
188
|
-
*
|
|
189
|
-
* @param baseUrl - Laminar API url.
|
|
179
|
+
* @param env - Default environment passed to `run` requests, unless overriden at request
|
|
180
|
+
* time. Usually, model provider keys are stored here.
|
|
181
|
+
* @param baseUrl - Laminar API url. Do not include the port, use
|
|
182
|
+
* `httpPort` and `grpcPort` instead.
|
|
190
183
|
* If not specified, defaults to https://api.lmnr.ai.
|
|
191
184
|
* @param httpPort - Laminar API http port.
|
|
192
185
|
* If not specified, defaults to 443.
|
|
@@ -260,17 +253,6 @@ declare class Laminar {
|
|
|
260
253
|
* If specified as an integer, it must be epoch nanoseconds.
|
|
261
254
|
*/
|
|
262
255
|
static event(name: string, value?: AttributeValue, timestamp?: TimeInput): void;
|
|
263
|
-
/**
|
|
264
|
-
* Sends an event for evaluation to the Laminar backend.
|
|
265
|
-
*
|
|
266
|
-
* @param name - The name of the event.
|
|
267
|
-
* @param evaluator - The name of the pipeline that evaluates the event.
|
|
268
|
-
* @param data - A map from input node name to its value in the evaluator pipeline.
|
|
269
|
-
* @param env - Environment variables required to run the pipeline.
|
|
270
|
-
* @param timestamp - If specified as an integer, it must be epoch nanoseconds.
|
|
271
|
-
* If not specified, relies on the underlying OpenTelemetry implementation.
|
|
272
|
-
*/
|
|
273
|
-
static evaluateEvent(name: string, evaluator: string, data: Record<string, any>, env?: Record<string, string>, timestamp?: TimeInput): void;
|
|
274
256
|
/**
|
|
275
257
|
* Sets the session information for the current span and returns the context to use for the following spans.
|
|
276
258
|
*
|
|
@@ -293,15 +275,11 @@ declare class Laminar {
|
|
|
293
275
|
userId?: string;
|
|
294
276
|
}): _opentelemetry_api.Context;
|
|
295
277
|
static shutdown(): Promise<void>;
|
|
296
|
-
static createEvaluation(name
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
* @param evaluationId - The ID of the evaluation to update.
|
|
302
|
-
* @param status - The status to set for the evaluation.
|
|
303
|
-
*/
|
|
304
|
-
static updateEvaluationStatus(evaluationId: string, status: EvaluationStatus): Promise<UpdateEvaluationResponse>;
|
|
278
|
+
static createEvaluation<D, T, O>({ groupId, name, data }: {
|
|
279
|
+
groupId?: string;
|
|
280
|
+
name?: string;
|
|
281
|
+
data: EvaluationDatapoint<D, T, O>[];
|
|
282
|
+
}): Promise<CreateEvaluationResponse>;
|
|
305
283
|
private static getHeaders;
|
|
306
284
|
}
|
|
307
285
|
|
|
@@ -362,6 +340,10 @@ interface EvaluationConstructorProps<D, T, O> {
|
|
|
362
340
|
* `evaluator_${index}`, where index is the index of the evaluator function in the list starting from 1.
|
|
363
341
|
*/
|
|
364
342
|
evaluators: Record<string, EvaluatorFunction<O, T>>;
|
|
343
|
+
/**
|
|
344
|
+
* Optional group id of the evaluation. Defaults to "default".
|
|
345
|
+
*/
|
|
346
|
+
groupId?: string;
|
|
365
347
|
/**
|
|
366
348
|
* Name of the evaluation.
|
|
367
349
|
*/
|
|
@@ -373,30 +355,16 @@ interface EvaluationConstructorProps<D, T, O> {
|
|
|
373
355
|
}
|
|
374
356
|
declare class Evaluation<D, T, O> {
|
|
375
357
|
private isFinished;
|
|
376
|
-
private name?;
|
|
377
358
|
private progressReporter;
|
|
378
359
|
private data;
|
|
379
360
|
private executor;
|
|
380
361
|
private evaluators;
|
|
362
|
+
private groupId?;
|
|
363
|
+
private name?;
|
|
381
364
|
private batchSize;
|
|
382
|
-
|
|
383
|
-
* Create a new evaluation and prepare data.
|
|
384
|
-
* @param props.data List of data points to evaluate. `data` is the input to the executor function, `target` is the input to the evaluator function.
|
|
385
|
-
* @param props.executor The executor function. Takes the data point + any additional arguments and returns the output to evaluate.
|
|
386
|
-
* @param props.evaluators Map from evaluator name to evaluator function. Each evaluator function takes the output of the executor and the target data, and returns.
|
|
387
|
-
* @param props.name Optional name of the evaluation.
|
|
388
|
-
* @param props.config Optional override configurations for the evaluator.
|
|
389
|
-
*/
|
|
390
|
-
constructor({ data, executor, evaluators, name, config }: EvaluationConstructorProps<D, T, O>);
|
|
391
|
-
/**
|
|
392
|
-
* Runs the evaluation.
|
|
393
|
-
*
|
|
394
|
-
* Creates a new evaluation.
|
|
395
|
-
* Evaluates data points in batches of `batchSize`. The executor function is called on each data point
|
|
396
|
-
* to get the output, and then evaluate it by each evaluator function.
|
|
397
|
-
*/
|
|
365
|
+
constructor({ data, executor, evaluators, groupId, name, config }: EvaluationConstructorProps<D, T, O>);
|
|
398
366
|
run(): Promise<void>;
|
|
399
|
-
evaluateInBatches(
|
|
367
|
+
evaluateInBatches(): Promise<EvaluationDatapoint<D, T, O>[]>;
|
|
400
368
|
private evaluateBatch;
|
|
401
369
|
private getLength;
|
|
402
370
|
}
|
|
@@ -408,10 +376,11 @@ declare class Evaluation<D, T, O> {
|
|
|
408
376
|
* @param props.data List of data points to evaluate. `data` is the input to the executor function, `target` is the input to the evaluator function.
|
|
409
377
|
* @param props.executor The executor function. Takes the data point + any additional arguments and returns the output to evaluate.
|
|
410
378
|
* @param props.evaluators Map from evaluator name to evaluator function. Each evaluator function takes the output of the executor and the target data, and returns.
|
|
411
|
-
* @param props.
|
|
379
|
+
* @param props.groupId Group name which is same as the feature you are evaluating in your project or application. Defaults to "default".
|
|
380
|
+
* @param props.name Optional name of the evaluation. Used to easily identify the evaluation in the group.
|
|
412
381
|
* @param props.config Optional override configurations for the evaluator.
|
|
413
382
|
*/
|
|
414
|
-
declare function evaluate<D, T, O>({ data, executor, evaluators, name, config }: EvaluationConstructorProps<D, T, O>): Promise<void>;
|
|
383
|
+
declare function evaluate<D, T, O>({ data, executor, evaluators, groupId, name, config }: EvaluationConstructorProps<D, T, O>): Promise<void>;
|
|
415
384
|
|
|
416
385
|
interface ObserveOptions {
|
|
417
386
|
name?: string;
|
package/dist/index.d.ts
CHANGED
|
@@ -54,16 +54,9 @@ type Event = {
|
|
|
54
54
|
type CreateEvaluationResponse = {
|
|
55
55
|
id: StringUUID;
|
|
56
56
|
createdAt: Date;
|
|
57
|
+
groupId: string;
|
|
57
58
|
name: string;
|
|
58
|
-
status: EvaluationStatus;
|
|
59
59
|
projectId: StringUUID;
|
|
60
|
-
metadata: Record<string, any> | null;
|
|
61
|
-
};
|
|
62
|
-
type EvaluationStats = {
|
|
63
|
-
averageScores: Record<string, number>;
|
|
64
|
-
};
|
|
65
|
-
type UpdateEvaluationResponse = {
|
|
66
|
-
stats: EvaluationStats;
|
|
67
60
|
};
|
|
68
61
|
type EvaluationDatapoint<D, T, O> = {
|
|
69
62
|
data: Record<string, any> & D;
|
|
@@ -72,7 +65,6 @@ type EvaluationDatapoint<D, T, O> = {
|
|
|
72
65
|
scores: Record<string, number>;
|
|
73
66
|
traceId: string;
|
|
74
67
|
};
|
|
75
|
-
type EvaluationStatus = 'Started' | 'Finished' | 'Error';
|
|
76
68
|
/**
|
|
77
69
|
* Trace types to categorize traces.
|
|
78
70
|
* They are used as association properties passed to all spans in a trace.
|
|
@@ -184,9 +176,10 @@ declare class Laminar {
|
|
|
184
176
|
* @param project_api_key - Laminar project api key. You can generate one by going
|
|
185
177
|
* to the projects settings page on the Laminar dashboard.
|
|
186
178
|
* If not specified, it will try to read from the LMNR_PROJECT_API_KEY environment variable.
|
|
187
|
-
* @param env - Default environment passed to `run`
|
|
188
|
-
*
|
|
189
|
-
* @param baseUrl - Laminar API url.
|
|
179
|
+
* @param env - Default environment passed to `run` requests, unless overriden at request
|
|
180
|
+
* time. Usually, model provider keys are stored here.
|
|
181
|
+
* @param baseUrl - Laminar API url. Do not include the port, use
|
|
182
|
+
* `httpPort` and `grpcPort` instead.
|
|
190
183
|
* If not specified, defaults to https://api.lmnr.ai.
|
|
191
184
|
* @param httpPort - Laminar API http port.
|
|
192
185
|
* If not specified, defaults to 443.
|
|
@@ -260,17 +253,6 @@ declare class Laminar {
|
|
|
260
253
|
* If specified as an integer, it must be epoch nanoseconds.
|
|
261
254
|
*/
|
|
262
255
|
static event(name: string, value?: AttributeValue, timestamp?: TimeInput): void;
|
|
263
|
-
/**
|
|
264
|
-
* Sends an event for evaluation to the Laminar backend.
|
|
265
|
-
*
|
|
266
|
-
* @param name - The name of the event.
|
|
267
|
-
* @param evaluator - The name of the pipeline that evaluates the event.
|
|
268
|
-
* @param data - A map from input node name to its value in the evaluator pipeline.
|
|
269
|
-
* @param env - Environment variables required to run the pipeline.
|
|
270
|
-
* @param timestamp - If specified as an integer, it must be epoch nanoseconds.
|
|
271
|
-
* If not specified, relies on the underlying OpenTelemetry implementation.
|
|
272
|
-
*/
|
|
273
|
-
static evaluateEvent(name: string, evaluator: string, data: Record<string, any>, env?: Record<string, string>, timestamp?: TimeInput): void;
|
|
274
256
|
/**
|
|
275
257
|
* Sets the session information for the current span and returns the context to use for the following spans.
|
|
276
258
|
*
|
|
@@ -293,15 +275,11 @@ declare class Laminar {
|
|
|
293
275
|
userId?: string;
|
|
294
276
|
}): _opentelemetry_api.Context;
|
|
295
277
|
static shutdown(): Promise<void>;
|
|
296
|
-
static createEvaluation(name
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
* @param evaluationId - The ID of the evaluation to update.
|
|
302
|
-
* @param status - The status to set for the evaluation.
|
|
303
|
-
*/
|
|
304
|
-
static updateEvaluationStatus(evaluationId: string, status: EvaluationStatus): Promise<UpdateEvaluationResponse>;
|
|
278
|
+
static createEvaluation<D, T, O>({ groupId, name, data }: {
|
|
279
|
+
groupId?: string;
|
|
280
|
+
name?: string;
|
|
281
|
+
data: EvaluationDatapoint<D, T, O>[];
|
|
282
|
+
}): Promise<CreateEvaluationResponse>;
|
|
305
283
|
private static getHeaders;
|
|
306
284
|
}
|
|
307
285
|
|
|
@@ -362,6 +340,10 @@ interface EvaluationConstructorProps<D, T, O> {
|
|
|
362
340
|
* `evaluator_${index}`, where index is the index of the evaluator function in the list starting from 1.
|
|
363
341
|
*/
|
|
364
342
|
evaluators: Record<string, EvaluatorFunction<O, T>>;
|
|
343
|
+
/**
|
|
344
|
+
* Optional group id of the evaluation. Defaults to "default".
|
|
345
|
+
*/
|
|
346
|
+
groupId?: string;
|
|
365
347
|
/**
|
|
366
348
|
* Name of the evaluation.
|
|
367
349
|
*/
|
|
@@ -373,30 +355,16 @@ interface EvaluationConstructorProps<D, T, O> {
|
|
|
373
355
|
}
|
|
374
356
|
declare class Evaluation<D, T, O> {
|
|
375
357
|
private isFinished;
|
|
376
|
-
private name?;
|
|
377
358
|
private progressReporter;
|
|
378
359
|
private data;
|
|
379
360
|
private executor;
|
|
380
361
|
private evaluators;
|
|
362
|
+
private groupId?;
|
|
363
|
+
private name?;
|
|
381
364
|
private batchSize;
|
|
382
|
-
|
|
383
|
-
* Create a new evaluation and prepare data.
|
|
384
|
-
* @param props.data List of data points to evaluate. `data` is the input to the executor function, `target` is the input to the evaluator function.
|
|
385
|
-
* @param props.executor The executor function. Takes the data point + any additional arguments and returns the output to evaluate.
|
|
386
|
-
* @param props.evaluators Map from evaluator name to evaluator function. Each evaluator function takes the output of the executor and the target data, and returns.
|
|
387
|
-
* @param props.name Optional name of the evaluation.
|
|
388
|
-
* @param props.config Optional override configurations for the evaluator.
|
|
389
|
-
*/
|
|
390
|
-
constructor({ data, executor, evaluators, name, config }: EvaluationConstructorProps<D, T, O>);
|
|
391
|
-
/**
|
|
392
|
-
* Runs the evaluation.
|
|
393
|
-
*
|
|
394
|
-
* Creates a new evaluation.
|
|
395
|
-
* Evaluates data points in batches of `batchSize`. The executor function is called on each data point
|
|
396
|
-
* to get the output, and then evaluate it by each evaluator function.
|
|
397
|
-
*/
|
|
365
|
+
constructor({ data, executor, evaluators, groupId, name, config }: EvaluationConstructorProps<D, T, O>);
|
|
398
366
|
run(): Promise<void>;
|
|
399
|
-
evaluateInBatches(
|
|
367
|
+
evaluateInBatches(): Promise<EvaluationDatapoint<D, T, O>[]>;
|
|
400
368
|
private evaluateBatch;
|
|
401
369
|
private getLength;
|
|
402
370
|
}
|
|
@@ -408,10 +376,11 @@ declare class Evaluation<D, T, O> {
|
|
|
408
376
|
* @param props.data List of data points to evaluate. `data` is the input to the executor function, `target` is the input to the evaluator function.
|
|
409
377
|
* @param props.executor The executor function. Takes the data point + any additional arguments and returns the output to evaluate.
|
|
410
378
|
* @param props.evaluators Map from evaluator name to evaluator function. Each evaluator function takes the output of the executor and the target data, and returns.
|
|
411
|
-
* @param props.
|
|
379
|
+
* @param props.groupId Group name which is same as the feature you are evaluating in your project or application. Defaults to "default".
|
|
380
|
+
* @param props.name Optional name of the evaluation. Used to easily identify the evaluation in the group.
|
|
412
381
|
* @param props.config Optional override configurations for the evaluator.
|
|
413
382
|
*/
|
|
414
|
-
declare function evaluate<D, T, O>({ data, executor, evaluators, name, config }: EvaluationConstructorProps<D, T, O>): Promise<void>;
|
|
383
|
+
declare function evaluate<D, T, O>({ data, executor, evaluators, groupId, name, config }: EvaluationConstructorProps<D, T, O>): Promise<void>;
|
|
415
384
|
|
|
416
385
|
interface ObserveOptions {
|
|
417
386
|
name?: string;
|