@lmnr-ai/lmnr 0.4.10 → 0.4.11-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.mts +7 -0
- package/dist/cli.d.ts +7 -0
- package/dist/cli.js +191 -0
- package/dist/cli.js.map +1 -0
- package/dist/cli.mjs +164 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/index.d.mts +33 -8
- package/dist/index.d.ts +33 -8
- package/dist/index.js +131 -38
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +130 -37
- package/dist/index.mjs.map +1 -1
- package/package.json +10 -2
- package/src/cli.ts +86 -0
- package/src/evaluations.ts +111 -17
- package/src/index.ts +1 -1
- package/src/laminar.ts +32 -26
- package/src/types.ts +3 -0
- package/src/utils.ts +4 -0
package/dist/cli.d.mts
ADDED
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
var __create = Object.create;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
var __commonJS = (cb, mod) => function __require() {
|
|
10
|
+
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
11
|
+
};
|
|
12
|
+
var __export = (target, all) => {
|
|
13
|
+
for (var name in all)
|
|
14
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
15
|
+
};
|
|
16
|
+
var __copyProps = (to, from, except, desc) => {
|
|
17
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
18
|
+
for (let key of __getOwnPropNames(from))
|
|
19
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
20
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
21
|
+
}
|
|
22
|
+
return to;
|
|
23
|
+
};
|
|
24
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
25
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
26
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
27
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
28
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
29
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
30
|
+
mod
|
|
31
|
+
));
|
|
32
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
33
|
+
|
|
34
|
+
// package.json
|
|
35
|
+
var require_package = __commonJS({
|
|
36
|
+
"package.json"(exports2, module2) {
|
|
37
|
+
module2.exports = {
|
|
38
|
+
name: "@lmnr-ai/lmnr",
|
|
39
|
+
version: "0.4.11-alpha",
|
|
40
|
+
description: "TypeScript SDK for Laminar AI",
|
|
41
|
+
main: "dist/index.js",
|
|
42
|
+
types: "dist/index.d.ts",
|
|
43
|
+
scripts: {
|
|
44
|
+
build: "tsup src/index.ts src/cli.ts --format esm,cjs --dts",
|
|
45
|
+
test: 'echo "Error: no test specified" && exit 1'
|
|
46
|
+
},
|
|
47
|
+
bin: {
|
|
48
|
+
lmnr: "./dist/cli.js"
|
|
49
|
+
},
|
|
50
|
+
repository: {
|
|
51
|
+
type: "git",
|
|
52
|
+
url: "git+https://github.com/lmnr-ai/lmnr-ts.git"
|
|
53
|
+
},
|
|
54
|
+
keywords: [
|
|
55
|
+
"laminar",
|
|
56
|
+
"lmnr",
|
|
57
|
+
"sdk",
|
|
58
|
+
"lmnr.ai"
|
|
59
|
+
],
|
|
60
|
+
author: "founders@lmnr.ai",
|
|
61
|
+
license: "Apache-2.0",
|
|
62
|
+
bugs: {
|
|
63
|
+
url: "https://github.com/lmnr-ai/lmnr-ts/issues"
|
|
64
|
+
},
|
|
65
|
+
homepage: "https://github.com/lmnr-ai/lmnr-ts#README",
|
|
66
|
+
devDependencies: {
|
|
67
|
+
"@anthropic-ai/sdk": ">=0.18.0",
|
|
68
|
+
"@aws-sdk/client-bedrock-runtime": ">=3.422.0",
|
|
69
|
+
"@azure/openai": ">=1.0.0-beta.7",
|
|
70
|
+
"@google-cloud/aiplatform": ">=2.17.0",
|
|
71
|
+
"@google-cloud/vertexai": ">=1.2.0",
|
|
72
|
+
"@langchain/core": ">=0.0.11",
|
|
73
|
+
"@pinecone-database/pinecone": ">=2.0.1",
|
|
74
|
+
"@qdrant/js-client-rest": ">=1.5.0",
|
|
75
|
+
"@types/argparse": "^2.0.16",
|
|
76
|
+
"@types/cli-progress": "^3.11.6",
|
|
77
|
+
"@types/node": ">=14.18.63",
|
|
78
|
+
"@types/semver": ">=7.5.3",
|
|
79
|
+
"@types/uuid": ">=9.0.6",
|
|
80
|
+
bufferutil: ">=4.0.8",
|
|
81
|
+
chromadb: ">=1.5.3",
|
|
82
|
+
"cohere-ai": ">=6.2.2",
|
|
83
|
+
langchain: ">=0.0.56",
|
|
84
|
+
llamaindex: ">=0.0.30",
|
|
85
|
+
openai: ">=4.14.2",
|
|
86
|
+
runnables: "link:@langchain/core/runnables",
|
|
87
|
+
tsup: ">=7.2.0",
|
|
88
|
+
typescript: ">=5.1.6",
|
|
89
|
+
vectorstores: "link:@langchain/core/vectorstores"
|
|
90
|
+
},
|
|
91
|
+
dependencies: {
|
|
92
|
+
"@grpc/grpc-js": ">=1.9.9",
|
|
93
|
+
"@opentelemetry/api": ">=1.9.0",
|
|
94
|
+
"@opentelemetry/core": ">=1.26.0",
|
|
95
|
+
"@opentelemetry/exporter-trace-otlp-grpc": ">=0.53.0",
|
|
96
|
+
"@opentelemetry/exporter-trace-otlp-proto": ">=0.53.0",
|
|
97
|
+
"@opentelemetry/instrumentation": ">=0.53.0",
|
|
98
|
+
"@opentelemetry/sdk-node": ">=0.53.0",
|
|
99
|
+
"@opentelemetry/sdk-trace-base": "^1.26.0",
|
|
100
|
+
"@opentelemetry/sdk-trace-node": "^1.26.0",
|
|
101
|
+
"@opentelemetry/semantic-conventions": ">=1.27.0",
|
|
102
|
+
"@traceloop/ai-semantic-conventions": "^0.11.0",
|
|
103
|
+
"@traceloop/instrumentation-anthropic": "^0.11.1",
|
|
104
|
+
"@traceloop/instrumentation-azure": "^0.11.1",
|
|
105
|
+
"@traceloop/instrumentation-bedrock": "^0.11.1",
|
|
106
|
+
"@traceloop/instrumentation-chromadb": "^0.11.1",
|
|
107
|
+
"@traceloop/instrumentation-cohere": "^0.11.1",
|
|
108
|
+
"@traceloop/instrumentation-langchain": "^0.11.1",
|
|
109
|
+
"@traceloop/instrumentation-llamaindex": "^0.11.1",
|
|
110
|
+
"@traceloop/instrumentation-openai": "^0.11.1",
|
|
111
|
+
"@traceloop/instrumentation-pinecone": "^0.11.1",
|
|
112
|
+
"@traceloop/instrumentation-qdrant": "^0.11.1",
|
|
113
|
+
"@traceloop/instrumentation-vertexai": "^0.11.1",
|
|
114
|
+
argparse: "^2.0.1",
|
|
115
|
+
"cli-progress": "^3.12.0",
|
|
116
|
+
esbuild: "^0.23.1",
|
|
117
|
+
"posthog-node": ">=3.1.3",
|
|
118
|
+
uuid: ">=9.0.0"
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
// src/cli.ts
|
|
125
|
+
var cli_exports = {};
|
|
126
|
+
__export(cli_exports, {
|
|
127
|
+
loadModule: () => loadModule
|
|
128
|
+
});
|
|
129
|
+
module.exports = __toCommonJS(cli_exports);
|
|
130
|
+
var import_argparse = require("argparse");
|
|
131
|
+
var esbuild = __toESM(require("esbuild"));
|
|
132
|
+
var pjson = require_package();
|
|
133
|
+
function loadModule({
|
|
134
|
+
filename,
|
|
135
|
+
moduleText
|
|
136
|
+
}) {
|
|
137
|
+
globalThis._evaluation = void 0;
|
|
138
|
+
globalThis._set_global_evaluation = true;
|
|
139
|
+
new Function("require", moduleText)(require);
|
|
140
|
+
return globalThis._evaluation;
|
|
141
|
+
}
|
|
142
|
+
async function cli() {
|
|
143
|
+
const [, , ...args] = process.argv;
|
|
144
|
+
const parser = new import_argparse.ArgumentParser({
|
|
145
|
+
prog: "lmnr",
|
|
146
|
+
description: "CLI for Laminar"
|
|
147
|
+
});
|
|
148
|
+
parser.add_argument("-v", "--version", { action: "version", version: pjson.version });
|
|
149
|
+
const subparsers = parser.add_subparsers({
|
|
150
|
+
title: "subcommands",
|
|
151
|
+
dest: "subcommand"
|
|
152
|
+
});
|
|
153
|
+
const parser_eval = subparsers.add_parser("eval", {
|
|
154
|
+
description: "Run an evaluation"
|
|
155
|
+
});
|
|
156
|
+
parser_eval.add_argument("file", {
|
|
157
|
+
help: "A file containing the evaluation to run"
|
|
158
|
+
});
|
|
159
|
+
parser_eval.set_defaults({
|
|
160
|
+
func: async (args2) => {
|
|
161
|
+
const buildOptions = {
|
|
162
|
+
entryPoints: [args2.file],
|
|
163
|
+
outfile: "tmp_out.js",
|
|
164
|
+
write: false,
|
|
165
|
+
// will be loaded in memory as a temp file
|
|
166
|
+
platform: "node",
|
|
167
|
+
bundle: true,
|
|
168
|
+
external: ["*"]
|
|
169
|
+
};
|
|
170
|
+
const result = await esbuild.build(buildOptions);
|
|
171
|
+
if (!result.outputFiles) {
|
|
172
|
+
console.error("Error when building: No output files found");
|
|
173
|
+
process.exit(1);
|
|
174
|
+
}
|
|
175
|
+
const outputFileText = result.outputFiles[0].text;
|
|
176
|
+
const evaluation = loadModule({
|
|
177
|
+
filename: args2.file,
|
|
178
|
+
moduleText: outputFileText
|
|
179
|
+
});
|
|
180
|
+
await evaluation.run();
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
const parsed = parser.parse_args(args);
|
|
184
|
+
await parsed.func(parsed);
|
|
185
|
+
}
|
|
186
|
+
cli();
|
|
187
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
188
|
+
0 && (module.exports = {
|
|
189
|
+
loadModule
|
|
190
|
+
});
|
|
191
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.11-alpha\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \">=0.18.0\",\n \"@aws-sdk/client-bedrock-runtime\": \">=3.422.0\",\n \"@azure/openai\": \">=1.0.0-beta.7\",\n \"@google-cloud/aiplatform\": \">=2.17.0\",\n \"@google-cloud/vertexai\": \">=1.2.0\",\n \"@langchain/core\": \">=0.0.11\",\n \"@pinecone-database/pinecone\": \">=2.0.1\",\n \"@qdrant/js-client-rest\": \">=1.5.0\",\n \"@types/argparse\": \"^2.0.16\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \">=14.18.63\",\n \"@types/semver\": \">=7.5.3\",\n \"@types/uuid\": \">=9.0.6\",\n \"bufferutil\": \">=4.0.8\",\n \"chromadb\": \">=1.5.3\",\n \"cohere-ai\": \">=6.2.2\",\n \"langchain\": \">=0.0.56\",\n \"llamaindex\": \">=0.0.30\",\n \"openai\": \">=4.14.2\",\n \"runnables\": \"link:@langchain/core/runnables\",\n \"tsup\": \">=7.2.0\",\n \"typescript\": \">=5.1.6\",\n \"vectorstores\": \"link:@langchain/core/vectorstores\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \">=1.9.9\",\n \"@opentelemetry/api\": \">=1.9.0\",\n \"@opentelemetry/core\": \">=1.26.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \">=0.53.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \">=0.53.0\",\n \"@opentelemetry/instrumentation\": \">=0.53.0\",\n \"@opentelemetry/sdk-node\": \">=0.53.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.26.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.26.0\",\n \"@opentelemetry/semantic-conventions\": \">=1.27.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.1\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.1\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n \"@traceloop/instrumentation-openai\": \"^0.11.1\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.1\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.23.1\",\n \"posthog-node\": \">=3.1.3\",\n \"uuid\": \">=9.0.0\"\n }\n}","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parser_eval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n });\n\n parser_eval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run\",\n });\n\n parser_eval.set_defaults({\n func: async (args: any) => {\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [args.file],\n outfile: \"tmp_out.js\",\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n process.exit(1);\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n await evaluation.run();\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,iBAAAA,UAAAC,SAAA;AAAA,IAAAA,QAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,WAAa;AAAA,QACb,MAAQ;AAAA,QACR,YAAc;AAAA,QACd,cAAgB;AAAA,MAClB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,gBAAgB;AAAA,QAChB,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;ACnFA;AAAA;AAAA;AAAA;AAAA;AAEA,sBAA+B;AAC/B,cAAyB;AAEzB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAED,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,OAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACjB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAC9B,MAAM;AAAA,IACN,aAAa;AAAA,EACjB,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACrC,OAAO;AAAA,IACP,MAAM;AAAA,EACV,CAAC;AAED,QAAM,cAAc,WAAW,WAAW,QAAQ;AAAA,IAC9C,aAAa;AAAA,EACjB,CAAC;AAED,cAAY,aAAa,QAAQ;AAAA,IAC7B,MAAM;AAAA,EACV,CAAC;AAED,cAAY,aAAa;AAAA,IACrB,MAAM,OAAOC,UAAc;AAEvB,YAAM,eAAe;AAAA,QACnB,aAAa,CAACA,MAAK,IAAI;AAAA,QACvB,SAAS;AAAA,QACT,OAAO;AAAA;AAAA,QACP,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU,CAAC,GAAG;AAAA,MAChB;AAEA,YAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,UAAI,CAAC,OAAO,aAAa;AACrB,gBAAQ,MAAM,4CAA4C;AAC1D,gBAAQ,KAAK,CAAC;AAAA,MAClB;AAEA,YAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,YAAM,aAAa,WAAW;AAAA,QAC5B,UAAUA,MAAK;AAAA,QACf,YAAY;AAAA,MACd,CAAC;AAGD,YAAM,WAAW,IAAI;AAAA,IACzB;AAAA,EACJ,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC5B;AAEA,IAAI;","names":["exports","module","args"]}
|
package/dist/cli.mjs
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
3
|
+
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
4
|
+
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
5
|
+
}) : x)(function(x) {
|
|
6
|
+
if (typeof require !== "undefined") return require.apply(this, arguments);
|
|
7
|
+
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
8
|
+
});
|
|
9
|
+
var __commonJS = (cb, mod) => function __require2() {
|
|
10
|
+
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
// package.json
|
|
14
|
+
var require_package = __commonJS({
|
|
15
|
+
"package.json"(exports, module) {
|
|
16
|
+
module.exports = {
|
|
17
|
+
name: "@lmnr-ai/lmnr",
|
|
18
|
+
version: "0.4.11-alpha",
|
|
19
|
+
description: "TypeScript SDK for Laminar AI",
|
|
20
|
+
main: "dist/index.js",
|
|
21
|
+
types: "dist/index.d.ts",
|
|
22
|
+
scripts: {
|
|
23
|
+
build: "tsup src/index.ts src/cli.ts --format esm,cjs --dts",
|
|
24
|
+
test: 'echo "Error: no test specified" && exit 1'
|
|
25
|
+
},
|
|
26
|
+
bin: {
|
|
27
|
+
lmnr: "./dist/cli.js"
|
|
28
|
+
},
|
|
29
|
+
repository: {
|
|
30
|
+
type: "git",
|
|
31
|
+
url: "git+https://github.com/lmnr-ai/lmnr-ts.git"
|
|
32
|
+
},
|
|
33
|
+
keywords: [
|
|
34
|
+
"laminar",
|
|
35
|
+
"lmnr",
|
|
36
|
+
"sdk",
|
|
37
|
+
"lmnr.ai"
|
|
38
|
+
],
|
|
39
|
+
author: "founders@lmnr.ai",
|
|
40
|
+
license: "Apache-2.0",
|
|
41
|
+
bugs: {
|
|
42
|
+
url: "https://github.com/lmnr-ai/lmnr-ts/issues"
|
|
43
|
+
},
|
|
44
|
+
homepage: "https://github.com/lmnr-ai/lmnr-ts#README",
|
|
45
|
+
devDependencies: {
|
|
46
|
+
"@anthropic-ai/sdk": ">=0.18.0",
|
|
47
|
+
"@aws-sdk/client-bedrock-runtime": ">=3.422.0",
|
|
48
|
+
"@azure/openai": ">=1.0.0-beta.7",
|
|
49
|
+
"@google-cloud/aiplatform": ">=2.17.0",
|
|
50
|
+
"@google-cloud/vertexai": ">=1.2.0",
|
|
51
|
+
"@langchain/core": ">=0.0.11",
|
|
52
|
+
"@pinecone-database/pinecone": ">=2.0.1",
|
|
53
|
+
"@qdrant/js-client-rest": ">=1.5.0",
|
|
54
|
+
"@types/argparse": "^2.0.16",
|
|
55
|
+
"@types/cli-progress": "^3.11.6",
|
|
56
|
+
"@types/node": ">=14.18.63",
|
|
57
|
+
"@types/semver": ">=7.5.3",
|
|
58
|
+
"@types/uuid": ">=9.0.6",
|
|
59
|
+
bufferutil: ">=4.0.8",
|
|
60
|
+
chromadb: ">=1.5.3",
|
|
61
|
+
"cohere-ai": ">=6.2.2",
|
|
62
|
+
langchain: ">=0.0.56",
|
|
63
|
+
llamaindex: ">=0.0.30",
|
|
64
|
+
openai: ">=4.14.2",
|
|
65
|
+
runnables: "link:@langchain/core/runnables",
|
|
66
|
+
tsup: ">=7.2.0",
|
|
67
|
+
typescript: ">=5.1.6",
|
|
68
|
+
vectorstores: "link:@langchain/core/vectorstores"
|
|
69
|
+
},
|
|
70
|
+
dependencies: {
|
|
71
|
+
"@grpc/grpc-js": ">=1.9.9",
|
|
72
|
+
"@opentelemetry/api": ">=1.9.0",
|
|
73
|
+
"@opentelemetry/core": ">=1.26.0",
|
|
74
|
+
"@opentelemetry/exporter-trace-otlp-grpc": ">=0.53.0",
|
|
75
|
+
"@opentelemetry/exporter-trace-otlp-proto": ">=0.53.0",
|
|
76
|
+
"@opentelemetry/instrumentation": ">=0.53.0",
|
|
77
|
+
"@opentelemetry/sdk-node": ">=0.53.0",
|
|
78
|
+
"@opentelemetry/sdk-trace-base": "^1.26.0",
|
|
79
|
+
"@opentelemetry/sdk-trace-node": "^1.26.0",
|
|
80
|
+
"@opentelemetry/semantic-conventions": ">=1.27.0",
|
|
81
|
+
"@traceloop/ai-semantic-conventions": "^0.11.0",
|
|
82
|
+
"@traceloop/instrumentation-anthropic": "^0.11.1",
|
|
83
|
+
"@traceloop/instrumentation-azure": "^0.11.1",
|
|
84
|
+
"@traceloop/instrumentation-bedrock": "^0.11.1",
|
|
85
|
+
"@traceloop/instrumentation-chromadb": "^0.11.1",
|
|
86
|
+
"@traceloop/instrumentation-cohere": "^0.11.1",
|
|
87
|
+
"@traceloop/instrumentation-langchain": "^0.11.1",
|
|
88
|
+
"@traceloop/instrumentation-llamaindex": "^0.11.1",
|
|
89
|
+
"@traceloop/instrumentation-openai": "^0.11.1",
|
|
90
|
+
"@traceloop/instrumentation-pinecone": "^0.11.1",
|
|
91
|
+
"@traceloop/instrumentation-qdrant": "^0.11.1",
|
|
92
|
+
"@traceloop/instrumentation-vertexai": "^0.11.1",
|
|
93
|
+
argparse: "^2.0.1",
|
|
94
|
+
"cli-progress": "^3.12.0",
|
|
95
|
+
esbuild: "^0.23.1",
|
|
96
|
+
"posthog-node": ">=3.1.3",
|
|
97
|
+
uuid: ">=9.0.0"
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
// src/cli.ts
|
|
104
|
+
import { ArgumentParser } from "argparse";
|
|
105
|
+
import * as esbuild from "esbuild";
|
|
106
|
+
var pjson = require_package();
|
|
107
|
+
function loadModule({
|
|
108
|
+
filename,
|
|
109
|
+
moduleText
|
|
110
|
+
}) {
|
|
111
|
+
globalThis._evaluation = void 0;
|
|
112
|
+
globalThis._set_global_evaluation = true;
|
|
113
|
+
new Function("require", moduleText)(__require);
|
|
114
|
+
return globalThis._evaluation;
|
|
115
|
+
}
|
|
116
|
+
async function cli() {
|
|
117
|
+
const [, , ...args] = process.argv;
|
|
118
|
+
const parser = new ArgumentParser({
|
|
119
|
+
prog: "lmnr",
|
|
120
|
+
description: "CLI for Laminar"
|
|
121
|
+
});
|
|
122
|
+
parser.add_argument("-v", "--version", { action: "version", version: pjson.version });
|
|
123
|
+
const subparsers = parser.add_subparsers({
|
|
124
|
+
title: "subcommands",
|
|
125
|
+
dest: "subcommand"
|
|
126
|
+
});
|
|
127
|
+
const parser_eval = subparsers.add_parser("eval", {
|
|
128
|
+
description: "Run an evaluation"
|
|
129
|
+
});
|
|
130
|
+
parser_eval.add_argument("file", {
|
|
131
|
+
help: "A file containing the evaluation to run"
|
|
132
|
+
});
|
|
133
|
+
parser_eval.set_defaults({
|
|
134
|
+
func: async (args2) => {
|
|
135
|
+
const buildOptions = {
|
|
136
|
+
entryPoints: [args2.file],
|
|
137
|
+
outfile: "tmp_out.js",
|
|
138
|
+
write: false,
|
|
139
|
+
// will be loaded in memory as a temp file
|
|
140
|
+
platform: "node",
|
|
141
|
+
bundle: true,
|
|
142
|
+
external: ["*"]
|
|
143
|
+
};
|
|
144
|
+
const result = await esbuild.build(buildOptions);
|
|
145
|
+
if (!result.outputFiles) {
|
|
146
|
+
console.error("Error when building: No output files found");
|
|
147
|
+
process.exit(1);
|
|
148
|
+
}
|
|
149
|
+
const outputFileText = result.outputFiles[0].text;
|
|
150
|
+
const evaluation = loadModule({
|
|
151
|
+
filename: args2.file,
|
|
152
|
+
moduleText: outputFileText
|
|
153
|
+
});
|
|
154
|
+
await evaluation.run();
|
|
155
|
+
}
|
|
156
|
+
});
|
|
157
|
+
const parsed = parser.parse_args(args);
|
|
158
|
+
await parsed.func(parsed);
|
|
159
|
+
}
|
|
160
|
+
cli();
|
|
161
|
+
export {
|
|
162
|
+
loadModule
|
|
163
|
+
};
|
|
164
|
+
//# sourceMappingURL=cli.mjs.map
|
package/dist/cli.mjs.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../package.json","../src/cli.ts"],"sourcesContent":["{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.4.11-alpha\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup src/index.ts src/cli.ts --format esm,cjs --dts\",\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \">=0.18.0\",\n \"@aws-sdk/client-bedrock-runtime\": \">=3.422.0\",\n \"@azure/openai\": \">=1.0.0-beta.7\",\n \"@google-cloud/aiplatform\": \">=2.17.0\",\n \"@google-cloud/vertexai\": \">=1.2.0\",\n \"@langchain/core\": \">=0.0.11\",\n \"@pinecone-database/pinecone\": \">=2.0.1\",\n \"@qdrant/js-client-rest\": \">=1.5.0\",\n \"@types/argparse\": \"^2.0.16\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \">=14.18.63\",\n \"@types/semver\": \">=7.5.3\",\n \"@types/uuid\": \">=9.0.6\",\n \"bufferutil\": \">=4.0.8\",\n \"chromadb\": \">=1.5.3\",\n \"cohere-ai\": \">=6.2.2\",\n \"langchain\": \">=0.0.56\",\n \"llamaindex\": \">=0.0.30\",\n \"openai\": \">=4.14.2\",\n \"runnables\": \"link:@langchain/core/runnables\",\n \"tsup\": \">=7.2.0\",\n \"typescript\": \">=5.1.6\",\n \"vectorstores\": \"link:@langchain/core/vectorstores\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \">=1.9.9\",\n \"@opentelemetry/api\": \">=1.9.0\",\n \"@opentelemetry/core\": \">=1.26.0\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \">=0.53.0\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \">=0.53.0\",\n \"@opentelemetry/instrumentation\": \">=0.53.0\",\n \"@opentelemetry/sdk-node\": \">=0.53.0\",\n \"@opentelemetry/sdk-trace-base\": \"^1.26.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.26.0\",\n \"@opentelemetry/semantic-conventions\": \">=1.27.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.11.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.11.1\",\n \"@traceloop/instrumentation-azure\": \"^0.11.1\",\n \"@traceloop/instrumentation-bedrock\": \"^0.11.1\",\n \"@traceloop/instrumentation-chromadb\": \"^0.11.1\",\n \"@traceloop/instrumentation-cohere\": \"^0.11.1\",\n \"@traceloop/instrumentation-langchain\": \"^0.11.1\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.11.1\",\n \"@traceloop/instrumentation-openai\": \"^0.11.1\",\n \"@traceloop/instrumentation-pinecone\": \"^0.11.1\",\n \"@traceloop/instrumentation-qdrant\": \"^0.11.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.11.1\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.23.1\",\n \"posthog-node\": \">=3.1.3\",\n \"uuid\": \">=9.0.0\"\n }\n}","#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\n\nconst pjson = require('../package.json');\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}) {\n // @ts-ignore\n globalThis._evaluation = undefined; // @ts-ignore\n globalThis._set_global_evaluation = true; // @ts-ignore\n\n // it needs \"require\" to be passed in\n new Function(\"require\", moduleText)(require);\n\n // Return the modified _evals global variable\n // @ts-ignore\n return globalThis._evaluation;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version: pjson.version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parser_eval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n });\n\n parser_eval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run\",\n });\n\n parser_eval.set_defaults({\n func: async (args: any) => {\n // TODO: Add various optimizations, e.g. minify, pure, tree shaking, etc.\n const buildOptions = {\n entryPoints: [args.file],\n outfile: \"tmp_out.js\",\n write: false, // will be loaded in memory as a temp file\n platform: \"node\" as esbuild.Platform,\n bundle: true,\n external: [\"*\"],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n console.error(\"Error when building: No output files found\");\n process.exit(1);\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n // @ts-ignore\n await evaluation.run();\n }\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli();\n"],"mappings":";;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA,MACE,MAAQ;AAAA,MACR,SAAW;AAAA,MACX,aAAe;AAAA,MACf,MAAQ;AAAA,MACR,OAAS;AAAA,MACT,SAAW;AAAA,QACT,OAAS;AAAA,QACT,MAAQ;AAAA,MACV;AAAA,MACA,KAAO;AAAA,QACL,MAAQ;AAAA,MACV;AAAA,MACA,YAAc;AAAA,QACZ,MAAQ;AAAA,QACR,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,QACV;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA,QAAU;AAAA,MACV,SAAW;AAAA,MACX,MAAQ;AAAA,QACN,KAAO;AAAA,MACT;AAAA,MACA,UAAY;AAAA,MACZ,iBAAmB;AAAA,QACjB,qBAAqB;AAAA,QACrB,mCAAmC;AAAA,QACnC,iBAAiB;AAAA,QACjB,4BAA4B;AAAA,QAC5B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,+BAA+B;AAAA,QAC/B,0BAA0B;AAAA,QAC1B,mBAAmB;AAAA,QACnB,uBAAuB;AAAA,QACvB,eAAe;AAAA,QACf,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,YAAc;AAAA,QACd,UAAY;AAAA,QACZ,aAAa;AAAA,QACb,WAAa;AAAA,QACb,YAAc;AAAA,QACd,QAAU;AAAA,QACV,WAAa;AAAA,QACb,MAAQ;AAAA,QACR,YAAc;AAAA,QACd,cAAgB;AAAA,MAClB;AAAA,MACA,cAAgB;AAAA,QACd,iBAAiB;AAAA,QACjB,sBAAsB;AAAA,QACtB,uBAAuB;AAAA,QACvB,2CAA2C;AAAA,QAC3C,4CAA4C;AAAA,QAC5C,kCAAkC;AAAA,QAClC,2BAA2B;AAAA,QAC3B,iCAAiC;AAAA,QACjC,iCAAiC;AAAA,QACjC,uCAAuC;AAAA,QACvC,sCAAsC;AAAA,QACtC,wCAAwC;AAAA,QACxC,oCAAoC;AAAA,QACpC,sCAAsC;AAAA,QACtC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,wCAAwC;AAAA,QACxC,yCAAyC;AAAA,QACzC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,qCAAqC;AAAA,QACrC,uCAAuC;AAAA,QACvC,UAAY;AAAA,QACZ,gBAAgB;AAAA,QAChB,SAAW;AAAA,QACX,gBAAgB;AAAA,QAChB,MAAQ;AAAA,MACV;AAAA,IACF;AAAA;AAAA;;;ACjFA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AAEzB,IAAM,QAAQ;AAEP,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGG;AAED,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAGpC,MAAI,SAAS,WAAW,UAAU,EAAE,SAAO;AAI3C,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACjB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAC9B,MAAM;AAAA,IACN,aAAa;AAAA,EACjB,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,SAAS,MAAM,QAAQ,CAAC;AAEpF,QAAM,aAAa,OAAO,eAAe;AAAA,IACrC,OAAO;AAAA,IACP,MAAM;AAAA,EACV,CAAC;AAED,QAAM,cAAc,WAAW,WAAW,QAAQ;AAAA,IAC9C,aAAa;AAAA,EACjB,CAAC;AAED,cAAY,aAAa,QAAQ;AAAA,IAC7B,MAAM;AAAA,EACV,CAAC;AAED,cAAY,aAAa;AAAA,IACrB,MAAM,OAAOA,UAAc;AAEvB,YAAM,eAAe;AAAA,QACnB,aAAa,CAACA,MAAK,IAAI;AAAA,QACvB,SAAS;AAAA,QACT,OAAO;AAAA;AAAA,QACP,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,UAAU,CAAC,GAAG;AAAA,MAChB;AAEA,YAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,UAAI,CAAC,OAAO,aAAa;AACrB,gBAAQ,MAAM,4CAA4C;AAC1D,gBAAQ,KAAK,CAAC;AAAA,MAClB;AAEA,YAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,YAAM,aAAa,WAAW;AAAA,QAC5B,UAAUA,MAAK;AAAA,QACf,YAAY;AAAA,MACd,CAAC;AAGD,YAAM,WAAW,IAAI;AAAA,IACzB;AAAA,EACJ,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC5B;AAEA,IAAI;","names":["args"]}
|
package/dist/index.d.mts
CHANGED
|
@@ -51,6 +51,16 @@ type Event = {
|
|
|
51
51
|
spanId: StringUUID;
|
|
52
52
|
value: number | string | null;
|
|
53
53
|
};
|
|
54
|
+
type CreateEvaluationResponse = {
|
|
55
|
+
id: StringUUID;
|
|
56
|
+
createdAt: Date;
|
|
57
|
+
name: string;
|
|
58
|
+
status: EvaluationStatus;
|
|
59
|
+
projectId: StringUUID;
|
|
60
|
+
metadata: Record<string, any> | null;
|
|
61
|
+
averageScores: Record<string, number> | null;
|
|
62
|
+
};
|
|
63
|
+
type UpdateEvaluationResponse = CreateEvaluationResponse;
|
|
54
64
|
type EvaluationDatapoint<D, T, O> = {
|
|
55
65
|
data: Record<string, any> & D;
|
|
56
66
|
target: Record<string, any> & T;
|
|
@@ -164,9 +174,8 @@ declare class Laminar {
|
|
|
164
174
|
* If not specified, it will try to read from the LMNR_PROJECT_API_KEY environment variable.
|
|
165
175
|
* @param env - Default environment passed to `run` and `evaluateEvent` requests,
|
|
166
176
|
* unless overriden at request time. Usually, model provider keys are stored here.
|
|
167
|
-
* @param baseUrl -
|
|
168
|
-
* If not specified, defaults to https://api.lmnr.ai
|
|
169
|
-
* default setting must be http://localhost:8001.
|
|
177
|
+
* @param baseUrl - Laminar API url.
|
|
178
|
+
* If not specified, defaults to https://api.lmnr.ai.
|
|
170
179
|
* @param instrumentModules - List of modules to instrument.
|
|
171
180
|
* If not specified, all auto-instrumentable modules will be instrumented, which include
|
|
172
181
|
* LLM calls (OpenAI, Anthropic, etc), Langchain, VectorDB calls (Pinecone, Qdrant, etc).
|
|
@@ -268,12 +277,22 @@ declare class Laminar {
|
|
|
268
277
|
userId?: string;
|
|
269
278
|
}): _opentelemetry_api.Context;
|
|
270
279
|
static shutdown(): Promise<void>;
|
|
271
|
-
static createEvaluation(name: string): Promise<
|
|
272
|
-
static postEvaluationResults<D, T, O>(
|
|
273
|
-
|
|
280
|
+
static createEvaluation(name: string): Promise<CreateEvaluationResponse>;
|
|
281
|
+
static postEvaluationResults<D, T, O>(evaluationId: string, data: EvaluationDatapoint<D, T, O>[]): Promise<void>;
|
|
282
|
+
/**
|
|
283
|
+
* Updates the status of an evaluation. Returns the updated evaluation object.
|
|
284
|
+
*
|
|
285
|
+
* @param evaluationId - The ID of the evaluation to update.
|
|
286
|
+
* @param status - The status to set for the evaluation.
|
|
287
|
+
*/
|
|
288
|
+
static updateEvaluationStatus(evaluationId: string, status: EvaluationStatus): Promise<UpdateEvaluationResponse>;
|
|
274
289
|
private static getHeaders;
|
|
275
290
|
}
|
|
276
291
|
|
|
292
|
+
declare global {
|
|
293
|
+
var _evaluation: Evaluation<any, any, any> | undefined;
|
|
294
|
+
var _set_global_evaluation: boolean;
|
|
295
|
+
}
|
|
277
296
|
/**
|
|
278
297
|
* Configuration for the Evaluator
|
|
279
298
|
*/
|
|
@@ -330,7 +349,9 @@ interface EvaluatorConstructorProps<D, T, O> {
|
|
|
330
349
|
config?: EvaluatorConfig;
|
|
331
350
|
}
|
|
332
351
|
declare class Evaluation<D, T, O> {
|
|
352
|
+
private isFinished;
|
|
333
353
|
private name;
|
|
354
|
+
private progressReporter;
|
|
334
355
|
private data;
|
|
335
356
|
private executor;
|
|
336
357
|
private evaluators;
|
|
@@ -342,18 +363,22 @@ declare class Evaluation<D, T, O> {
|
|
|
342
363
|
* @param props.data List of data points to evaluate. `data` is the input to the executor function, `target` is the input to the evaluator function.
|
|
343
364
|
* @param props.executor The executor function. Takes the data point + any additional arguments and returns the output to evaluate.
|
|
344
365
|
* @param props.evaluators List of evaluator functions. Each evaluator function takes the output of the executor and the target data, and returns.
|
|
366
|
+
* @param props.config Optional override configurations for the evaluator.
|
|
345
367
|
*/
|
|
346
368
|
constructor(name: string, { data, executor, evaluators, config }: EvaluatorConstructorProps<D, T, O>);
|
|
347
369
|
/**
|
|
348
370
|
* Runs the evaluation.
|
|
349
371
|
*
|
|
350
|
-
* Creates a new evaluation
|
|
372
|
+
* Creates a new evaluation.
|
|
351
373
|
* Evaluates data points in batches of `batchSize`. The executor function is called on each data point
|
|
352
374
|
* to get the output, and then evaluate it by each evaluator function.
|
|
353
375
|
*/
|
|
354
376
|
run(): Promise<void>;
|
|
377
|
+
evaluateInBatches(evaluation: CreateEvaluationResponse): Promise<void>;
|
|
355
378
|
private evaluateBatch;
|
|
379
|
+
private getLength;
|
|
356
380
|
}
|
|
381
|
+
declare function evaluate<D, T, O>(name: string, { data, executor, evaluators, config }: EvaluatorConstructorProps<D, T, O>): void;
|
|
357
382
|
|
|
358
383
|
interface ObserveOptions {
|
|
359
384
|
name?: string;
|
|
@@ -380,4 +405,4 @@ interface ObserveOptions {
|
|
|
380
405
|
*/
|
|
381
406
|
declare function observe<A extends unknown[], F extends (...args: A) => ReturnType<F>>({ name, sessionId, userId, }: ObserveOptions, fn: F, ...args: A): Promise<ReturnType<F>>;
|
|
382
407
|
|
|
383
|
-
export { type ChatMessage, type Datapoint, Dataset, type EvaluateEvent,
|
|
408
|
+
export { type ChatMessage, type Datapoint, Dataset, type EvaluateEvent, type Event, Laminar, type NodeInput, type PipelineRunRequest, type PipelineRunResponse, evaluate, observe };
|
package/dist/index.d.ts
CHANGED
|
@@ -51,6 +51,16 @@ type Event = {
|
|
|
51
51
|
spanId: StringUUID;
|
|
52
52
|
value: number | string | null;
|
|
53
53
|
};
|
|
54
|
+
type CreateEvaluationResponse = {
|
|
55
|
+
id: StringUUID;
|
|
56
|
+
createdAt: Date;
|
|
57
|
+
name: string;
|
|
58
|
+
status: EvaluationStatus;
|
|
59
|
+
projectId: StringUUID;
|
|
60
|
+
metadata: Record<string, any> | null;
|
|
61
|
+
averageScores: Record<string, number> | null;
|
|
62
|
+
};
|
|
63
|
+
type UpdateEvaluationResponse = CreateEvaluationResponse;
|
|
54
64
|
type EvaluationDatapoint<D, T, O> = {
|
|
55
65
|
data: Record<string, any> & D;
|
|
56
66
|
target: Record<string, any> & T;
|
|
@@ -164,9 +174,8 @@ declare class Laminar {
|
|
|
164
174
|
* If not specified, it will try to read from the LMNR_PROJECT_API_KEY environment variable.
|
|
165
175
|
* @param env - Default environment passed to `run` and `evaluateEvent` requests,
|
|
166
176
|
* unless overriden at request time. Usually, model provider keys are stored here.
|
|
167
|
-
* @param baseUrl -
|
|
168
|
-
* If not specified, defaults to https://api.lmnr.ai
|
|
169
|
-
* default setting must be http://localhost:8001.
|
|
177
|
+
* @param baseUrl - Laminar API url.
|
|
178
|
+
* If not specified, defaults to https://api.lmnr.ai.
|
|
170
179
|
* @param instrumentModules - List of modules to instrument.
|
|
171
180
|
* If not specified, all auto-instrumentable modules will be instrumented, which include
|
|
172
181
|
* LLM calls (OpenAI, Anthropic, etc), Langchain, VectorDB calls (Pinecone, Qdrant, etc).
|
|
@@ -268,12 +277,22 @@ declare class Laminar {
|
|
|
268
277
|
userId?: string;
|
|
269
278
|
}): _opentelemetry_api.Context;
|
|
270
279
|
static shutdown(): Promise<void>;
|
|
271
|
-
static createEvaluation(name: string): Promise<
|
|
272
|
-
static postEvaluationResults<D, T, O>(
|
|
273
|
-
|
|
280
|
+
static createEvaluation(name: string): Promise<CreateEvaluationResponse>;
|
|
281
|
+
static postEvaluationResults<D, T, O>(evaluationId: string, data: EvaluationDatapoint<D, T, O>[]): Promise<void>;
|
|
282
|
+
/**
|
|
283
|
+
* Updates the status of an evaluation. Returns the updated evaluation object.
|
|
284
|
+
*
|
|
285
|
+
* @param evaluationId - The ID of the evaluation to update.
|
|
286
|
+
* @param status - The status to set for the evaluation.
|
|
287
|
+
*/
|
|
288
|
+
static updateEvaluationStatus(evaluationId: string, status: EvaluationStatus): Promise<UpdateEvaluationResponse>;
|
|
274
289
|
private static getHeaders;
|
|
275
290
|
}
|
|
276
291
|
|
|
292
|
+
declare global {
|
|
293
|
+
var _evaluation: Evaluation<any, any, any> | undefined;
|
|
294
|
+
var _set_global_evaluation: boolean;
|
|
295
|
+
}
|
|
277
296
|
/**
|
|
278
297
|
* Configuration for the Evaluator
|
|
279
298
|
*/
|
|
@@ -330,7 +349,9 @@ interface EvaluatorConstructorProps<D, T, O> {
|
|
|
330
349
|
config?: EvaluatorConfig;
|
|
331
350
|
}
|
|
332
351
|
declare class Evaluation<D, T, O> {
|
|
352
|
+
private isFinished;
|
|
333
353
|
private name;
|
|
354
|
+
private progressReporter;
|
|
334
355
|
private data;
|
|
335
356
|
private executor;
|
|
336
357
|
private evaluators;
|
|
@@ -342,18 +363,22 @@ declare class Evaluation<D, T, O> {
|
|
|
342
363
|
* @param props.data List of data points to evaluate. `data` is the input to the executor function, `target` is the input to the evaluator function.
|
|
343
364
|
* @param props.executor The executor function. Takes the data point + any additional arguments and returns the output to evaluate.
|
|
344
365
|
* @param props.evaluators List of evaluator functions. Each evaluator function takes the output of the executor and the target data, and returns.
|
|
366
|
+
* @param props.config Optional override configurations for the evaluator.
|
|
345
367
|
*/
|
|
346
368
|
constructor(name: string, { data, executor, evaluators, config }: EvaluatorConstructorProps<D, T, O>);
|
|
347
369
|
/**
|
|
348
370
|
* Runs the evaluation.
|
|
349
371
|
*
|
|
350
|
-
* Creates a new evaluation
|
|
372
|
+
* Creates a new evaluation.
|
|
351
373
|
* Evaluates data points in batches of `batchSize`. The executor function is called on each data point
|
|
352
374
|
* to get the output, and then evaluate it by each evaluator function.
|
|
353
375
|
*/
|
|
354
376
|
run(): Promise<void>;
|
|
377
|
+
evaluateInBatches(evaluation: CreateEvaluationResponse): Promise<void>;
|
|
355
378
|
private evaluateBatch;
|
|
379
|
+
private getLength;
|
|
356
380
|
}
|
|
381
|
+
declare function evaluate<D, T, O>(name: string, { data, executor, evaluators, config }: EvaluatorConstructorProps<D, T, O>): void;
|
|
357
382
|
|
|
358
383
|
interface ObserveOptions {
|
|
359
384
|
name?: string;
|
|
@@ -380,4 +405,4 @@ interface ObserveOptions {
|
|
|
380
405
|
*/
|
|
381
406
|
declare function observe<A extends unknown[], F extends (...args: A) => ReturnType<F>>({ name, sessionId, userId, }: ObserveOptions, fn: F, ...args: A): Promise<ReturnType<F>>;
|
|
382
407
|
|
|
383
|
-
export { type ChatMessage, type Datapoint, Dataset, type EvaluateEvent,
|
|
408
|
+
export { type ChatMessage, type Datapoint, Dataset, type EvaluateEvent, type Event, Laminar, type NodeInput, type PipelineRunRequest, type PipelineRunResponse, evaluate, observe };
|