@lmnr-ai/lmnr 0.6.4 → 0.6.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. package/dist/cli.d.mts +3 -3
  2. package/dist/cli.d.ts +3 -3
  3. package/dist/cli.js +20 -23
  4. package/dist/cli.js.map +1 -1
  5. package/dist/cli.mjs +20 -23
  6. package/dist/cli.mjs.map +1 -1
  7. package/dist/{evaluations-DQIsUKAP.d.mts → evaluations-De0Vbq1A.d.mts} +49 -2
  8. package/dist/{evaluations-DQIsUKAP.d.ts → evaluations-De0Vbq1A.d.ts} +49 -2
  9. package/dist/index.d.mts +13 -3
  10. package/dist/index.d.ts +13 -3
  11. package/dist/index.js +157 -18
  12. package/dist/index.js.map +1 -1
  13. package/dist/index.mjs +157 -18
  14. package/dist/index.mjs.map +1 -1
  15. package/package.json +19 -19
package/dist/cli.d.mts CHANGED
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env node
2
- import { E as Evaluation } from './evaluations-DQIsUKAP.mjs';
2
+ import { E as Evaluation } from './evaluations-De0Vbq1A.mjs';
3
3
  import '@anthropic-ai/sdk';
4
4
  import '@aws-sdk/client-bedrock-runtime';
5
5
  import '@azure/openai';
@@ -23,12 +23,12 @@ import 'puppeteer';
23
23
  import 'together-ai';
24
24
 
25
25
  declare global {
26
- var _evaluation: Evaluation<any, any, any> | undefined;
26
+ var _evaluations: Evaluation<any, any, any>[] | undefined;
27
27
  var _set_global_evaluation: boolean;
28
28
  }
29
29
  declare function loadModule({ filename, moduleText, }: {
30
30
  filename: string;
31
31
  moduleText: string;
32
- }): Evaluation<any, any, any>;
32
+ }): Evaluation<any, any, any>[];
33
33
 
34
34
  export { loadModule };
package/dist/cli.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env node
2
- import { E as Evaluation } from './evaluations-DQIsUKAP.js';
2
+ import { E as Evaluation } from './evaluations-De0Vbq1A.js';
3
3
  import '@anthropic-ai/sdk';
4
4
  import '@aws-sdk/client-bedrock-runtime';
5
5
  import '@azure/openai';
@@ -23,12 +23,12 @@ import 'puppeteer';
23
23
  import 'together-ai';
24
24
 
25
25
  declare global {
26
- var _evaluation: Evaluation<any, any, any> | undefined;
26
+ var _evaluations: Evaluation<any, any, any>[] | undefined;
27
27
  var _set_global_evaluation: boolean;
28
28
  }
29
29
  declare function loadModule({ filename, moduleText, }: {
30
30
  filename: string;
31
31
  moduleText: string;
32
- }): Evaluation<any, any, any>;
32
+ }): Evaluation<any, any, any>[];
33
33
 
34
34
  export { loadModule };
package/dist/cli.js CHANGED
@@ -34,21 +34,20 @@ __export(cli_exports, {
34
34
  loadModule: () => loadModule
35
35
  });
36
36
  module.exports = __toCommonJS(cli_exports);
37
- var import_api2 = require("@opentelemetry/api");
38
37
  var import_argparse = require("argparse");
39
38
  var esbuild = __toESM(require("esbuild"));
40
39
  var glob = __toESM(require("glob"));
41
40
 
42
41
  // package.json
43
- var version = "0.6.4";
42
+ var version = "0.6.6";
44
43
 
45
44
  // src/utils.ts
46
45
  var import_api = require("@opentelemetry/api");
46
+ var import_path = __toESM(require("path"));
47
47
  var import_pino = __toESM(require("pino"));
48
48
  var import_pino_pretty = __toESM(require("pino-pretty"));
49
- var import_uuid = require("uuid");
50
49
  var import_url = require("url");
51
- var import_path = __toESM(require("path"));
50
+ var import_uuid = require("uuid");
52
51
  var import_meta = {};
53
52
  function initializeLogger(options) {
54
53
  const colorize = options?.colorize ?? true;
@@ -75,7 +74,7 @@ function loadModule({
75
74
  filename,
76
75
  moduleText
77
76
  }) {
78
- globalThis._evaluation = void 0;
77
+ globalThis._evaluations = [];
79
78
  globalThis._set_global_evaluation = true;
80
79
  const __filename = filename;
81
80
  const __dirname2 = getDirname();
@@ -91,7 +90,7 @@ function loadModule({
91
90
  __filename,
92
91
  __dirname2
93
92
  );
94
- return globalThis._evaluation;
93
+ return globalThis._evaluations;
95
94
  }
96
95
  async function cli() {
97
96
  const [, , ...args] = process.argv;
@@ -130,12 +129,10 @@ async function cli() {
130
129
  const setLaminarAsExternalPlugin = {
131
130
  name: "set-laminar-as-external",
132
131
  setup(build2) {
133
- build2.onResolve({ filter: /^@lmnr-ai\/lmnr$/ }, (args3) => {
134
- return {
135
- path: args3.path,
136
- external: true
137
- };
138
- });
132
+ build2.onResolve({ filter: /^@lmnr-ai\/lmnr$/ }, (args3) => ({
133
+ path: args3.path,
134
+ external: true
135
+ }));
139
136
  }
140
137
  };
141
138
  for (const file of files) {
@@ -167,21 +164,21 @@ async function cli() {
167
164
  continue;
168
165
  }
169
166
  const outputFileText = result.outputFiles[0].text;
170
- const evaluation = loadModule({
171
- filename: args2.file,
167
+ const evaluations = loadModule({
168
+ filename: file,
172
169
  moduleText: outputFileText
173
170
  });
174
- if (!evaluation?.run) {
175
- logger2.error(`Evaluation ${file} does not properly call evaluate()`);
176
- if (args2.fail_on_error) {
177
- process.exit(1);
171
+ logger2.info(`Loaded ${evaluations.length} evaluations from ${file}`);
172
+ for (const evaluation of evaluations) {
173
+ if (!evaluation?.run) {
174
+ logger2.error(`Evaluation ${file} does not properly call evaluate()`);
175
+ if (args2.fail_on_error) {
176
+ process.exit(1);
177
+ }
178
+ continue;
178
179
  }
179
- continue;
180
+ await evaluation.run();
180
181
  }
181
- await evaluation.run();
182
- import_api2.context.disable();
183
- import_api2.trace.disable();
184
- import_api2.propagation.disable();
185
182
  }
186
183
  }
187
184
  });
package/dist/cli.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/cli.ts","../package.json","../src/utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { context, propagation, trace } from \"@opentelemetry/api\";\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\n\nimport { version } from \"../package.json\";\nimport { Evaluation } from \"./evaluations\";\nimport { initializeLogger } from \"./utils\";\nimport { getDirname } from \"./utils\";\n\nconst logger = initializeLogger();\n\ndeclare global {\n // eslint-disable-next-line no-var\n var _evaluation: Evaluation<any, any, any> | undefined;\n // eslint-disable-next-line no-var\n var _set_global_evaluation: boolean;\n}\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}): Evaluation<any, any, any> {\n globalThis._evaluation = undefined;\n globalThis._set_global_evaluation = true;\n\n const __filename = filename;\n const __dirname = getDirname();\n\n // add some arguments for proper cjs/esm interop\n new Function(\n \"require\",\n \"module\",\n \"__filename\",\n \"__dirname\",\n moduleText,\n )(\n require,\n module,\n __filename,\n __dirname,\n );\n\n // Return the modified _evals global variable\n return globalThis._evaluation!;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n const files = args.file\n ? [args.file]\n : glob.sync('evals/**/*.eval.{ts,js}');\n\n files.sort();\n\n if (files.length === 0) {\n logger.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are eval files that are named like `*.eval.{ts,js}` in\" +\n \"the `evals` directory or its subdirectories.\");\n process.exit(1);\n }\n\n if (!args.file) {\n logger.info(`Located ${files.length} evaluation files in evals/`);\n }\n\n // Laminar should be marked external by passing 'node_modules/*' to the\n // external option. This plugin ensures that @lmnr-ai/lmnr is marked\n // external when imported from path or url.\n // Adapted from https://github.com/evanw/esbuild/issues/619#issuecomment-751769812\n const setLaminarAsExternalPlugin = {\n name: \"set-laminar-as-external\",\n setup(build: esbuild.PluginBuild) {\n build.onResolve({ filter: /^@lmnr-ai\\/lmnr$/ }, (args) => {\n return {\n path: args.path,\n external: true,\n };\n });\n }\n }\n\n for (const file of files) {\n logger.info(`Loading ${file}...`);\n const buildOptions = {\n bundle: true,\n platform: \"node\" as esbuild.Platform,\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n external: [\n \"node_modules/*\",\n \"playwright\",\n \"puppeteer\",\n \"puppeteer-core\",\n \"playwright-core\",\n \"fsevents\",\n ],\n treeShaking: true,\n plugins: [setLaminarAsExternalPlugin],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n logger.error(\"Error when building: No output files found \" +\n \"it is likely that all eval files are not valid TypeScript or JavaScript files.\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n if (!evaluation?.run) {\n logger.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n await evaluation.run();\n\n // FIXME: Now every evaluation file creates a new tracer provider.\n // Attempt to re-initialize it in the same process breaks it.\n // For now, we disable all APIs after running each file, but ideally\n // we should keep a global tracer provider that is initialized once\n // here in the CLI.\n context.disable();\n trace.disable();\n propagation.disable();\n }\n },\n });\n\n parser.set_defaults({\n func: () => {\n parser.print_help();\n process.exit(0);\n },\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli().catch((err) => {\n logger.error(err instanceof Error ? err.message : err);\n throw err;\n});\n","{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.6.4\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup\",\n \"test\": \"tsx --test test/*.test.ts\",\n \"lint\": \"eslint\",\n \"lint:fix\": \"eslint --fix\"\n },\n \"files\": [\n \"dist\",\n \"assets\"\n ],\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.39.0\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.806.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^1.14.0\",\n \"@eslint/eslintrc\": \"^3.3.1\",\n \"@eslint/js\": \"^9.26.0\",\n \"@google-cloud/aiplatform\": \"^4.1.0\",\n \"@google-cloud/vertexai\": \"^1.10.0\",\n \"@langchain/core\": \"^0.3.55\",\n \"@pinecone-database/pinecone\": \"^5.1.2\",\n \"@playwright/test\": \"^1.52.0\",\n \"@qdrant/js-client-rest\": \"^1.14.0\",\n \"@stylistic/eslint-plugin-js\": \"^4.2.0\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.15.17\",\n \"@types/semver\": \"^7.7.0\",\n \"@types/uuid\": \"^10.0.0\",\n \"chromadb\": \"^2.4.3\",\n \"cohere-ai\": \"^7.17.1\",\n \"eslint-plugin-simple-import-sort\": \"^12.1.1\",\n \"eslint-plugin-unused-imports\": \"^4.1.4\",\n \"langchain\": \"^0.3.24\",\n \"llamaindex\": \"^0.9.19\",\n \"nock\": \"^14.0.4\",\n \"openai\": \"^4.98.0\",\n \"playwright\": \"^1.52.0\",\n \"puppeteer\": \"^24.8.2\",\n \"puppeteer-core\": \"^24.8.2\",\n \"together-ai\": \"^0.14.0\",\n \"tsup\": \"^8.4.0\",\n \"tsx\": \"^4.19.4\",\n \"typescript\": \"^5.8.3\",\n \"typescript-eslint\": \"^8.32.0\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.13.3\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/context-async-hooks\": \"^1.30.1\",\n \"@opentelemetry/core\": \"^1.30.1\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.2\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.2\",\n \"@opentelemetry/instrumentation\": \"^0.57.2\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/resources\": \"^1.30.1\",\n \"@opentelemetry/sdk-node\": \"^0.57.2\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.1\",\n \"@opentelemetry/sdk-trace-base-v2\": \"npm:@opentelemetry/sdk-trace-base@2.0.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.33.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.12.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.12.0\",\n \"@traceloop/instrumentation-azure\": \"^0.12.0\",\n \"@traceloop/instrumentation-bedrock\": \"^0.12.0\",\n \"@traceloop/instrumentation-chromadb\": \"^0.12.0\",\n \"@traceloop/instrumentation-cohere\": \"^0.12.0\",\n \"@traceloop/instrumentation-langchain\": \"^0.12.0\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.12.0\",\n \"@traceloop/instrumentation-openai\": \"^0.12.0\",\n \"@traceloop/instrumentation-pinecone\": \"^0.12.0\",\n \"@traceloop/instrumentation-qdrant\": \"^0.12.0\",\n \"@traceloop/instrumentation-together\": \"^0.12.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.12.0\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.25.4\",\n \"glob\": \"^11.0.2\",\n \"pino\": \"^9.6.0\",\n \"pino-pretty\": \"^13.0.0\",\n \"uuid\": \"^11.1.0\",\n \"zod\": \"^3.24.4\"\n }\n}","import { SpanContext, TraceFlags } from '@opentelemetry/api';\nimport pino, { Level } from 'pino';\nimport pinoPretty from 'pino-pretty';\nimport { v4 as uuidv4 } from 'uuid';\nimport { fileURLToPath } from \"url\";\nimport path from \"path\";\n\nimport { LaminarSpanContext } from './types';\n\nexport function initializeLogger(options?: { colorize?: boolean, level?: Level }) {\n const colorize = options?.colorize ?? true;\n const level = options?.level\n ?? (process.env.LMNR_LOG_LEVEL?.toLowerCase()?.trim() as Level)\n ?? 'info';\n\n return pino(pinoPretty({\n colorize,\n minimumLevel: level,\n }));\n}\n\nconst logger = initializeLogger();\n\nexport type StringUUID = `${string}-${string}-${string}-${string}-${string}`;\n\nexport const isStringUUID = (id: string): id is StringUUID =>\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id);\n\nexport const NIL_UUID: StringUUID = '00000000-0000-0000-0000-000000000000';\n\nexport const newUUID = (): StringUUID => {\n // crypto.randomUUID is available in most of the modern browsers and node,\n // but is not available in \"insecure\" contexts, e.g. not https, not localhost\n // so we fallback to uuidv4 in those cases, which is less secure, but works\n // just fine.\n if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {\n return crypto.randomUUID();\n } else {\n return uuidv4() as `${string}-${string}-${string}-${string}-${string}`;\n }\n};\n\nexport const otelSpanIdToUUID = (spanId: string): string => {\n let id = spanId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 16) {\n logger.warn(`Span ID ${spanId} is not 16 hex chars long. ` +\n 'This is not a valid OpenTelemetry span ID.');\n }\n\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Span ID ${spanId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.padStart(32, '0').replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, '$1-$2-$3-$4-$5');\n};\n\nexport const otelTraceIdToUUID = (traceId: string): string => {\n let id = traceId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 32) {\n logger.warn(`Trace ID ${traceId} is not 32 hex chars long. ` +\n 'This is not a valid OpenTelemetry trace ID.');\n }\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Trace ID ${traceId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, '$1-$2-$3-$4-$5');\n};\n\nexport const uuidToOtelTraceId = (uuid: string): string => uuid.replace(/-/g, '');\nexport const uuidToOtelSpanId = (uuid: string): string => uuid.replace(/-/g, '').slice(16);\n\n/**\n * This is a simple implementation of a semaphore to replicate\n * the behavior of the `asyncio.Semaphore` in Python.\n */\nexport class Semaphore {\n /**\n * Number of permits available.\n */\n private _value: number;\n /**\n * List of promises that will be resolved when a permit becomes available.\n */\n private _waiters: ((...args: any[]) => any)[] = [];\n\n constructor(value = 1) {\n if (value < 0) {\n throw new Error(\"Semaphore value must be >= 0\");\n }\n this._value = value;\n this._waiters = [];\n }\n\n async acquire() {\n if (this._value > 0) {\n this._value--;\n return;\n }\n\n // Create a promise that will be resolved when a permit becomes available\n return new Promise(resolve => {\n this._waiters.push(resolve);\n });\n }\n\n release() {\n if (this._waiters.length > 0) {\n // If there are waiters, wake up the first one\n const resolve = this._waiters.shift();\n resolve?.();\n } else {\n this._value++;\n }\n }\n\n // Python-like context manager functionality\n async using<T>(fn: (...args: any[]) => Promise<T>) {\n try {\n await this.acquire();\n return await fn();\n } finally {\n this.release();\n }\n }\n}\n\nexport const tryToOtelSpanContext = (\n spanContext: LaminarSpanContext | Record<string, unknown> | string | SpanContext,\n): SpanContext => {\n if (typeof spanContext === 'string') {\n try {\n const record = JSON.parse(spanContext) as Record<string, unknown>;\n return recordToOtelSpanContext(record);\n } catch (e) {\n throw new Error(`Failed to parse span context ${spanContext}. ` +\n 'The string must be a json representation of a LaminarSpanContext.'\n + `Error: ${e instanceof Error ? e.message : String(e)}`);\n }\n } else if (isRecord(spanContext)) {\n // This covers the `LaminarSpanContext` case too.\n return recordToOtelSpanContext(spanContext);\n } else if (typeof spanContext.traceId === 'string'\n && typeof spanContext.spanId === 'string'\n && spanContext.traceId.length === 32\n && spanContext.spanId.length === 16) {\n logger.warn('The span context is already an OpenTelemetry SpanContext. ' +\n 'Returning it as is. Please use `LaminarSpanContext` objects instead.');\n return spanContext;\n }\n else {\n throw new Error(`Invalid span context ${JSON.stringify(spanContext)}. ` +\n 'Must be a LaminarSpanContext or its json representation.');\n }\n};\n\nconst recordToOtelSpanContext = (record: Record<string, unknown>): SpanContext => {\n if (typeof record.spanId === 'string' && typeof record.traceId === 'string') {\n return {\n spanId: uuidToOtelSpanId(record?.spanId ?? record?.['span_id']),\n traceId: uuidToOtelTraceId(record?.traceId ?? record?.['trace_id']),\n isRemote: record?.isRemote ?? record?.['is_remote'] ?? false,\n traceFlags: record?.traceFlags ?? TraceFlags.SAMPLED,\n } as SpanContext;\n } else {\n throw new Error(`Invalid span context ${JSON.stringify(record)}. ` +\n 'Must be a json representation of a LaminarSpanContext.');\n }\n};\n\nconst isRecord = (value: unknown): value is Record<string, unknown> =>\n typeof value === 'object' && !Array.isArray(value) && value !== null;\n\n\nexport const getDirname = () => {\n if (typeof __dirname !== 'undefined') {\n return __dirname;\n }\n\n if (typeof import.meta?.url !== 'undefined') {\n return path.dirname(fileURLToPath(import.meta.url));\n }\n\n return process.cwd();\n};"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAEA,IAAAA,cAA4C;AAC5C,sBAA+B;AAC/B,cAAyB;AACzB,WAAsB;;;ACHpB,cAAW;;;ACFb,iBAAwC;AACxC,kBAA4B;AAC5B,yBAAuB;AACvB,kBAA6B;AAC7B,iBAA8B;AAC9B,kBAAiB;AALjB;AASO,SAAS,iBAAiB,SAAiD;AAChF,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,QAAQ,SAAS,SACjB,QAAQ,IAAI,gBAAgB,YAAY,GAAG,KAAK,KACjD;AAEL,aAAO,YAAAC,aAAK,mBAAAC,SAAW;AAAA,IACrB;AAAA,IACA,cAAc;AAAA,EAChB,CAAC,CAAC;AACJ;AAEA,IAAM,SAAS,iBAAiB;AAmKzB,IAAM,aAAa,MAAM;AAC9B,MAAI,OAAO,cAAc,aAAa;AACpC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,aAAa,QAAQ,aAAa;AAC3C,WAAO,YAAAC,QAAK,YAAQ,0BAAc,YAAY,GAAG,CAAC;AAAA,EACpD;AAEA,SAAO,QAAQ,IAAI;AACrB;;;AFtLA,IAAMC,UAAS,iBAAiB;AASzB,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAG8B;AAC5B,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAEpC,QAAM,aAAa;AACnB,QAAMC,aAAY,WAAW;AAG7B,MAAI;AAAA,IACF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACAA;AAAA,EACF;AAGA,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,QAAQ,CAAC;AAErE,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,QAAAF,QAAO,MAAM,qKAEmC;AAChD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACE,MAAK,MAAM;AACd,QAAAF,QAAO,KAAK,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAMA,YAAM,6BAA6B;AAAA,QACjC,MAAM;AAAA,QACN,MAAMG,QAA4B;AAChC,UAAAA,OAAM,UAAU,EAAE,QAAQ,mBAAmB,GAAG,CAACD,UAAS;AACxD,mBAAO;AAAA,cACL,MAAMA,MAAK;AAAA,cACX,UAAU;AAAA,YACZ;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,iBAAW,QAAQ,OAAO;AACxB,QAAAF,QAAO,KAAK,WAAW,IAAI,KAAK;AAChC,cAAM,eAAe;AAAA,UACnB,QAAQ;AAAA,UACR,UAAU;AAAA,UACV,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,UACA,aAAa;AAAA,UACb,SAAS,CAAC,0BAA0B;AAAA,QACtC;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,UAAAA,QAAO,MAAM,2HACqE;AAClF,cAAIE,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,aAAa,WAAW;AAAA,UAC5B,UAAUA,MAAK;AAAA,UACf,YAAY;AAAA,QACd,CAAC;AAED,YAAI,CAAC,YAAY,KAAK;AACpB,UAAAF,QAAO,MAAM,cAAc,IAAI,oCAAoC;AACnE,cAAIE,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,WAAW,IAAI;AAOrB,4BAAQ,QAAQ;AAChB,0BAAM,QAAQ;AACd,gCAAY,QAAQ;AAAA,MACtB;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,MAAM;AACV,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI,EAAE,MAAM,CAAC,QAAQ;AACnB,EAAAF,QAAO,MAAM,eAAe,QAAQ,IAAI,UAAU,GAAG;AACrD,QAAM;AACR,CAAC;","names":["import_api","pino","pinoPretty","path","logger","__dirname","args","build"]}
1
+ {"version":3,"sources":["../src/cli.ts","../package.json","../src/utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { context, propagation, trace } from \"@opentelemetry/api\";\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\n\nimport { version } from \"../package.json\";\nimport { Evaluation } from \"./evaluations\";\nimport { getDirname, initializeLogger } from \"./utils\";\n\nconst logger = initializeLogger();\n\ndeclare global {\n // eslint-disable-next-line no-var\n var _evaluations: Evaluation<any, any, any>[] | undefined;\n // eslint-disable-next-line no-var\n var _set_global_evaluation: boolean;\n}\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}): Evaluation<any, any, any>[] {\n globalThis._evaluations = [];\n globalThis._set_global_evaluation = true;\n\n const __filename = filename;\n const __dirname = getDirname();\n\n // add some arguments for proper cjs/esm interop\n\n /* eslint-disable @typescript-eslint/no-implied-eval */\n new Function(\n \"require\",\n \"module\",\n \"__filename\",\n \"__dirname\",\n moduleText,\n )(\n require,\n module,\n __filename,\n __dirname,\n );\n /* eslint-enable @typescript-eslint/no-implied-eval */\n\n // Return the modified _evals global variable\n return globalThis._evaluations!;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n const files = args.file\n ? [args.file]\n : glob.sync('evals/**/*.eval.{ts,js}');\n\n files.sort();\n\n if (files.length === 0) {\n logger.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are eval files that are named like `*.eval.{ts,js}` in\" +\n \"the `evals` directory or its subdirectories.\");\n process.exit(1);\n }\n\n if (!args.file) {\n logger.info(`Located ${files.length} evaluation files in evals/`);\n }\n\n // Laminar should be marked external by passing 'node_modules/*' to the\n // external option. This plugin ensures that @lmnr-ai/lmnr is marked\n // external when imported from path or url.\n // Adapted from https://github.com/evanw/esbuild/issues/619#issuecomment-751769812\n const setLaminarAsExternalPlugin = {\n name: \"set-laminar-as-external\",\n setup(build: esbuild.PluginBuild) {\n build.onResolve({ filter: /^@lmnr-ai\\/lmnr$/ }, (args) => ({\n path: args.path,\n external: true,\n }));\n },\n };\n\n for (const file of files) {\n logger.info(`Loading ${file}...`);\n const buildOptions = {\n bundle: true,\n platform: \"node\" as esbuild.Platform,\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n external: [\n \"node_modules/*\",\n \"playwright\",\n \"puppeteer\",\n \"puppeteer-core\",\n \"playwright-core\",\n \"fsevents\",\n ],\n treeShaking: true,\n plugins: [setLaminarAsExternalPlugin],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n logger.error(\"Error when building: No output files found \" +\n \"it is likely that all eval files are not valid TypeScript or JavaScript files.\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluations = loadModule({\n filename: file,\n moduleText: outputFileText,\n });\n\n logger.info(`Loaded ${evaluations.length} evaluations from ${file}`);\n\n for (const evaluation of evaluations) {\n if (!evaluation?.run) {\n logger.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n await evaluation.run();\n }\n }\n },\n });\n\n parser.set_defaults({\n func: () => {\n parser.print_help();\n process.exit(0);\n },\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli().catch((err) => {\n logger.error(err instanceof Error ? err.message : err);\n throw err;\n});\n","{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.6.6\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup\",\n \"test\": \"tsx --test test/*.test.ts\",\n \"lint\": \"eslint\",\n \"lint:fix\": \"eslint --fix\"\n },\n \"files\": [\n \"dist\",\n \"assets\"\n ],\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.39.0\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.817.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^1.14.0\",\n \"@eslint/eslintrc\": \"^3.3.1\",\n \"@eslint/js\": \"^9.27.0\",\n \"@google-cloud/aiplatform\": \"^4.2.0\",\n \"@google-cloud/vertexai\": \"^1.10.0\",\n \"@langchain/core\": \"^0.3.57\",\n \"@pinecone-database/pinecone\": \"^5.1.2\",\n \"@playwright/test\": \"^1.52.0\",\n \"@qdrant/js-client-rest\": \"^1.14.1\",\n \"@stylistic/eslint-plugin-js\": \"^4.4.0\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.15.21\",\n \"@types/semver\": \"^7.7.0\",\n \"@types/uuid\": \"^10.0.0\",\n \"chromadb\": \"^2.4.6\",\n \"cohere-ai\": \"^7.17.1\",\n \"eslint-plugin-simple-import-sort\": \"^12.1.1\",\n \"eslint-plugin-unused-imports\": \"^4.1.4\",\n \"langchain\": \"^0.3.27\",\n \"llamaindex\": \"^0.9.19\",\n \"nock\": \"^14.0.4\",\n \"openai\": \"^4.103.0\",\n \"playwright\": \"^1.52.0\",\n \"puppeteer\": \"^24.9.0\",\n \"puppeteer-core\": \"^24.9.0\",\n \"together-ai\": \"^0.14.0\",\n \"tsup\": \"^8.5.0\",\n \"tsx\": \"^4.19.4\",\n \"typescript\": \"^5.8.3\",\n \"typescript-eslint\": \"^8.32.1\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.13.4\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/context-async-hooks\": \"^1.30.1\",\n \"@opentelemetry/core\": \"^1.30.1\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.2\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.2\",\n \"@opentelemetry/instrumentation\": \"^0.57.2\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/resources\": \"^1.30.1\",\n \"@opentelemetry/sdk-node\": \"^0.57.2\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.1\",\n \"@opentelemetry/sdk-trace-base-v2\": \"npm:@opentelemetry/sdk-trace-base@2.0.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.34.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.12.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.12.0\",\n \"@traceloop/instrumentation-azure\": \"^0.12.0\",\n \"@traceloop/instrumentation-bedrock\": \"^0.12.0\",\n \"@traceloop/instrumentation-chromadb\": \"^0.12.0\",\n \"@traceloop/instrumentation-cohere\": \"^0.12.0\",\n \"@traceloop/instrumentation-langchain\": \"^0.12.0\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.12.0\",\n \"@traceloop/instrumentation-openai\": \"^0.12.0\",\n \"@traceloop/instrumentation-pinecone\": \"^0.12.0\",\n \"@traceloop/instrumentation-qdrant\": \"^0.12.0\",\n \"@traceloop/instrumentation-together\": \"^0.12.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.12.0\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.25.4\",\n \"glob\": \"^11.0.2\",\n \"pino\": \"^9.7.0\",\n \"pino-pretty\": \"^13.0.0\",\n \"uuid\": \"^11.1.0\",\n \"zod\": \"^3.25.28\"\n }\n}","import { AttributeValue, SpanContext, TraceFlags } from '@opentelemetry/api';\nimport path from \"path\";\nimport pino, { Level } from 'pino';\nimport pinoPretty from 'pino-pretty';\nimport { fileURLToPath } from \"url\";\nimport { v4 as uuidv4 } from 'uuid';\n\nimport { LaminarSpanContext } from './types';\n\nexport function initializeLogger(options?: { colorize?: boolean, level?: Level }) {\n const colorize = options?.colorize ?? true;\n const level = options?.level\n ?? (process.env.LMNR_LOG_LEVEL?.toLowerCase()?.trim() as Level)\n ?? 'info';\n\n return pino(pinoPretty({\n colorize,\n minimumLevel: level,\n }));\n}\n\nconst logger = initializeLogger();\n\nexport type StringUUID = `${string}-${string}-${string}-${string}-${string}`;\n\nexport const isStringUUID = (id: string): id is StringUUID =>\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id);\n\nexport const NIL_UUID: StringUUID = '00000000-0000-0000-0000-000000000000';\n\nexport const newUUID = (): StringUUID => {\n // crypto.randomUUID is available in most of the modern browsers and node,\n // but is not available in \"insecure\" contexts, e.g. not https, not localhost\n // so we fallback to uuidv4 in those cases, which is less secure, but works\n // just fine.\n if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {\n return crypto.randomUUID();\n } else {\n return uuidv4() as `${string}-${string}-${string}-${string}-${string}`;\n }\n};\n\nexport const otelSpanIdToUUID = (spanId: string): string => {\n let id = spanId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 16) {\n logger.warn(`Span ID ${spanId} is not 16 hex chars long. ` +\n 'This is not a valid OpenTelemetry span ID.');\n }\n\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Span ID ${spanId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.padStart(32, '0').replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5'\n );\n};\n\nexport const otelTraceIdToUUID = (traceId: string): string => {\n let id = traceId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 32) {\n logger.warn(`Trace ID ${traceId} is not 32 hex chars long. ` +\n 'This is not a valid OpenTelemetry trace ID.');\n }\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Trace ID ${traceId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5'\n );\n};\n\nexport const uuidToOtelTraceId = (uuid: string): string => uuid.replace(/-/g, '');\nexport const uuidToOtelSpanId = (uuid: string): string => uuid.replace(/-/g, '').slice(16);\n\n/**\n * This is a simple implementation of a semaphore to replicate\n * the behavior of the `asyncio.Semaphore` in Python.\n */\nexport class Semaphore {\n /**\n * Number of permits available.\n */\n private _value: number;\n /**\n * List of promises that will be resolved when a permit becomes available.\n */\n private _waiters: ((...args: any[]) => any)[] = [];\n\n constructor(value = 1) {\n if (value < 0) {\n throw new Error(\"Semaphore value must be >= 0\");\n }\n this._value = value;\n this._waiters = [];\n }\n\n async acquire() {\n if (this._value > 0) {\n this._value--;\n return;\n }\n\n // Create a promise that will be resolved when a permit becomes available\n return new Promise(resolve => {\n this._waiters.push(resolve);\n });\n }\n\n release() {\n if (this._waiters.length > 0) {\n // If there are waiters, wake up the first one\n const resolve = this._waiters.shift();\n resolve?.();\n } else {\n this._value++;\n }\n }\n\n // Python-like context manager functionality\n async using<T>(fn: (...args: any[]) => Promise<T>) {\n try {\n await this.acquire();\n return await fn();\n } finally {\n this.release();\n }\n }\n}\n\nexport const tryToOtelSpanContext = (\n spanContext: LaminarSpanContext | Record<string, unknown> | string | SpanContext,\n): SpanContext => {\n if (typeof spanContext === 'string') {\n try {\n const record = JSON.parse(spanContext) as Record<string, unknown>;\n return recordToOtelSpanContext(record);\n } catch (e) {\n throw new Error(`Failed to parse span context ${spanContext}. ` +\n 'The string must be a json representation of a LaminarSpanContext.'\n + `Error: ${e instanceof Error ? e.message : String(e)}`);\n }\n } else if (isRecord(spanContext)) {\n // This covers the `LaminarSpanContext` case too.\n return recordToOtelSpanContext(spanContext);\n } else if (typeof spanContext.traceId === 'string'\n && typeof spanContext.spanId === 'string'\n && spanContext.traceId.length === 32\n && spanContext.spanId.length === 16) {\n logger.warn('The span context is already an OpenTelemetry SpanContext. ' +\n 'Returning it as is. Please use `LaminarSpanContext` objects instead.');\n return spanContext;\n }\n else {\n throw new Error(`Invalid span context ${JSON.stringify(spanContext)}. ` +\n 'Must be a LaminarSpanContext or its json representation.');\n }\n};\n\nconst recordToOtelSpanContext = (record: Record<string, unknown>): SpanContext => {\n if (typeof record.spanId === 'string' && typeof record.traceId === 'string') {\n return {\n spanId: uuidToOtelSpanId(record?.spanId ?? record?.['span_id']),\n traceId: uuidToOtelTraceId(record?.traceId ?? record?.['trace_id']),\n isRemote: record?.isRemote ?? record?.['is_remote'] ?? false,\n traceFlags: record?.traceFlags ?? TraceFlags.SAMPLED,\n } as SpanContext;\n } else {\n throw new Error(`Invalid span context ${JSON.stringify(record)}. ` +\n 'Must be a json representation of a LaminarSpanContext.');\n }\n};\n\nconst isRecord = (value: unknown): value is Record<string, unknown> =>\n typeof value === 'object' && !Array.isArray(value) && value !== null;\n\n\nexport const getDirname = () => {\n if (typeof __dirname !== 'undefined') {\n return __dirname;\n }\n\n if (typeof import.meta?.url !== 'undefined') {\n return path.dirname(fileURLToPath(import.meta.url));\n }\n\n return process.cwd();\n};\n\nexport const slicePayload = <T>(value: T, length: number) => {\n if (value === null || value === undefined) {\n return value;\n }\n\n const str = JSON.stringify(value);\n if (str.length <= length) {\n return value;\n }\n\n return (str.slice(0, length) + '...');\n};\n\nexport const isOtelAttributeValueType = (value: unknown): value is AttributeValue => {\n if (typeof value === 'string'\n || typeof value === 'number'\n || typeof value === 'boolean') {\n return true;\n }\n\n if (Array.isArray(value)) {\n const allStrings = value.every(value => (value == null) || typeof value === 'string');\n const allNumbers = value.every(value => (value == null) || typeof value === 'number');\n const allBooleans = value.every(value => (value == null) || typeof value === 'boolean');\n return allStrings || allNumbers || allBooleans;\n }\n return false;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,sBAA+B;AAC/B,cAAyB;AACzB,WAAsB;;;ACHpB,cAAW;;;ACFb,iBAAwD;AACxD,kBAAiB;AACjB,kBAA4B;AAC5B,yBAAuB;AACvB,iBAA8B;AAC9B,kBAA6B;AAL7B;AASO,SAAS,iBAAiB,SAAiD;AAChF,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,QAAQ,SAAS,SACjB,QAAQ,IAAI,gBAAgB,YAAY,GAAG,KAAK,KACjD;AAEL,aAAO,YAAAA,aAAK,mBAAAC,SAAW;AAAA,IACrB;AAAA,IACA,cAAc;AAAA,EAChB,CAAC,CAAC;AACJ;AAEA,IAAM,SAAS,iBAAiB;AAyKzB,IAAM,aAAa,MAAM;AAC9B,MAAI,OAAO,cAAc,aAAa;AACpC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,aAAa,QAAQ,aAAa;AAC3C,WAAO,YAAAC,QAAK,YAAQ,0BAAc,YAAY,GAAG,CAAC;AAAA,EACpD;AAEA,SAAO,QAAQ,IAAI;AACrB;;;AF7LA,IAAMC,UAAS,iBAAiB;AASzB,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGgC;AAC9B,aAAW,eAAe,CAAC;AAC3B,aAAW,yBAAyB;AAEpC,QAAM,aAAa;AACnB,QAAMC,aAAY,WAAW;AAK7B,MAAI;AAAA,IACF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACAA;AAAA,EACF;AAIA,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,QAAQ,CAAC;AAErE,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,QAAAF,QAAO,MAAM,qKAEmC;AAChD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACE,MAAK,MAAM;AACd,QAAAF,QAAO,KAAK,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAMA,YAAM,6BAA6B;AAAA,QACjC,MAAM;AAAA,QACN,MAAMG,QAA4B;AAChC,UAAAA,OAAM,UAAU,EAAE,QAAQ,mBAAmB,GAAG,CAACD,WAAU;AAAA,YACzD,MAAMA,MAAK;AAAA,YACX,UAAU;AAAA,UACZ,EAAE;AAAA,QACJ;AAAA,MACF;AAEA,iBAAW,QAAQ,OAAO;AACxB,QAAAF,QAAO,KAAK,WAAW,IAAI,KAAK;AAChC,cAAM,eAAe;AAAA,UACnB,QAAQ;AAAA,UACR,UAAU;AAAA,UACV,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,UACA,aAAa;AAAA,UACb,SAAS,CAAC,0BAA0B;AAAA,QACtC;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,UAAAA,QAAO,MAAM,2HACqE;AAClF,cAAIE,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,cAAc,WAAW;AAAA,UAC7B,UAAU;AAAA,UACV,YAAY;AAAA,QACd,CAAC;AAED,QAAAF,QAAO,KAAK,UAAU,YAAY,MAAM,qBAAqB,IAAI,EAAE;AAEnE,mBAAW,cAAc,aAAa;AACpC,cAAI,CAAC,YAAY,KAAK;AACpB,YAAAA,QAAO,MAAM,cAAc,IAAI,oCAAoC;AACnE,gBAAIE,MAAK,eAAe;AACtB,sBAAQ,KAAK,CAAC;AAAA,YAChB;AACA;AAAA,UACF;AAEA,gBAAM,WAAW,IAAI;AAAA,QACvB;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,MAAM;AACV,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI,EAAE,MAAM,CAAC,QAAQ;AACnB,EAAAF,QAAO,MAAM,eAAe,QAAQ,IAAI,UAAU,GAAG;AACrD,QAAM;AACR,CAAC;","names":["pino","pinoPretty","path","logger","__dirname","args","build"]}
package/dist/cli.mjs CHANGED
@@ -7,21 +7,20 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
7
7
  });
8
8
 
9
9
  // src/cli.ts
10
- import { context, propagation, trace } from "@opentelemetry/api";
11
10
  import { ArgumentParser } from "argparse";
12
11
  import * as esbuild from "esbuild";
13
12
  import * as glob from "glob";
14
13
 
15
14
  // package.json
16
- var version = "0.6.4";
15
+ var version = "0.6.6";
17
16
 
18
17
  // src/utils.ts
19
18
  import { TraceFlags } from "@opentelemetry/api";
19
+ import path from "path";
20
20
  import pino from "pino";
21
21
  import pinoPretty from "pino-pretty";
22
- import { v4 as uuidv4 } from "uuid";
23
22
  import { fileURLToPath } from "url";
24
- import path from "path";
23
+ import { v4 as uuidv4 } from "uuid";
25
24
  function initializeLogger(options) {
26
25
  const colorize = options?.colorize ?? true;
27
26
  const level = options?.level ?? process.env.LMNR_LOG_LEVEL?.toLowerCase()?.trim() ?? "info";
@@ -47,7 +46,7 @@ function loadModule({
47
46
  filename,
48
47
  moduleText
49
48
  }) {
50
- globalThis._evaluation = void 0;
49
+ globalThis._evaluations = [];
51
50
  globalThis._set_global_evaluation = true;
52
51
  const __filename = filename;
53
52
  const __dirname2 = getDirname();
@@ -63,7 +62,7 @@ function loadModule({
63
62
  __filename,
64
63
  __dirname2
65
64
  );
66
- return globalThis._evaluation;
65
+ return globalThis._evaluations;
67
66
  }
68
67
  async function cli() {
69
68
  const [, , ...args] = process.argv;
@@ -102,12 +101,10 @@ async function cli() {
102
101
  const setLaminarAsExternalPlugin = {
103
102
  name: "set-laminar-as-external",
104
103
  setup(build2) {
105
- build2.onResolve({ filter: /^@lmnr-ai\/lmnr$/ }, (args3) => {
106
- return {
107
- path: args3.path,
108
- external: true
109
- };
110
- });
104
+ build2.onResolve({ filter: /^@lmnr-ai\/lmnr$/ }, (args3) => ({
105
+ path: args3.path,
106
+ external: true
107
+ }));
111
108
  }
112
109
  };
113
110
  for (const file of files) {
@@ -139,21 +136,21 @@ async function cli() {
139
136
  continue;
140
137
  }
141
138
  const outputFileText = result.outputFiles[0].text;
142
- const evaluation = loadModule({
143
- filename: args2.file,
139
+ const evaluations = loadModule({
140
+ filename: file,
144
141
  moduleText: outputFileText
145
142
  });
146
- if (!evaluation?.run) {
147
- logger2.error(`Evaluation ${file} does not properly call evaluate()`);
148
- if (args2.fail_on_error) {
149
- process.exit(1);
143
+ logger2.info(`Loaded ${evaluations.length} evaluations from ${file}`);
144
+ for (const evaluation of evaluations) {
145
+ if (!evaluation?.run) {
146
+ logger2.error(`Evaluation ${file} does not properly call evaluate()`);
147
+ if (args2.fail_on_error) {
148
+ process.exit(1);
149
+ }
150
+ continue;
150
151
  }
151
- continue;
152
+ await evaluation.run();
152
153
  }
153
- await evaluation.run();
154
- context.disable();
155
- trace.disable();
156
- propagation.disable();
157
154
  }
158
155
  }
159
156
  });
package/dist/cli.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/cli.ts","../package.json","../src/utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { context, propagation, trace } from \"@opentelemetry/api\";\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\n\nimport { version } from \"../package.json\";\nimport { Evaluation } from \"./evaluations\";\nimport { initializeLogger } from \"./utils\";\nimport { getDirname } from \"./utils\";\n\nconst logger = initializeLogger();\n\ndeclare global {\n // eslint-disable-next-line no-var\n var _evaluation: Evaluation<any, any, any> | undefined;\n // eslint-disable-next-line no-var\n var _set_global_evaluation: boolean;\n}\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}): Evaluation<any, any, any> {\n globalThis._evaluation = undefined;\n globalThis._set_global_evaluation = true;\n\n const __filename = filename;\n const __dirname = getDirname();\n\n // add some arguments for proper cjs/esm interop\n new Function(\n \"require\",\n \"module\",\n \"__filename\",\n \"__dirname\",\n moduleText,\n )(\n require,\n module,\n __filename,\n __dirname,\n );\n\n // Return the modified _evals global variable\n return globalThis._evaluation!;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n const files = args.file\n ? [args.file]\n : glob.sync('evals/**/*.eval.{ts,js}');\n\n files.sort();\n\n if (files.length === 0) {\n logger.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are eval files that are named like `*.eval.{ts,js}` in\" +\n \"the `evals` directory or its subdirectories.\");\n process.exit(1);\n }\n\n if (!args.file) {\n logger.info(`Located ${files.length} evaluation files in evals/`);\n }\n\n // Laminar should be marked external by passing 'node_modules/*' to the\n // external option. This plugin ensures that @lmnr-ai/lmnr is marked\n // external when imported from path or url.\n // Adapted from https://github.com/evanw/esbuild/issues/619#issuecomment-751769812\n const setLaminarAsExternalPlugin = {\n name: \"set-laminar-as-external\",\n setup(build: esbuild.PluginBuild) {\n build.onResolve({ filter: /^@lmnr-ai\\/lmnr$/ }, (args) => {\n return {\n path: args.path,\n external: true,\n };\n });\n }\n }\n\n for (const file of files) {\n logger.info(`Loading ${file}...`);\n const buildOptions = {\n bundle: true,\n platform: \"node\" as esbuild.Platform,\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n external: [\n \"node_modules/*\",\n \"playwright\",\n \"puppeteer\",\n \"puppeteer-core\",\n \"playwright-core\",\n \"fsevents\",\n ],\n treeShaking: true,\n plugins: [setLaminarAsExternalPlugin],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n logger.error(\"Error when building: No output files found \" +\n \"it is likely that all eval files are not valid TypeScript or JavaScript files.\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluation = loadModule({\n filename: args.file,\n moduleText: outputFileText,\n });\n\n if (!evaluation?.run) {\n logger.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n await evaluation.run();\n\n // FIXME: Now every evaluation file creates a new tracer provider.\n // Attempt to re-initialize it in the same process breaks it.\n // For now, we disable all APIs after running each file, but ideally\n // we should keep a global tracer provider that is initialized once\n // here in the CLI.\n context.disable();\n trace.disable();\n propagation.disable();\n }\n },\n });\n\n parser.set_defaults({\n func: () => {\n parser.print_help();\n process.exit(0);\n },\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli().catch((err) => {\n logger.error(err instanceof Error ? err.message : err);\n throw err;\n});\n","{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.6.4\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup\",\n \"test\": \"tsx --test test/*.test.ts\",\n \"lint\": \"eslint\",\n \"lint:fix\": \"eslint --fix\"\n },\n \"files\": [\n \"dist\",\n \"assets\"\n ],\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.39.0\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.806.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^1.14.0\",\n \"@eslint/eslintrc\": \"^3.3.1\",\n \"@eslint/js\": \"^9.26.0\",\n \"@google-cloud/aiplatform\": \"^4.1.0\",\n \"@google-cloud/vertexai\": \"^1.10.0\",\n \"@langchain/core\": \"^0.3.55\",\n \"@pinecone-database/pinecone\": \"^5.1.2\",\n \"@playwright/test\": \"^1.52.0\",\n \"@qdrant/js-client-rest\": \"^1.14.0\",\n \"@stylistic/eslint-plugin-js\": \"^4.2.0\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.15.17\",\n \"@types/semver\": \"^7.7.0\",\n \"@types/uuid\": \"^10.0.0\",\n \"chromadb\": \"^2.4.3\",\n \"cohere-ai\": \"^7.17.1\",\n \"eslint-plugin-simple-import-sort\": \"^12.1.1\",\n \"eslint-plugin-unused-imports\": \"^4.1.4\",\n \"langchain\": \"^0.3.24\",\n \"llamaindex\": \"^0.9.19\",\n \"nock\": \"^14.0.4\",\n \"openai\": \"^4.98.0\",\n \"playwright\": \"^1.52.0\",\n \"puppeteer\": \"^24.8.2\",\n \"puppeteer-core\": \"^24.8.2\",\n \"together-ai\": \"^0.14.0\",\n \"tsup\": \"^8.4.0\",\n \"tsx\": \"^4.19.4\",\n \"typescript\": \"^5.8.3\",\n \"typescript-eslint\": \"^8.32.0\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.13.3\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/context-async-hooks\": \"^1.30.1\",\n \"@opentelemetry/core\": \"^1.30.1\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.2\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.2\",\n \"@opentelemetry/instrumentation\": \"^0.57.2\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/resources\": \"^1.30.1\",\n \"@opentelemetry/sdk-node\": \"^0.57.2\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.1\",\n \"@opentelemetry/sdk-trace-base-v2\": \"npm:@opentelemetry/sdk-trace-base@2.0.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.33.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.12.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.12.0\",\n \"@traceloop/instrumentation-azure\": \"^0.12.0\",\n \"@traceloop/instrumentation-bedrock\": \"^0.12.0\",\n \"@traceloop/instrumentation-chromadb\": \"^0.12.0\",\n \"@traceloop/instrumentation-cohere\": \"^0.12.0\",\n \"@traceloop/instrumentation-langchain\": \"^0.12.0\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.12.0\",\n \"@traceloop/instrumentation-openai\": \"^0.12.0\",\n \"@traceloop/instrumentation-pinecone\": \"^0.12.0\",\n \"@traceloop/instrumentation-qdrant\": \"^0.12.0\",\n \"@traceloop/instrumentation-together\": \"^0.12.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.12.0\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.25.4\",\n \"glob\": \"^11.0.2\",\n \"pino\": \"^9.6.0\",\n \"pino-pretty\": \"^13.0.0\",\n \"uuid\": \"^11.1.0\",\n \"zod\": \"^3.24.4\"\n }\n}","import { SpanContext, TraceFlags } from '@opentelemetry/api';\nimport pino, { Level } from 'pino';\nimport pinoPretty from 'pino-pretty';\nimport { v4 as uuidv4 } from 'uuid';\nimport { fileURLToPath } from \"url\";\nimport path from \"path\";\n\nimport { LaminarSpanContext } from './types';\n\nexport function initializeLogger(options?: { colorize?: boolean, level?: Level }) {\n const colorize = options?.colorize ?? true;\n const level = options?.level\n ?? (process.env.LMNR_LOG_LEVEL?.toLowerCase()?.trim() as Level)\n ?? 'info';\n\n return pino(pinoPretty({\n colorize,\n minimumLevel: level,\n }));\n}\n\nconst logger = initializeLogger();\n\nexport type StringUUID = `${string}-${string}-${string}-${string}-${string}`;\n\nexport const isStringUUID = (id: string): id is StringUUID =>\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id);\n\nexport const NIL_UUID: StringUUID = '00000000-0000-0000-0000-000000000000';\n\nexport const newUUID = (): StringUUID => {\n // crypto.randomUUID is available in most of the modern browsers and node,\n // but is not available in \"insecure\" contexts, e.g. not https, not localhost\n // so we fallback to uuidv4 in those cases, which is less secure, but works\n // just fine.\n if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {\n return crypto.randomUUID();\n } else {\n return uuidv4() as `${string}-${string}-${string}-${string}-${string}`;\n }\n};\n\nexport const otelSpanIdToUUID = (spanId: string): string => {\n let id = spanId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 16) {\n logger.warn(`Span ID ${spanId} is not 16 hex chars long. ` +\n 'This is not a valid OpenTelemetry span ID.');\n }\n\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Span ID ${spanId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.padStart(32, '0').replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, '$1-$2-$3-$4-$5');\n};\n\nexport const otelTraceIdToUUID = (traceId: string): string => {\n let id = traceId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 32) {\n logger.warn(`Trace ID ${traceId} is not 32 hex chars long. ` +\n 'This is not a valid OpenTelemetry trace ID.');\n }\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Trace ID ${traceId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, '$1-$2-$3-$4-$5');\n};\n\nexport const uuidToOtelTraceId = (uuid: string): string => uuid.replace(/-/g, '');\nexport const uuidToOtelSpanId = (uuid: string): string => uuid.replace(/-/g, '').slice(16);\n\n/**\n * This is a simple implementation of a semaphore to replicate\n * the behavior of the `asyncio.Semaphore` in Python.\n */\nexport class Semaphore {\n /**\n * Number of permits available.\n */\n private _value: number;\n /**\n * List of promises that will be resolved when a permit becomes available.\n */\n private _waiters: ((...args: any[]) => any)[] = [];\n\n constructor(value = 1) {\n if (value < 0) {\n throw new Error(\"Semaphore value must be >= 0\");\n }\n this._value = value;\n this._waiters = [];\n }\n\n async acquire() {\n if (this._value > 0) {\n this._value--;\n return;\n }\n\n // Create a promise that will be resolved when a permit becomes available\n return new Promise(resolve => {\n this._waiters.push(resolve);\n });\n }\n\n release() {\n if (this._waiters.length > 0) {\n // If there are waiters, wake up the first one\n const resolve = this._waiters.shift();\n resolve?.();\n } else {\n this._value++;\n }\n }\n\n // Python-like context manager functionality\n async using<T>(fn: (...args: any[]) => Promise<T>) {\n try {\n await this.acquire();\n return await fn();\n } finally {\n this.release();\n }\n }\n}\n\nexport const tryToOtelSpanContext = (\n spanContext: LaminarSpanContext | Record<string, unknown> | string | SpanContext,\n): SpanContext => {\n if (typeof spanContext === 'string') {\n try {\n const record = JSON.parse(spanContext) as Record<string, unknown>;\n return recordToOtelSpanContext(record);\n } catch (e) {\n throw new Error(`Failed to parse span context ${spanContext}. ` +\n 'The string must be a json representation of a LaminarSpanContext.'\n + `Error: ${e instanceof Error ? e.message : String(e)}`);\n }\n } else if (isRecord(spanContext)) {\n // This covers the `LaminarSpanContext` case too.\n return recordToOtelSpanContext(spanContext);\n } else if (typeof spanContext.traceId === 'string'\n && typeof spanContext.spanId === 'string'\n && spanContext.traceId.length === 32\n && spanContext.spanId.length === 16) {\n logger.warn('The span context is already an OpenTelemetry SpanContext. ' +\n 'Returning it as is. Please use `LaminarSpanContext` objects instead.');\n return spanContext;\n }\n else {\n throw new Error(`Invalid span context ${JSON.stringify(spanContext)}. ` +\n 'Must be a LaminarSpanContext or its json representation.');\n }\n};\n\nconst recordToOtelSpanContext = (record: Record<string, unknown>): SpanContext => {\n if (typeof record.spanId === 'string' && typeof record.traceId === 'string') {\n return {\n spanId: uuidToOtelSpanId(record?.spanId ?? record?.['span_id']),\n traceId: uuidToOtelTraceId(record?.traceId ?? record?.['trace_id']),\n isRemote: record?.isRemote ?? record?.['is_remote'] ?? false,\n traceFlags: record?.traceFlags ?? TraceFlags.SAMPLED,\n } as SpanContext;\n } else {\n throw new Error(`Invalid span context ${JSON.stringify(record)}. ` +\n 'Must be a json representation of a LaminarSpanContext.');\n }\n};\n\nconst isRecord = (value: unknown): value is Record<string, unknown> =>\n typeof value === 'object' && !Array.isArray(value) && value !== null;\n\n\nexport const getDirname = () => {\n if (typeof __dirname !== 'undefined') {\n return __dirname;\n }\n\n if (typeof import.meta?.url !== 'undefined') {\n return path.dirname(fileURLToPath(import.meta.url));\n }\n\n return process.cwd();\n};"],"mappings":";;;;;;;;;AAEA,SAAS,SAAS,aAAa,aAAa;AAC5C,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AACzB,YAAY,UAAU;;;ACHpB,cAAW;;;ACFb,SAAsB,kBAAkB;AACxC,OAAO,UAAqB;AAC5B,OAAO,gBAAgB;AACvB,SAAS,MAAM,cAAc;AAC7B,SAAS,qBAAqB;AAC9B,OAAO,UAAU;AAIV,SAAS,iBAAiB,SAAiD;AAChF,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,QAAQ,SAAS,SACjB,QAAQ,IAAI,gBAAgB,YAAY,GAAG,KAAK,KACjD;AAEL,SAAO,KAAK,WAAW;AAAA,IACrB;AAAA,IACA,cAAc;AAAA,EAChB,CAAC,CAAC;AACJ;AAEA,IAAM,SAAS,iBAAiB;AAmKzB,IAAM,aAAa,MAAM;AAC9B,MAAI,OAAO,cAAc,aAAa;AACpC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,aAAa,QAAQ,aAAa;AAC3C,WAAO,KAAK,QAAQ,cAAc,YAAY,GAAG,CAAC;AAAA,EACpD;AAEA,SAAO,QAAQ,IAAI;AACrB;;;AFtLA,IAAMA,UAAS,iBAAiB;AASzB,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAG8B;AAC5B,aAAW,cAAc;AACzB,aAAW,yBAAyB;AAEpC,QAAM,aAAa;AACnB,QAAMC,aAAY,WAAW;AAG7B,MAAI;AAAA,IACF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACAA;AAAA,EACF;AAGA,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,QAAQ,CAAC;AAErE,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,QAAAF,QAAO,MAAM,qKAEmC;AAChD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACE,MAAK,MAAM;AACd,QAAAF,QAAO,KAAK,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAMA,YAAM,6BAA6B;AAAA,QACjC,MAAM;AAAA,QACN,MAAMG,QAA4B;AAChC,UAAAA,OAAM,UAAU,EAAE,QAAQ,mBAAmB,GAAG,CAACD,UAAS;AACxD,mBAAO;AAAA,cACL,MAAMA,MAAK;AAAA,cACX,UAAU;AAAA,YACZ;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF;AAEA,iBAAW,QAAQ,OAAO;AACxB,QAAAF,QAAO,KAAK,WAAW,IAAI,KAAK;AAChC,cAAM,eAAe;AAAA,UACnB,QAAQ;AAAA,UACR,UAAU;AAAA,UACV,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,UACA,aAAa;AAAA,UACb,SAAS,CAAC,0BAA0B;AAAA,QACtC;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,UAAAA,QAAO,MAAM,2HACqE;AAClF,cAAIE,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,aAAa,WAAW;AAAA,UAC5B,UAAUA,MAAK;AAAA,UACf,YAAY;AAAA,QACd,CAAC;AAED,YAAI,CAAC,YAAY,KAAK;AACpB,UAAAF,QAAO,MAAM,cAAc,IAAI,oCAAoC;AACnE,cAAIE,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,WAAW,IAAI;AAOrB,gBAAQ,QAAQ;AAChB,cAAM,QAAQ;AACd,oBAAY,QAAQ;AAAA,MACtB;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,MAAM;AACV,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI,EAAE,MAAM,CAAC,QAAQ;AACnB,EAAAF,QAAO,MAAM,eAAe,QAAQ,IAAI,UAAU,GAAG;AACrD,QAAM;AACR,CAAC;","names":["logger","__dirname","args","build"]}
1
+ {"version":3,"sources":["../src/cli.ts","../package.json","../src/utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { context, propagation, trace } from \"@opentelemetry/api\";\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as glob from \"glob\";\n\nimport { version } from \"../package.json\";\nimport { Evaluation } from \"./evaluations\";\nimport { getDirname, initializeLogger } from \"./utils\";\n\nconst logger = initializeLogger();\n\ndeclare global {\n // eslint-disable-next-line no-var\n var _evaluations: Evaluation<any, any, any>[] | undefined;\n // eslint-disable-next-line no-var\n var _set_global_evaluation: boolean;\n}\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}): Evaluation<any, any, any>[] {\n globalThis._evaluations = [];\n globalThis._set_global_evaluation = true;\n\n const __filename = filename;\n const __dirname = getDirname();\n\n // add some arguments for proper cjs/esm interop\n\n /* eslint-disable @typescript-eslint/no-implied-eval */\n new Function(\n \"require\",\n \"module\",\n \"__filename\",\n \"__dirname\",\n moduleText,\n )(\n require,\n module,\n __filename,\n __dirname,\n );\n /* eslint-enable @typescript-eslint/no-implied-eval */\n\n // Return the modified _evals global variable\n return globalThis._evaluations!;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"file\", {\n help: \"A file containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n const files = args.file\n ? [args.file]\n : glob.sync('evals/**/*.eval.{ts,js}');\n\n files.sort();\n\n if (files.length === 0) {\n logger.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are eval files that are named like `*.eval.{ts,js}` in\" +\n \"the `evals` directory or its subdirectories.\");\n process.exit(1);\n }\n\n if (!args.file) {\n logger.info(`Located ${files.length} evaluation files in evals/`);\n }\n\n // Laminar should be marked external by passing 'node_modules/*' to the\n // external option. This plugin ensures that @lmnr-ai/lmnr is marked\n // external when imported from path or url.\n // Adapted from https://github.com/evanw/esbuild/issues/619#issuecomment-751769812\n const setLaminarAsExternalPlugin = {\n name: \"set-laminar-as-external\",\n setup(build: esbuild.PluginBuild) {\n build.onResolve({ filter: /^@lmnr-ai\\/lmnr$/ }, (args) => ({\n path: args.path,\n external: true,\n }));\n },\n };\n\n for (const file of files) {\n logger.info(`Loading ${file}...`);\n const buildOptions = {\n bundle: true,\n platform: \"node\" as esbuild.Platform,\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n external: [\n \"node_modules/*\",\n \"playwright\",\n \"puppeteer\",\n \"puppeteer-core\",\n \"playwright-core\",\n \"fsevents\",\n ],\n treeShaking: true,\n plugins: [setLaminarAsExternalPlugin],\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n logger.error(\"Error when building: No output files found \" +\n \"it is likely that all eval files are not valid TypeScript or JavaScript files.\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluations = loadModule({\n filename: file,\n moduleText: outputFileText,\n });\n\n logger.info(`Loaded ${evaluations.length} evaluations from ${file}`);\n\n for (const evaluation of evaluations) {\n if (!evaluation?.run) {\n logger.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n await evaluation.run();\n }\n }\n },\n });\n\n parser.set_defaults({\n func: () => {\n parser.print_help();\n process.exit(0);\n },\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli().catch((err) => {\n logger.error(err instanceof Error ? err.message : err);\n throw err;\n});\n","{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.6.6\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup\",\n \"test\": \"tsx --test test/*.test.ts\",\n \"lint\": \"eslint\",\n \"lint:fix\": \"eslint --fix\"\n },\n \"files\": [\n \"dist\",\n \"assets\"\n ],\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"devDependencies\": {\n \"@anthropic-ai/sdk\": \"^0.39.0\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.817.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^1.14.0\",\n \"@eslint/eslintrc\": \"^3.3.1\",\n \"@eslint/js\": \"^9.27.0\",\n \"@google-cloud/aiplatform\": \"^4.2.0\",\n \"@google-cloud/vertexai\": \"^1.10.0\",\n \"@langchain/core\": \"^0.3.57\",\n \"@pinecone-database/pinecone\": \"^5.1.2\",\n \"@playwright/test\": \"^1.52.0\",\n \"@qdrant/js-client-rest\": \"^1.14.1\",\n \"@stylistic/eslint-plugin-js\": \"^4.4.0\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^22.15.21\",\n \"@types/semver\": \"^7.7.0\",\n \"@types/uuid\": \"^10.0.0\",\n \"chromadb\": \"^2.4.6\",\n \"cohere-ai\": \"^7.17.1\",\n \"eslint-plugin-simple-import-sort\": \"^12.1.1\",\n \"eslint-plugin-unused-imports\": \"^4.1.4\",\n \"langchain\": \"^0.3.27\",\n \"llamaindex\": \"^0.9.19\",\n \"nock\": \"^14.0.4\",\n \"openai\": \"^4.103.0\",\n \"playwright\": \"^1.52.0\",\n \"puppeteer\": \"^24.9.0\",\n \"puppeteer-core\": \"^24.9.0\",\n \"together-ai\": \"^0.14.0\",\n \"tsup\": \"^8.5.0\",\n \"tsx\": \"^4.19.4\",\n \"typescript\": \"^5.8.3\",\n \"typescript-eslint\": \"^8.32.1\"\n },\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.13.4\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/context-async-hooks\": \"^1.30.1\",\n \"@opentelemetry/core\": \"^1.30.1\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.2\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.2\",\n \"@opentelemetry/instrumentation\": \"^0.57.2\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/resources\": \"^1.30.1\",\n \"@opentelemetry/sdk-node\": \"^0.57.2\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.1\",\n \"@opentelemetry/sdk-trace-base-v2\": \"npm:@opentelemetry/sdk-trace-base@2.0.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.34.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.12.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.12.0\",\n \"@traceloop/instrumentation-azure\": \"^0.12.0\",\n \"@traceloop/instrumentation-bedrock\": \"^0.12.0\",\n \"@traceloop/instrumentation-chromadb\": \"^0.12.0\",\n \"@traceloop/instrumentation-cohere\": \"^0.12.0\",\n \"@traceloop/instrumentation-langchain\": \"^0.12.0\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.12.0\",\n \"@traceloop/instrumentation-openai\": \"^0.12.0\",\n \"@traceloop/instrumentation-pinecone\": \"^0.12.0\",\n \"@traceloop/instrumentation-qdrant\": \"^0.12.0\",\n \"@traceloop/instrumentation-together\": \"^0.12.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.12.0\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.25.4\",\n \"glob\": \"^11.0.2\",\n \"pino\": \"^9.7.0\",\n \"pino-pretty\": \"^13.0.0\",\n \"uuid\": \"^11.1.0\",\n \"zod\": \"^3.25.28\"\n }\n}","import { AttributeValue, SpanContext, TraceFlags } from '@opentelemetry/api';\nimport path from \"path\";\nimport pino, { Level } from 'pino';\nimport pinoPretty from 'pino-pretty';\nimport { fileURLToPath } from \"url\";\nimport { v4 as uuidv4 } from 'uuid';\n\nimport { LaminarSpanContext } from './types';\n\nexport function initializeLogger(options?: { colorize?: boolean, level?: Level }) {\n const colorize = options?.colorize ?? true;\n const level = options?.level\n ?? (process.env.LMNR_LOG_LEVEL?.toLowerCase()?.trim() as Level)\n ?? 'info';\n\n return pino(pinoPretty({\n colorize,\n minimumLevel: level,\n }));\n}\n\nconst logger = initializeLogger();\n\nexport type StringUUID = `${string}-${string}-${string}-${string}-${string}`;\n\nexport const isStringUUID = (id: string): id is StringUUID =>\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id);\n\nexport const NIL_UUID: StringUUID = '00000000-0000-0000-0000-000000000000';\n\nexport const newUUID = (): StringUUID => {\n // crypto.randomUUID is available in most of the modern browsers and node,\n // but is not available in \"insecure\" contexts, e.g. not https, not localhost\n // so we fallback to uuidv4 in those cases, which is less secure, but works\n // just fine.\n if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {\n return crypto.randomUUID();\n } else {\n return uuidv4() as `${string}-${string}-${string}-${string}-${string}`;\n }\n};\n\nexport const otelSpanIdToUUID = (spanId: string): string => {\n let id = spanId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 16) {\n logger.warn(`Span ID ${spanId} is not 16 hex chars long. ` +\n 'This is not a valid OpenTelemetry span ID.');\n }\n\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Span ID ${spanId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.padStart(32, '0').replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5'\n );\n};\n\nexport const otelTraceIdToUUID = (traceId: string): string => {\n let id = traceId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 32) {\n logger.warn(`Trace ID ${traceId} is not 32 hex chars long. ` +\n 'This is not a valid OpenTelemetry trace ID.');\n }\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Trace ID ${traceId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5'\n );\n};\n\nexport const uuidToOtelTraceId = (uuid: string): string => uuid.replace(/-/g, '');\nexport const uuidToOtelSpanId = (uuid: string): string => uuid.replace(/-/g, '').slice(16);\n\n/**\n * This is a simple implementation of a semaphore to replicate\n * the behavior of the `asyncio.Semaphore` in Python.\n */\nexport class Semaphore {\n /**\n * Number of permits available.\n */\n private _value: number;\n /**\n * List of promises that will be resolved when a permit becomes available.\n */\n private _waiters: ((...args: any[]) => any)[] = [];\n\n constructor(value = 1) {\n if (value < 0) {\n throw new Error(\"Semaphore value must be >= 0\");\n }\n this._value = value;\n this._waiters = [];\n }\n\n async acquire() {\n if (this._value > 0) {\n this._value--;\n return;\n }\n\n // Create a promise that will be resolved when a permit becomes available\n return new Promise(resolve => {\n this._waiters.push(resolve);\n });\n }\n\n release() {\n if (this._waiters.length > 0) {\n // If there are waiters, wake up the first one\n const resolve = this._waiters.shift();\n resolve?.();\n } else {\n this._value++;\n }\n }\n\n // Python-like context manager functionality\n async using<T>(fn: (...args: any[]) => Promise<T>) {\n try {\n await this.acquire();\n return await fn();\n } finally {\n this.release();\n }\n }\n}\n\nexport const tryToOtelSpanContext = (\n spanContext: LaminarSpanContext | Record<string, unknown> | string | SpanContext,\n): SpanContext => {\n if (typeof spanContext === 'string') {\n try {\n const record = JSON.parse(spanContext) as Record<string, unknown>;\n return recordToOtelSpanContext(record);\n } catch (e) {\n throw new Error(`Failed to parse span context ${spanContext}. ` +\n 'The string must be a json representation of a LaminarSpanContext.'\n + `Error: ${e instanceof Error ? e.message : String(e)}`);\n }\n } else if (isRecord(spanContext)) {\n // This covers the `LaminarSpanContext` case too.\n return recordToOtelSpanContext(spanContext);\n } else if (typeof spanContext.traceId === 'string'\n && typeof spanContext.spanId === 'string'\n && spanContext.traceId.length === 32\n && spanContext.spanId.length === 16) {\n logger.warn('The span context is already an OpenTelemetry SpanContext. ' +\n 'Returning it as is. Please use `LaminarSpanContext` objects instead.');\n return spanContext;\n }\n else {\n throw new Error(`Invalid span context ${JSON.stringify(spanContext)}. ` +\n 'Must be a LaminarSpanContext or its json representation.');\n }\n};\n\nconst recordToOtelSpanContext = (record: Record<string, unknown>): SpanContext => {\n if (typeof record.spanId === 'string' && typeof record.traceId === 'string') {\n return {\n spanId: uuidToOtelSpanId(record?.spanId ?? record?.['span_id']),\n traceId: uuidToOtelTraceId(record?.traceId ?? record?.['trace_id']),\n isRemote: record?.isRemote ?? record?.['is_remote'] ?? false,\n traceFlags: record?.traceFlags ?? TraceFlags.SAMPLED,\n } as SpanContext;\n } else {\n throw new Error(`Invalid span context ${JSON.stringify(record)}. ` +\n 'Must be a json representation of a LaminarSpanContext.');\n }\n};\n\nconst isRecord = (value: unknown): value is Record<string, unknown> =>\n typeof value === 'object' && !Array.isArray(value) && value !== null;\n\n\nexport const getDirname = () => {\n if (typeof __dirname !== 'undefined') {\n return __dirname;\n }\n\n if (typeof import.meta?.url !== 'undefined') {\n return path.dirname(fileURLToPath(import.meta.url));\n }\n\n return process.cwd();\n};\n\nexport const slicePayload = <T>(value: T, length: number) => {\n if (value === null || value === undefined) {\n return value;\n }\n\n const str = JSON.stringify(value);\n if (str.length <= length) {\n return value;\n }\n\n return (str.slice(0, length) + '...');\n};\n\nexport const isOtelAttributeValueType = (value: unknown): value is AttributeValue => {\n if (typeof value === 'string'\n || typeof value === 'number'\n || typeof value === 'boolean') {\n return true;\n }\n\n if (Array.isArray(value)) {\n const allStrings = value.every(value => (value == null) || typeof value === 'string');\n const allNumbers = value.every(value => (value == null) || typeof value === 'number');\n const allBooleans = value.every(value => (value == null) || typeof value === 'boolean');\n return allStrings || allNumbers || allBooleans;\n }\n return false;\n};\n"],"mappings":";;;;;;;;;AAGA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AACzB,YAAY,UAAU;;;ACHpB,cAAW;;;ACFb,SAAsC,kBAAkB;AACxD,OAAO,UAAU;AACjB,OAAO,UAAqB;AAC5B,OAAO,gBAAgB;AACvB,SAAS,qBAAqB;AAC9B,SAAS,MAAM,cAAc;AAItB,SAAS,iBAAiB,SAAiD;AAChF,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,QAAQ,SAAS,SACjB,QAAQ,IAAI,gBAAgB,YAAY,GAAG,KAAK,KACjD;AAEL,SAAO,KAAK,WAAW;AAAA,IACrB;AAAA,IACA,cAAc;AAAA,EAChB,CAAC,CAAC;AACJ;AAEA,IAAM,SAAS,iBAAiB;AAyKzB,IAAM,aAAa,MAAM;AAC9B,MAAI,OAAO,cAAc,aAAa;AACpC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,aAAa,QAAQ,aAAa;AAC3C,WAAO,KAAK,QAAQ,cAAc,YAAY,GAAG,CAAC;AAAA,EACpD;AAEA,SAAO,QAAQ,IAAI;AACrB;;;AF7LA,IAAMA,UAAS,iBAAiB;AASzB,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGgC;AAC9B,aAAW,eAAe,CAAC;AAC3B,aAAW,yBAAyB;AAEpC,QAAM,aAAa;AACnB,QAAMC,aAAY,WAAW;AAK7B,MAAI;AAAA,IACF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACAA;AAAA,EACF;AAIA,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,QAAQ,CAAC;AAErE,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,QAAQ;AAAA,IAC9B,MAAM;AAAA,IAEN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,YAAM,QAAQA,MAAK,OACf,CAACA,MAAK,IAAI,IACL,UAAK,yBAAyB;AAEvC,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,QAAAF,QAAO,MAAM,qKAEmC;AAChD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAACE,MAAK,MAAM;AACd,QAAAF,QAAO,KAAK,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE;AAMA,YAAM,6BAA6B;AAAA,QACjC,MAAM;AAAA,QACN,MAAMG,QAA4B;AAChC,UAAAA,OAAM,UAAU,EAAE,QAAQ,mBAAmB,GAAG,CAACD,WAAU;AAAA,YACzD,MAAMA,MAAK;AAAA,YACX,UAAU;AAAA,UACZ,EAAE;AAAA,QACJ;AAAA,MACF;AAEA,iBAAW,QAAQ,OAAO;AACxB,QAAAF,QAAO,KAAK,WAAW,IAAI,KAAK;AAChC,cAAM,eAAe;AAAA,UACnB,QAAQ;AAAA,UACR,UAAU;AAAA,UACV,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,UACF;AAAA,UACA,aAAa;AAAA,UACb,SAAS,CAAC,0BAA0B;AAAA,QACtC;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,UAAAA,QAAO,MAAM,2HACqE;AAClF,cAAIE,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,cAAc,WAAW;AAAA,UAC7B,UAAU;AAAA,UACV,YAAY;AAAA,QACd,CAAC;AAED,QAAAF,QAAO,KAAK,UAAU,YAAY,MAAM,qBAAqB,IAAI,EAAE;AAEnE,mBAAW,cAAc,aAAa;AACpC,cAAI,CAAC,YAAY,KAAK;AACpB,YAAAA,QAAO,MAAM,cAAc,IAAI,oCAAoC;AACnE,gBAAIE,MAAK,eAAe;AACtB,sBAAQ,KAAK,CAAC;AAAA,YAChB;AACA;AAAA,UACF;AAEA,gBAAM,WAAW,IAAI;AAAA,QACvB;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,MAAM;AACV,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI,EAAE,MAAM,CAAC,QAAQ;AACnB,EAAAF,QAAO,MAAM,eAAe,QAAQ,IAAI,UAAU,GAAG;AACrD,QAAM;AACR,CAAC;","names":["logger","__dirname","args","build"]}
package/dist/{evaluations-DQIsUKAP.d.mts → evaluations-De0Vbq1A.d.mts} RENAMED
@@ -55,6 +55,7 @@ type EvaluationDatapoint<D, T, O> = {
55
55
  id: StringUUID;
56
56
  data: D;
57
57
  target?: T;
58
+ metadata?: Record<string, any>;
58
59
  executorOutput?: O;
59
60
  scores?: Record<string, number>;
60
61
  traceId: string;
@@ -378,12 +379,53 @@ declare class EvalsResource extends BaseResource {
378
379
  }): Promise<GetDatapointsResponse<D, T>>;
379
380
  }
380
381
 
382
+ /** Resource for tagging traces. */
383
+
384
+ declare class TagsResource extends BaseResource {
385
+ /** Resource for tagging traces. */
386
+ constructor(baseHttpUrl: string, projectApiKey: string);
387
+ /**
388
+ * Tag a trace with a list of tags. Note that the trace must be ended before
389
+ * tagging it. You may want to call `await Laminar.flush()` after the trace
390
+ * that you want to tag.
391
+ *
392
+ * @param {string | StringUUID} trace_id - The trace id to tag.
393
+ * @param {string[] | string} tags - The tag or list of tags to add to the trace.
394
+ * @returns {Promise<any>} The response from the server.
395
+ * @example
396
+ * ```javascript
397
+ * import { Laminar, observe, LaminarClient } from "@lmnr-ai/lmnr";
398
+ * Laminar.initialize();
399
+ * const client = new LaminarClient();
400
+ * let traceId: StringUUID | null = null;
401
+ * // Make sure this is called outside of traced context.
402
+ * await observe(
403
+ * {
404
+ * name: "my-trace",
405
+ * },
406
+ * async () => {
407
+ * traceId = await Laminar.getTraceId();
408
+ * await foo();
409
+ * },
410
+ * );
411
+ *
412
+ * // or make sure the trace is ended by this point.
413
+ * await Laminar.flush();
414
+ * if (traceId) {
415
+ * await client.tags.tag(traceId, ["tag1", "tag2"]);
416
+ * }
417
+ * ```
418
+ */
419
+ tag(trace_id: string | StringUUID, tags: string[] | string): Promise<any>;
420
+ }
421
+
381
422
  declare class LaminarClient {
382
423
  private baseUrl;
383
424
  private projectApiKey;
384
425
  private _agent;
385
426
  private _browserEvents;
386
427
  private _evals;
428
+ private _tags;
387
429
  constructor({ baseUrl, projectApiKey, port, }: {
388
430
  baseUrl?: string;
389
431
  projectApiKey?: string;
@@ -392,6 +434,7 @@ declare class LaminarClient {
392
434
  get agent(): AgentResource;
393
435
  get browserEvents(): BrowserEventsResource;
394
436
  get evals(): EvalsResource;
437
+ get tags(): TagsResource;
395
438
  }
396
439
 
397
440
  declare abstract class EvaluationDataset<D, T> {
@@ -709,7 +752,7 @@ interface InitializeOptions {
709
752
  }
710
753
 
711
754
  declare global {
712
- var _evaluation: Evaluation<any, any, any> | undefined;
755
+ var _evaluations: Evaluation<any, any, any>[] | undefined;
713
756
  var _set_global_evaluation: boolean;
714
757
  }
715
758
  /**
@@ -777,6 +820,10 @@ type Datapoint<D, T> = {
777
820
  * Must be json serializable.
778
821
  */
779
822
  target?: T;
823
+ /**
824
+ * metadata to the evaluator function. Must be json serializable.
825
+ */
826
+ metadata?: Record<string, any>;
780
827
  };
781
828
  type EvaluatorFunctionReturn = number | Record<string, number>;
782
829
  /**
@@ -867,4 +914,4 @@ declare class Evaluation<D, T, O> {
867
914
  */
868
915
  declare function evaluate<D, T, O>({ data, executor, evaluators, groupName, name, config, }: EvaluationConstructorProps<D, T, O>): Promise<void>;
869
916
 
870
- export { type ChatMessage as C, type Datapoint as D, Evaluation as E, HumanEvaluator as H, type InitializeOptions as I, type LaminarSpanContext as L, type NodeInput as N, type PipelineRunRequest as P, type TraceType as T, TracingLevel as a, LaminarClient as b, EvaluationDataset as c, LaminarDataset as d, evaluate as e, type Event as f, type PipelineRunResponse as g };
917
+ export { type ChatMessage as C, type Datapoint as D, Evaluation as E, HumanEvaluator as H, type InitializeOptions as I, type LaminarSpanContext as L, type NodeInput as N, type PipelineRunRequest as P, type StringUUID as S, type TraceType as T, TracingLevel as a, LaminarClient as b, EvaluationDataset as c, LaminarDataset as d, evaluate as e, type Event as f, type PipelineRunResponse as g };
package/dist/{evaluations-DQIsUKAP.d.ts → evaluations-De0Vbq1A.d.ts} RENAMED
@@ -55,6 +55,7 @@ type EvaluationDatapoint<D, T, O> = {
55
55
  id: StringUUID;
56
56
  data: D;
57
57
  target?: T;
58
+ metadata?: Record<string, any>;
58
59
  executorOutput?: O;
59
60
  scores?: Record<string, number>;
60
61
  traceId: string;
@@ -378,12 +379,53 @@ declare class EvalsResource extends BaseResource {
378
379
  }): Promise<GetDatapointsResponse<D, T>>;
379
380
  }
380
381
 
382
+ /** Resource for tagging traces. */
383
+
384
+ declare class TagsResource extends BaseResource {
385
+ /** Resource for tagging traces. */
386
+ constructor(baseHttpUrl: string, projectApiKey: string);
387
+ /**
388
+ * Tag a trace with a list of tags. Note that the trace must be ended before
389
+ * tagging it. You may want to call `await Laminar.flush()` after the trace
390
+ * that you want to tag.
391
+ *
392
+ * @param {string | StringUUID} trace_id - The trace id to tag.
393
+ * @param {string[] | string} tags - The tag or list of tags to add to the trace.
394
+ * @returns {Promise<any>} The response from the server.
395
+ * @example
396
+ * ```javascript
397
+ * import { Laminar, observe, LaminarClient } from "@lmnr-ai/lmnr";
398
+ * Laminar.initialize();
399
+ * const client = new LaminarClient();
400
+ * let traceId: StringUUID | null = null;
401
+ * // Make sure this is called outside of traced context.
402
+ * await observe(
403
+ * {
404
+ * name: "my-trace",
405
+ * },
406
+ * async () => {
407
+ * traceId = await Laminar.getTraceId();
408
+ * await foo();
409
+ * },
410
+ * );
411
+ *
412
+ * // or make sure the trace is ended by this point.
413
+ * await Laminar.flush();
414
+ * if (traceId) {
415
+ * await client.tags.tag(traceId, ["tag1", "tag2"]);
416
+ * }
417
+ * ```
418
+ */
419
+ tag(trace_id: string | StringUUID, tags: string[] | string): Promise<any>;
420
+ }
421
+
381
422
  declare class LaminarClient {
382
423
  private baseUrl;
383
424
  private projectApiKey;
384
425
  private _agent;
385
426
  private _browserEvents;
386
427
  private _evals;
428
+ private _tags;
387
429
  constructor({ baseUrl, projectApiKey, port, }: {
388
430
  baseUrl?: string;
389
431
  projectApiKey?: string;
@@ -392,6 +434,7 @@ declare class LaminarClient {
392
434
  get agent(): AgentResource;
393
435
  get browserEvents(): BrowserEventsResource;
394
436
  get evals(): EvalsResource;
437
+ get tags(): TagsResource;
395
438
  }
396
439
 
397
440
  declare abstract class EvaluationDataset<D, T> {
@@ -709,7 +752,7 @@ interface InitializeOptions {
709
752
  }
710
753
 
711
754
  declare global {
712
- var _evaluation: Evaluation<any, any, any> | undefined;
755
+ var _evaluations: Evaluation<any, any, any>[] | undefined;
713
756
  var _set_global_evaluation: boolean;
714
757
  }
715
758
  /**
@@ -777,6 +820,10 @@ type Datapoint<D, T> = {
777
820
  * Must be json serializable.
778
821
  */
779
822
  target?: T;
823
+ /**
824
+ * metadata to the evaluator function. Must be json serializable.
825
+ */
826
+ metadata?: Record<string, any>;
780
827
  };
781
828
  type EvaluatorFunctionReturn = number | Record<string, number>;
782
829
  /**
@@ -867,4 +914,4 @@ declare class Evaluation<D, T, O> {
867
914
  */
868
915
  declare function evaluate<D, T, O>({ data, executor, evaluators, groupName, name, config, }: EvaluationConstructorProps<D, T, O>): Promise<void>;
869
916
 
870
- export { type ChatMessage as C, type Datapoint as D, Evaluation as E, HumanEvaluator as H, type InitializeOptions as I, type LaminarSpanContext as L, type NodeInput as N, type PipelineRunRequest as P, type TraceType as T, TracingLevel as a, LaminarClient as b, EvaluationDataset as c, LaminarDataset as d, evaluate as e, type Event as f, type PipelineRunResponse as g };
917
+ export { type ChatMessage as C, type Datapoint as D, Evaluation as E, HumanEvaluator as H, type InitializeOptions as I, type LaminarSpanContext as L, type NodeInput as N, type PipelineRunRequest as P, type StringUUID as S, type TraceType as T, TracingLevel as a, LaminarClient as b, EvaluationDataset as c, LaminarDataset as d, evaluate as e, type Event as f, type PipelineRunResponse as g };