@lmnr-ai/lmnr 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1 -1
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +1 -1
- package/dist/cli.mjs.map +1 -1
- package/dist/index.js +75 -48
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +77 -50
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/cli.ts","../package.json","../src/utils.ts","../src/opentelemetry-lib/tracing/attributes.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as fs from \"fs\";\nimport * as glob from \"glob\";\n\nimport { version } from \"../package.json\";\nimport { Evaluation } from \"./evaluations\";\nimport { getDirname, initializeLogger } from \"./utils\";\n\nconst logger = initializeLogger();\n\n// esbuild plugin to skip dynamic imports\nconst createSkipDynamicImportsPlugin = (skipModules: string[]): esbuild.Plugin => ({\n name: 'skip-dynamic-imports',\n setup(build) {\n if (!skipModules || skipModules.length === 0) return;\n\n build.onResolve({ filter: /.*/ }, (args) => {\n // Only handle dynamic imports\n if (args.kind === 'dynamic-import' && skipModules.includes(args.path)) {\n logger.warn(`Skipping dynamic import: ${args.path}`);\n // Return a virtual module that exports an empty object\n return {\n path: args.path,\n namespace: 'lmnr-skip-dynamic-import',\n };\n }\n });\n\n // Provide empty module content for skipped dynamic imports\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n build.onLoad({ filter: /.*/, namespace: 'lmnr-skip-dynamic-import' }, (args) => ({\n contents: 'export default {};',\n loader: 'js',\n }));\n },\n});\n\ndeclare global {\n var _evaluations: Evaluation<any, any, any>[] | undefined;\n var _set_global_evaluation: boolean;\n}\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}): Evaluation<any, any, any>[] {\n globalThis._evaluations = [];\n globalThis._set_global_evaluation = true;\n\n const __filename = filename;\n const __dirname = getDirname();\n\n // add some arguments for proper cjs/esm interop\n\n /* eslint-disable @typescript-eslint/no-implied-eval */\n new Function(\n \"require\",\n \"module\",\n \"__filename\",\n \"__dirname\",\n moduleText,\n )(\n require,\n module,\n __filename,\n __dirname,\n );\n /* eslint-enable @typescript-eslint/no-implied-eval */\n\n // Return the modified _evals global variable\n return globalThis._evaluations;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"files\", {\n help: \"A file or files containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory. \" +\n \"If multiple files are provided, the evaluation will run each file in order.\",\n nargs: \"*\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.add_argument(\"--output-file\", {\n help: \"Output file to write the results to. Outputs are written in JSON format.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--external-packages\", {\n help: \"[ADVANCED] List of packages to pass as external to esbuild. This will not link \" +\n \"the packages directly into the eval file, but will instead require them at runtime. \" +\n \"Read more: https://esbuild.github.io/api/#external\",\n nargs: \"*\",\n });\n\n parserEval.add_argument(\"--dynamic-imports-to-skip\", {\n help: \"[ADVANCED] List of module names to skip when encountered as dynamic imports. \" +\n \"These dynamic imports will resolve to an empty module to prevent build failures. \" +\n \"This is meant to skip the imports that are not used in the evaluation itself.\",\n nargs: \"*\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n let files: string[];\n if (args.files && args.files.length > 0) {\n files = args.files.flatMap((file: string) => glob.sync(file));\n } else {\n // No files provided, use default pattern\n files = glob.sync('evals/**/*.eval.{ts,js}');\n }\n\n files.sort();\n\n if (files.length === 0) {\n logger.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are eval files that are named like `*.eval.{ts,js}` in\" +\n \"the `evals` directory or its subdirectories.\");\n process.exit(1);\n }\n\n if (args.files.length === 0) {\n logger.info(`Located ${files.length} evaluation files in evals/`);\n } else {\n logger.info(`Running ${files.length} evaluation files.`);\n }\n\n const scores: { file: string, scores: Record<string, number> }[] = [];\n\n for (const file of files) {\n logger.info(`Loading ${file}...`);\n const buildOptions = {\n bundle: true,\n platform: \"node\" as esbuild.Platform,\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n external: [\n \"node_modules/*\",\n \"playwright\",\n \"puppeteer\",\n \"puppeteer-core\",\n \"playwright-core\",\n \"fsevents\",\n ...(args.external_packages ? args.external_packages : []),\n ],\n plugins: [\n createSkipDynamicImportsPlugin(args.dynamic_imports_to_skip || []),\n ],\n treeShaking: true,\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n logger.error(\"Error when building: No output files found \" +\n \"it is likely that all eval files are not valid TypeScript or JavaScript files.\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluations = loadModule({\n filename: file,\n moduleText: outputFileText,\n });\n\n logger.info(`Loaded ${evaluations.length} evaluations from ${file}`);\n\n for (const evaluation of evaluations) {\n if (!evaluation?.run) {\n logger.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const evalScores = await evaluation.run();\n scores.push({ file, scores: evalScores });\n }\n }\n\n if (args.output_file) {\n fs.writeFileSync(args.output_file, JSON.stringify(scores, null, 2));\n }\n },\n });\n\n parser.set_defaults({\n func: () => {\n parser.print_help();\n process.exit(0);\n },\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli().catch((err) => {\n logger.error(err instanceof Error ? err.message : err);\n throw err;\n});\n","{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.7.0\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup\",\n \"test\": \"tsx --test test/*.test.ts\",\n \"lint\": \"eslint\",\n \"lint:fix\": \"eslint --fix\"\n },\n \"files\": [\n \"dist\"\n ],\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.13.4\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/context-async-hooks\": \"^1.30.1\",\n \"@opentelemetry/core\": \"^1.30.1\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.2\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.2\",\n \"@opentelemetry/instrumentation\": \"^0.57.2\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/resources\": \"^1.30.1\",\n \"@opentelemetry/sdk-node\": \"^0.57.2\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.1\",\n \"@opentelemetry/sdk-trace-base-v2\": \"npm:@opentelemetry/sdk-trace-base@2.0.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.36.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.12.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.12.0\",\n \"@traceloop/instrumentation-azure\": \"^0.12.0\",\n \"@traceloop/instrumentation-bedrock\": \"^0.12.0\",\n \"@traceloop/instrumentation-chromadb\": \"^0.12.0\",\n \"@traceloop/instrumentation-cohere\": \"^0.12.0\",\n \"@traceloop/instrumentation-langchain\": \"^0.12.0\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.12.0\",\n \"@traceloop/instrumentation-openai\": \"^0.12.0\",\n \"@traceloop/instrumentation-pinecone\": \"^0.12.0\",\n \"@traceloop/instrumentation-qdrant\": \"^0.12.0\",\n \"@traceloop/instrumentation-together\": \"^0.12.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.12.0\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.25.8\",\n \"glob\": \"^11.0.3\",\n \"pino\": \"^9.7.0\",\n \"pino-pretty\": \"^13.1.1\",\n \"uuid\": \"^11.1.0\",\n \"zod\": \"^3.25.76\"\n },\n \"devDependencies\": {\n \"@ai-sdk/openai\": \"^2.0.22\",\n \"@anthropic-ai/sdk\": \"^0.57.0\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.859.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^2.4.2\",\n \"@eslint/eslintrc\": \"^3.3.1\",\n \"@eslint/js\": \"^9.32.0\",\n \"@google-cloud/aiplatform\": \"^5.1.0\",\n \"@google-cloud/vertexai\": \"^1.10.0\",\n \"@langchain/core\": \"^0.3.66\",\n \"@pinecone-database/pinecone\": \"^5.1.2\",\n \"@playwright/test\": \"^1.54.2\",\n \"@qdrant/js-client-rest\": \"^1.15.0\",\n \"@stylistic/eslint-plugin\": \"^5.2.2\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^24.2.0\",\n \"@types/semver\": \"^7.7.0\",\n \"@types/uuid\": \"^10.0.0\",\n \"ai\": \"^5.0.26\",\n \"chromadb\": \"^3.0.10\",\n \"cohere-ai\": \"^7.18.0\",\n \"eslint\": \"^9.32.0\",\n \"eslint-plugin-simple-import-sort\": \"^12.1.1\",\n \"eslint-plugin-unused-imports\": \"^4.1.4\",\n \"langchain\": \"^0.3.30\",\n \"llamaindex\": \"^0.11.25\",\n \"nock\": \"^14.0.8\",\n \"openai\": \"^5.11.0\",\n \"playwright\": \"^1.54.2\",\n \"puppeteer\": \"^24.15.0\",\n \"puppeteer-core\": \"^24.15.0\",\n \"together-ai\": \"^0.21.0\",\n \"tsup\": \"^8.5.0\",\n \"tsx\": \"^4.20.3\",\n \"typescript\": \"^5.8.3\",\n \"typescript-eslint\": \"^8.38.0\"\n }\n}","import { AttributeValue, SpanContext, TraceFlags } from '@opentelemetry/api';\nimport * as path from \"path\";\nimport pino, { Level } from 'pino';\nimport { PinoPretty } from 'pino-pretty';\nimport { fileURLToPath } from \"url\";\nimport { v4 as uuidv4 } from 'uuid';\n\nimport { ASSOCIATION_PROPERTIES } from './opentelemetry-lib/tracing/attributes';\nimport { LaminarSpanContext } from './types';\n\nexport function initializeLogger(options?: { colorize?: boolean, level?: Level }) {\n const colorize = options?.colorize ?? true;\n const level = options?.level\n ?? (process.env.LMNR_LOG_LEVEL?.toLowerCase()?.trim() as Level)\n ?? 'info';\n\n return pino(PinoPretty({\n colorize,\n minimumLevel: level,\n }));\n}\n\nconst logger = initializeLogger();\n\nexport type StringUUID = `${string}-${string}-${string}-${string}-${string}`;\n\nexport const isStringUUID = (id: string): id is StringUUID =>\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id);\n\nexport const NIL_UUID: StringUUID = '00000000-0000-0000-0000-000000000000';\n\nexport const newUUID = (): StringUUID => {\n // crypto.randomUUID is available in most of the modern browsers and node,\n // but is not available in \"insecure\" contexts, e.g. not https, not localhost\n // so we fallback to uuidv4 in those cases, which is less secure, but works\n // just fine.\n if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {\n return crypto.randomUUID();\n } else {\n return uuidv4() as `${string}-${string}-${string}-${string}-${string}`;\n }\n};\n\nexport const otelSpanIdToUUID = (spanId: string): string => {\n let id = spanId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 16) {\n logger.warn(`Span ID ${spanId} is not 16 hex chars long. ` +\n 'This is not a valid OpenTelemetry span ID.');\n }\n\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Span ID ${spanId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.padStart(32, '0').replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5',\n );\n};\n\nexport const otelTraceIdToUUID = (traceId: string): StringUUID => {\n let id = traceId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 32) {\n logger.warn(`Trace ID ${traceId} is not 32 hex chars long. ` +\n 'This is not a valid OpenTelemetry trace ID.');\n }\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Trace ID ${traceId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5',\n ) as StringUUID;\n};\n\nexport const uuidToOtelTraceId = (uuid: string): string => uuid.replace(/-/g, '');\nexport const uuidToOtelSpanId = (uuid: string): string => uuid.replace(/-/g, '').slice(16);\n\n/**\n * This is a simple implementation of a semaphore to replicate\n * the behavior of the `asyncio.Semaphore` in Python.\n */\nexport class Semaphore {\n /**\n * Number of permits available.\n */\n private _value: number;\n /**\n * List of promises that will be resolved when a permit becomes available.\n */\n private _waiters: ((...args: any[]) => any)[] = [];\n\n constructor(value = 1) {\n if (value < 0) {\n throw new Error(\"Semaphore value must be >= 0\");\n }\n this._value = value;\n this._waiters = [];\n }\n\n async acquire() {\n if (this._value > 0) {\n this._value--;\n return;\n }\n\n // Create a promise that will be resolved when a permit becomes available\n return new Promise(resolve => {\n this._waiters.push(resolve);\n });\n }\n\n release() {\n if (this._waiters.length > 0) {\n // If there are waiters, wake up the first one\n const resolve = this._waiters.shift();\n resolve?.();\n } else {\n this._value++;\n }\n }\n\n // Python-like context manager functionality\n async using<T>(fn: (...args: any[]) => Promise<T>) {\n try {\n await this.acquire();\n return await fn();\n } finally {\n this.release();\n }\n }\n}\n\nexport const tryToOtelSpanContext = (\n spanContext: LaminarSpanContext | Record<string, unknown> | string | SpanContext,\n): SpanContext => {\n if (typeof spanContext === 'string') {\n try {\n const record = JSON.parse(spanContext) as Record<string, unknown>;\n return recordToOtelSpanContext(record);\n } catch (e) {\n throw new Error(`Failed to parse span context ${spanContext}. ` +\n 'The string must be a json representation of a LaminarSpanContext.'\n + `Error: ${e instanceof Error ? e.message : String(e)}`);\n }\n } else if (isRecord(spanContext)) {\n // This covers the `LaminarSpanContext` case too.\n return recordToOtelSpanContext(spanContext);\n } else if (typeof spanContext.traceId === 'string'\n && typeof spanContext.spanId === 'string'\n && spanContext.traceId.length === 32\n && spanContext.spanId.length === 16) {\n logger.warn('The span context is already an OpenTelemetry SpanContext. ' +\n 'Returning it as is. Please use `LaminarSpanContext` objects instead.');\n return spanContext;\n }\n else {\n throw new Error(`Invalid span context ${JSON.stringify(spanContext)}. ` +\n 'Must be a LaminarSpanContext or its json representation.');\n }\n};\n\nconst recordToOtelSpanContext = (record: Record<string, unknown>): SpanContext => {\n if ((typeof record.spanId === 'string' && typeof record.traceId === 'string') ||\n (typeof record.span_id === 'string' && typeof record.trace_id === 'string')) {\n return {\n spanId: uuidToOtelSpanId(record?.spanId as string ?? record?.['span_id'] as string),\n traceId: uuidToOtelTraceId(record?.traceId as string ?? record?.['trace_id'] as string),\n isRemote: record?.isRemote ?? record?.['is_remote'] ?? false,\n traceFlags: record?.traceFlags ?? TraceFlags.SAMPLED,\n } as SpanContext;\n } else {\n throw new Error(`Invalid span context ${JSON.stringify(record)}. ` +\n 'Must be a json representation of a LaminarSpanContext.');\n }\n};\n\nconst isRecord = (value: unknown): value is Record<string, unknown> =>\n typeof value === 'object' && !Array.isArray(value) && value !== null;\n\n\nexport const getDirname = () => {\n if (typeof __dirname !== 'undefined') {\n return __dirname;\n }\n\n if (typeof import.meta?.url !== 'undefined') {\n return path.dirname(fileURLToPath(import.meta.url));\n }\n\n return process.cwd();\n};\n\nexport const slicePayload = <T>(value: T, length: number) => {\n if (value === null || value === undefined) {\n return value;\n }\n\n const str = JSON.stringify(value);\n if (str.length <= length) {\n return value;\n }\n\n return (str.slice(0, length) + '...');\n};\n\nexport const isOtelAttributeValueType = (value: unknown): value is AttributeValue => {\n if (typeof value === 'string'\n || typeof value === 'number'\n || typeof value === 'boolean') {\n return true;\n }\n\n if (Array.isArray(value)) {\n const allStrings = value.every(value => (value == null) || typeof value === 'string');\n const allNumbers = value.every(value => (value == null) || typeof value === 'number');\n const allBooleans = value.every(value => (value == null) || typeof value === 'boolean');\n return allStrings || allNumbers || allBooleans;\n }\n return false;\n};\n\nexport const metadataToAttributes = (\n metadata: Record<string, unknown>,\n): Record<string, AttributeValue> => Object.fromEntries(\n Object.entries(metadata).map(([key, value]) => {\n if (isOtelAttributeValueType(value)) {\n return [`${ASSOCIATION_PROPERTIES}.metadata.${key}`, value];\n } else {\n return [`${ASSOCIATION_PROPERTIES}.metadata.${key}`, JSON.stringify(value)];\n }\n }),\n);\n","import { SpanAttributes } from '@traceloop/ai-semantic-conventions';\n\nexport const SPAN_INPUT = \"lmnr.span.input\";\nexport const SPAN_OUTPUT = \"lmnr.span.output\";\nexport const SPAN_TYPE = \"lmnr.span.type\";\nexport const SPAN_PATH = \"lmnr.span.path\";\nexport const SPAN_IDS_PATH = \"lmnr.span.ids_path\";\nexport const PARENT_SPAN_PATH = \"lmnr.span.parent_path\";\nexport const PARENT_SPAN_IDS_PATH = \"lmnr.span.parent_ids_path\";\nexport const SPAN_INSTRUMENTATION_SOURCE = \"lmnr.span.instrumentation_source\";\nexport const SPAN_SDK_VERSION = \"lmnr.span.sdk_version\";\nexport const SPAN_LANGUAGE_VERSION = \"lmnr.span.language_version\";\nexport const OVERRIDE_PARENT_SPAN = \"lmnr.internal.override_parent_span\";\nexport const TRACE_HAS_BROWSER_SESSION = \"lmnr.internal.has_browser_session\";\nexport const EXTRACTED_FROM_NEXT_JS = \"lmnr.span.extracted_from.next_js\";\nexport const HUMAN_EVALUATOR_OPTIONS = 'lmnr.span.human_evaluator_options';\nexport const ASSOCIATION_PROPERTIES = \"lmnr.association.properties\";\nexport const SESSION_ID = \"lmnr.association.properties.session_id\";\nexport const USER_ID = \"lmnr.association.properties.user_id\";\nexport const TRACE_TYPE = \"lmnr.association.properties.trace_type\";\n\nexport const ASSOCIATION_PROPERTIES_OVERRIDES: Record<string, string> = {\n \"span_type\": SPAN_TYPE,\n};\n\nexport const LaminarAttributes = {\n // == This is the minimum set of attributes for a proper LLM span ==\n //\n // not SpanAttributes.LLM_USAGE_PROMPT_TOKENS\n INPUT_TOKEN_COUNT: \"gen_ai.usage.input_tokens\",\n // not SpanAttributes.LLM_USAGE_COMPLETION_TOKENS\n OUTPUT_TOKEN_COUNT: \"gen_ai.usage.output_tokens\",\n TOTAL_TOKEN_COUNT: SpanAttributes.LLM_USAGE_TOTAL_TOKENS,\n PROVIDER: SpanAttributes.LLM_SYSTEM,\n REQUEST_MODEL: SpanAttributes.LLM_REQUEST_MODEL,\n RESPONSE_MODEL: SpanAttributes.LLM_RESPONSE_MODEL,\n //\n // == End of minimum set ==\n // == Additional attributes ==\n //\n INPUT_COST: \"gen_ai.usage.input_cost\",\n OUTPUT_COST: \"gen_ai.usage.output_cost\",\n TOTAL_COST: \"gen_ai.usage.cost\",\n //\n // == End of additional attributes ==\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAEA,sBAA+B;AAC/B,cAAyB;AACzB,SAAoB;AACpB,WAAsB;;;ACHpB,cAAW;;;ACFb,iBAAwD;AACxD,WAAsB;AACtB,kBAA4B;AAC5B,yBAA2B;AAC3B,iBAA8B;AAC9B,kBAA6B;;;ACL7B,qCAA+B;AAyBxB,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA,EAI/B,mBAAmB;AAAA;AAAA,EAEnB,oBAAoB;AAAA,EACpB,mBAAmB,8CAAe;AAAA,EAClC,UAAU,8CAAe;AAAA,EACzB,eAAe,8CAAe;AAAA,EAC9B,gBAAgB,8CAAe;AAAA;AAAA;AAAA;AAAA;AAAA,EAK/B,YAAY;AAAA,EACZ,aAAa;AAAA,EACb,YAAY;AAAA;AAAA;AAGd;;;AD7CA;AAUO,SAAS,iBAAiB,SAAiD;AAChF,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,QAAQ,SAAS,SACjB,QAAQ,IAAI,gBAAgB,YAAY,GAAG,KAAK,KACjD;AAEL,aAAO,YAAAA,aAAK,+BAAW;AAAA,IACrB;AAAA,IACA,cAAc;AAAA,EAChB,CAAC,CAAC;AACJ;AAEA,IAAM,SAAS,iBAAiB;AA0KzB,IAAM,aAAa,MAAM;AAC9B,MAAI,OAAO,cAAc,aAAa;AACpC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,aAAa,QAAQ,aAAa;AAC3C,WAAY,iBAAQ,0BAAc,YAAY,GAAG,CAAC;AAAA,EACpD;AAEA,SAAO,QAAQ,IAAI;AACrB;;;AF/LA,IAAMC,UAAS,iBAAiB;AAGhC,IAAM,iCAAiC,CAAC,iBAA2C;AAAA,EACjF,MAAM;AAAA,EACN,MAAMC,QAAO;AACX,QAAI,CAAC,eAAe,YAAY,WAAW,EAAG;AAE9C,IAAAA,OAAM,UAAU,EAAE,QAAQ,KAAK,GAAG,CAAC,SAAS;AAE1C,UAAI,KAAK,SAAS,oBAAoB,YAAY,SAAS,KAAK,IAAI,GAAG;AACrE,QAAAD,QAAO,KAAK,4BAA4B,KAAK,IAAI,EAAE;AAEnD,eAAO;AAAA,UACL,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,QACb;AAAA,MACF;AAAA,IACF,CAAC;AAID,IAAAC,OAAM,OAAO,EAAE,QAAQ,MAAM,WAAW,2BAA2B,GAAG,CAAC,UAAU;AAAA,MAC/E,UAAU;AAAA,MACV,QAAQ;AAAA,IACV,EAAE;AAAA,EACJ;AACF;AAOO,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGgC;AAC9B,aAAW,eAAe,CAAC;AAC3B,aAAW,yBAAyB;AAEpC,QAAM,aAAa;AACnB,QAAMC,aAAY,WAAW;AAK7B,MAAI;AAAA,IACF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACAA;AAAA,EACF;AAIA,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,QAAQ,CAAC;AAErE,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,SAAS;AAAA,IAC/B,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa,iBAAiB;AAAA,IACvC,MAAM;AAAA,IACN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,uBAAuB;AAAA,IAC7C,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,6BAA6B;AAAA,IACnD,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,UAAI;AACJ,UAAIA,MAAK,SAASA,MAAK,MAAM,SAAS,GAAG;AACvC,gBAAQA,MAAK,MAAM,QAAQ,CAAC,SAAsB,UAAK,IAAI,CAAC;AAAA,MAC9D,OAAO;AAEL,gBAAa,UAAK,yBAAyB;AAAA,MAC7C;AAEA,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,QAAAH,QAAO,MAAM,qKAEmC;AAChD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAIG,MAAK,MAAM,WAAW,GAAG;AAC3B,QAAAH,QAAO,KAAK,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE,OAAO;AACL,QAAAA,QAAO,KAAK,WAAW,MAAM,MAAM,oBAAoB;AAAA,MACzD;AAEA,YAAM,SAA6D,CAAC;AAEpE,iBAAW,QAAQ,OAAO;AACxB,QAAAA,QAAO,KAAK,WAAW,IAAI,KAAK;AAChC,cAAM,eAAe;AAAA,UACnB,QAAQ;AAAA,UACR,UAAU;AAAA,UACV,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,GAAIG,MAAK,oBAAoBA,MAAK,oBAAoB,CAAC;AAAA,UACzD;AAAA,UACA,SAAS;AAAA,YACP,+BAA+BA,MAAK,2BAA2B,CAAC,CAAC;AAAA,UACnE;AAAA,UACA,aAAa;AAAA,QACf;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,UAAAH,QAAO,MAAM,2HACqE;AAClF,cAAIG,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,cAAc,WAAW;AAAA,UAC7B,UAAU;AAAA,UACV,YAAY;AAAA,QACd,CAAC;AAED,QAAAH,QAAO,KAAK,UAAU,YAAY,MAAM,qBAAqB,IAAI,EAAE;AAEnE,mBAAW,cAAc,aAAa;AACpC,cAAI,CAAC,YAAY,KAAK;AACpB,YAAAA,QAAO,MAAM,cAAc,IAAI,oCAAoC;AACnE,gBAAIG,MAAK,eAAe;AACtB,sBAAQ,KAAK,CAAC;AAAA,YAChB;AACA;AAAA,UACF;AAEA,gBAAM,aAAa,MAAM,WAAW,IAAI;AACxC,iBAAO,KAAK,EAAE,MAAM,QAAQ,WAAW,CAAC;AAAA,QAC1C;AAAA,MACF;AAEA,UAAIA,MAAK,aAAa;AACpB,QAAG,iBAAcA,MAAK,aAAa,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,MACpE;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,MAAM;AACV,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI,EAAE,MAAM,CAAC,QAAQ;AACnB,EAAAH,QAAO,MAAM,eAAe,QAAQ,IAAI,UAAU,GAAG;AACrD,QAAM;AACR,CAAC;","names":["pino","logger","build","__dirname","args"]}
|
|
1
|
+
{"version":3,"sources":["../src/cli.ts","../package.json","../src/utils.ts","../src/opentelemetry-lib/tracing/attributes.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as fs from \"fs\";\nimport * as glob from \"glob\";\n\nimport { version } from \"../package.json\";\nimport { Evaluation } from \"./evaluations\";\nimport { getDirname, initializeLogger } from \"./utils\";\n\nconst logger = initializeLogger();\n\n// esbuild plugin to skip dynamic imports\nconst createSkipDynamicImportsPlugin = (skipModules: string[]): esbuild.Plugin => ({\n name: 'skip-dynamic-imports',\n setup(build) {\n if (!skipModules || skipModules.length === 0) return;\n\n build.onResolve({ filter: /.*/ }, (args) => {\n // Only handle dynamic imports\n if (args.kind === 'dynamic-import' && skipModules.includes(args.path)) {\n logger.warn(`Skipping dynamic import: ${args.path}`);\n // Return a virtual module that exports an empty object\n return {\n path: args.path,\n namespace: 'lmnr-skip-dynamic-import',\n };\n }\n });\n\n // Provide empty module content for skipped dynamic imports\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n build.onLoad({ filter: /.*/, namespace: 'lmnr-skip-dynamic-import' }, (args) => ({\n contents: 'export default {};',\n loader: 'js',\n }));\n },\n});\n\ndeclare global {\n var _evaluations: Evaluation<any, any, any>[] | undefined;\n var _set_global_evaluation: boolean;\n}\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}): Evaluation<any, any, any>[] {\n globalThis._evaluations = [];\n globalThis._set_global_evaluation = true;\n\n const __filename = filename;\n const __dirname = getDirname();\n\n // add some arguments for proper cjs/esm interop\n\n /* eslint-disable @typescript-eslint/no-implied-eval */\n new Function(\n \"require\",\n \"module\",\n \"__filename\",\n \"__dirname\",\n moduleText,\n )(\n require,\n module,\n __filename,\n __dirname,\n );\n /* eslint-enable @typescript-eslint/no-implied-eval */\n\n // Return the modified _evals global variable\n return globalThis._evaluations;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"files\", {\n help: \"A file or files containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory. \" +\n \"If multiple files are provided, the evaluation will run each file in order.\",\n nargs: \"*\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.add_argument(\"--output-file\", {\n help: \"Output file to write the results to. Outputs are written in JSON format.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--external-packages\", {\n help: \"[ADVANCED] List of packages to pass as external to esbuild. This will not link \" +\n \"the packages directly into the eval file, but will instead require them at runtime. \" +\n \"Read more: https://esbuild.github.io/api/#external\",\n nargs: \"*\",\n });\n\n parserEval.add_argument(\"--dynamic-imports-to-skip\", {\n help: \"[ADVANCED] List of module names to skip when encountered as dynamic imports. \" +\n \"These dynamic imports will resolve to an empty module to prevent build failures. \" +\n \"This is meant to skip the imports that are not used in the evaluation itself.\",\n nargs: \"*\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n let files: string[];\n if (args.files && args.files.length > 0) {\n files = args.files.flatMap((file: string) => glob.sync(file));\n } else {\n // No files provided, use default pattern\n files = glob.sync('evals/**/*.eval.{ts,js}');\n }\n\n files.sort();\n\n if (files.length === 0) {\n logger.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are eval files that are named like `*.eval.{ts,js}` in\" +\n \"the `evals` directory or its subdirectories.\");\n process.exit(1);\n }\n\n if (args.files.length === 0) {\n logger.info(`Located ${files.length} evaluation files in evals/`);\n } else {\n logger.info(`Running ${files.length} evaluation files.`);\n }\n\n const scores: { file: string, scores: Record<string, number> }[] = [];\n\n for (const file of files) {\n logger.info(`Loading ${file}...`);\n const buildOptions = {\n bundle: true,\n platform: \"node\" as esbuild.Platform,\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n external: [\n \"node_modules/*\",\n \"playwright\",\n \"puppeteer\",\n \"puppeteer-core\",\n \"playwright-core\",\n \"fsevents\",\n ...(args.external_packages ? args.external_packages : []),\n ],\n plugins: [\n createSkipDynamicImportsPlugin(args.dynamic_imports_to_skip || []),\n ],\n treeShaking: true,\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n logger.error(\"Error when building: No output files found \" +\n \"it is likely that all eval files are not valid TypeScript or JavaScript files.\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluations = loadModule({\n filename: file,\n moduleText: outputFileText,\n });\n\n logger.info(`Loaded ${evaluations.length} evaluations from ${file}`);\n\n for (const evaluation of evaluations) {\n if (!evaluation?.run) {\n logger.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const evalScores = await evaluation.run();\n scores.push({ file, scores: evalScores });\n }\n }\n\n if (args.output_file) {\n fs.writeFileSync(args.output_file, JSON.stringify(scores, null, 2));\n }\n },\n });\n\n parser.set_defaults({\n func: () => {\n parser.print_help();\n process.exit(0);\n },\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli().catch((err) => {\n logger.error(err instanceof Error ? err.message : err);\n throw err;\n});\n","{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.7.2\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup\",\n \"test\": \"tsx --test test/*.test.ts\",\n \"lint\": \"eslint\",\n \"lint:fix\": \"eslint --fix\"\n },\n \"files\": [\n \"dist\"\n ],\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.13.4\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/context-async-hooks\": \"^1.30.1\",\n \"@opentelemetry/core\": \"^1.30.1\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.2\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.2\",\n \"@opentelemetry/instrumentation\": \"^0.57.2\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/resources\": \"^1.30.1\",\n \"@opentelemetry/sdk-node\": \"^0.57.2\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.1\",\n \"@opentelemetry/sdk-trace-base-v2\": \"npm:@opentelemetry/sdk-trace-base@2.0.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.36.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.12.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.12.0\",\n \"@traceloop/instrumentation-azure\": \"^0.12.0\",\n \"@traceloop/instrumentation-bedrock\": \"^0.12.0\",\n \"@traceloop/instrumentation-chromadb\": \"^0.12.0\",\n \"@traceloop/instrumentation-cohere\": \"^0.12.0\",\n \"@traceloop/instrumentation-langchain\": \"^0.12.0\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.12.0\",\n \"@traceloop/instrumentation-openai\": \"^0.12.0\",\n \"@traceloop/instrumentation-pinecone\": \"^0.12.0\",\n \"@traceloop/instrumentation-qdrant\": \"^0.12.0\",\n \"@traceloop/instrumentation-together\": \"^0.12.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.12.0\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.25.8\",\n \"glob\": \"^11.0.3\",\n \"pino\": \"^9.7.0\",\n \"pino-pretty\": \"^13.1.1\",\n \"uuid\": \"^11.1.0\",\n \"zod\": \"^3.25.76\"\n },\n \"devDependencies\": {\n \"@ai-sdk/openai\": \"^2.0.22\",\n \"@anthropic-ai/sdk\": \"^0.57.0\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.859.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^2.4.2\",\n \"@eslint/eslintrc\": \"^3.3.1\",\n \"@eslint/js\": \"^9.32.0\",\n \"@google-cloud/aiplatform\": \"^5.1.0\",\n \"@google-cloud/vertexai\": \"^1.10.0\",\n \"@langchain/core\": \"^0.3.66\",\n \"@pinecone-database/pinecone\": \"^5.1.2\",\n \"@playwright/test\": \"^1.54.2\",\n \"@qdrant/js-client-rest\": \"^1.15.0\",\n \"@stylistic/eslint-plugin\": \"^5.2.2\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^24.2.0\",\n \"@types/semver\": \"^7.7.0\",\n \"@types/uuid\": \"^10.0.0\",\n \"ai\": \"^5.0.26\",\n \"chromadb\": \"^3.0.10\",\n \"cohere-ai\": \"^7.18.0\",\n \"eslint\": \"^9.32.0\",\n \"eslint-plugin-simple-import-sort\": \"^12.1.1\",\n \"eslint-plugin-unused-imports\": \"^4.1.4\",\n \"langchain\": \"^0.3.30\",\n \"llamaindex\": \"^0.11.25\",\n \"nock\": \"^14.0.8\",\n \"openai\": \"^5.11.0\",\n \"playwright\": \"^1.54.2\",\n \"puppeteer\": \"^24.15.0\",\n \"puppeteer-core\": \"^24.15.0\",\n \"together-ai\": \"^0.21.0\",\n \"tsup\": \"^8.5.0\",\n \"tsx\": \"^4.20.3\",\n \"typescript\": \"^5.8.3\",\n \"typescript-eslint\": \"^8.38.0\"\n }\n}","import { AttributeValue, SpanContext, TraceFlags } from '@opentelemetry/api';\nimport * as path from \"path\";\nimport pino, { Level } from 'pino';\nimport { PinoPretty } from 'pino-pretty';\nimport { fileURLToPath } from \"url\";\nimport { v4 as uuidv4 } from 'uuid';\n\nimport { ASSOCIATION_PROPERTIES } from './opentelemetry-lib/tracing/attributes';\nimport { LaminarSpanContext } from './types';\n\nexport function initializeLogger(options?: { colorize?: boolean, level?: Level }) {\n const colorize = options?.colorize ?? true;\n const level = options?.level\n ?? (process.env.LMNR_LOG_LEVEL?.toLowerCase()?.trim() as Level)\n ?? 'info';\n\n return pino(PinoPretty({\n colorize,\n minimumLevel: level,\n }));\n}\n\nconst logger = initializeLogger();\n\nexport type StringUUID = `${string}-${string}-${string}-${string}-${string}`;\n\nexport const isStringUUID = (id: string): id is StringUUID =>\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id);\n\nexport const NIL_UUID: StringUUID = '00000000-0000-0000-0000-000000000000';\n\nexport const newUUID = (): StringUUID => {\n // crypto.randomUUID is available in most of the modern browsers and node,\n // but is not available in \"insecure\" contexts, e.g. not https, not localhost\n // so we fallback to uuidv4 in those cases, which is less secure, but works\n // just fine.\n if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {\n return crypto.randomUUID();\n } else {\n return uuidv4() as `${string}-${string}-${string}-${string}-${string}`;\n }\n};\n\nexport const otelSpanIdToUUID = (spanId: string): string => {\n let id = spanId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 16) {\n logger.warn(`Span ID ${spanId} is not 16 hex chars long. ` +\n 'This is not a valid OpenTelemetry span ID.');\n }\n\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Span ID ${spanId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.padStart(32, '0').replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5',\n );\n};\n\nexport const otelTraceIdToUUID = (traceId: string): StringUUID => {\n let id = traceId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 32) {\n logger.warn(`Trace ID ${traceId} is not 32 hex chars long. ` +\n 'This is not a valid OpenTelemetry trace ID.');\n }\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Trace ID ${traceId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5',\n ) as StringUUID;\n};\n\nexport const uuidToOtelTraceId = (uuid: string): string => uuid.replace(/-/g, '');\nexport const uuidToOtelSpanId = (uuid: string): string => uuid.replace(/-/g, '').slice(16);\n\n/**\n * This is a simple implementation of a semaphore to replicate\n * the behavior of the `asyncio.Semaphore` in Python.\n */\nexport class Semaphore {\n /**\n * Number of permits available.\n */\n private _value: number;\n /**\n * List of promises that will be resolved when a permit becomes available.\n */\n private _waiters: ((...args: any[]) => any)[] = [];\n\n constructor(value = 1) {\n if (value < 0) {\n throw new Error(\"Semaphore value must be >= 0\");\n }\n this._value = value;\n this._waiters = [];\n }\n\n async acquire() {\n if (this._value > 0) {\n this._value--;\n return;\n }\n\n // Create a promise that will be resolved when a permit becomes available\n return new Promise(resolve => {\n this._waiters.push(resolve);\n });\n }\n\n release() {\n if (this._waiters.length > 0) {\n // If there are waiters, wake up the first one\n const resolve = this._waiters.shift();\n resolve?.();\n } else {\n this._value++;\n }\n }\n\n // Python-like context manager functionality\n async using<T>(fn: (...args: any[]) => Promise<T>) {\n try {\n await this.acquire();\n return await fn();\n } finally {\n this.release();\n }\n }\n}\n\nexport const tryToOtelSpanContext = (\n spanContext: LaminarSpanContext | Record<string, unknown> | string | SpanContext,\n): SpanContext => {\n if (typeof spanContext === 'string') {\n try {\n const record = JSON.parse(spanContext) as Record<string, unknown>;\n return recordToOtelSpanContext(record);\n } catch (e) {\n throw new Error(`Failed to parse span context ${spanContext}. ` +\n 'The string must be a json representation of a LaminarSpanContext.'\n + `Error: ${e instanceof Error ? e.message : String(e)}`);\n }\n } else if (isRecord(spanContext)) {\n // This covers the `LaminarSpanContext` case too.\n return recordToOtelSpanContext(spanContext);\n } else if (typeof spanContext.traceId === 'string'\n && typeof spanContext.spanId === 'string'\n && spanContext.traceId.length === 32\n && spanContext.spanId.length === 16) {\n logger.warn('The span context is already an OpenTelemetry SpanContext. ' +\n 'Returning it as is. Please use `LaminarSpanContext` objects instead.');\n return spanContext;\n }\n else {\n throw new Error(`Invalid span context ${JSON.stringify(spanContext)}. ` +\n 'Must be a LaminarSpanContext or its json representation.');\n }\n};\n\nconst recordToOtelSpanContext = (record: Record<string, unknown>): SpanContext => {\n if ((typeof record.spanId === 'string' && typeof record.traceId === 'string') ||\n (typeof record.span_id === 'string' && typeof record.trace_id === 'string')) {\n return {\n spanId: uuidToOtelSpanId(record?.spanId as string ?? record?.['span_id'] as string),\n traceId: uuidToOtelTraceId(record?.traceId as string ?? record?.['trace_id'] as string),\n isRemote: record?.isRemote ?? record?.['is_remote'] ?? false,\n traceFlags: record?.traceFlags ?? TraceFlags.SAMPLED,\n } as SpanContext;\n } else {\n throw new Error(`Invalid span context ${JSON.stringify(record)}. ` +\n 'Must be a json representation of a LaminarSpanContext.');\n }\n};\n\nconst isRecord = (value: unknown): value is Record<string, unknown> =>\n typeof value === 'object' && !Array.isArray(value) && value !== null;\n\n\nexport const getDirname = () => {\n if (typeof __dirname !== 'undefined') {\n return __dirname;\n }\n\n if (typeof import.meta?.url !== 'undefined') {\n return path.dirname(fileURLToPath(import.meta.url));\n }\n\n return process.cwd();\n};\n\nexport const slicePayload = <T>(value: T, length: number) => {\n if (value === null || value === undefined) {\n return value;\n }\n\n const str = JSON.stringify(value);\n if (str.length <= length) {\n return value;\n }\n\n return (str.slice(0, length) + '...');\n};\n\nexport const isOtelAttributeValueType = (value: unknown): value is AttributeValue => {\n if (typeof value === 'string'\n || typeof value === 'number'\n || typeof value === 'boolean') {\n return true;\n }\n\n if (Array.isArray(value)) {\n const allStrings = value.every(value => (value == null) || typeof value === 'string');\n const allNumbers = value.every(value => (value == null) || typeof value === 'number');\n const allBooleans = value.every(value => (value == null) || typeof value === 'boolean');\n return allStrings || allNumbers || allBooleans;\n }\n return false;\n};\n\nexport const metadataToAttributes = (\n metadata: Record<string, unknown>,\n): Record<string, AttributeValue> => Object.fromEntries(\n Object.entries(metadata).map(([key, value]) => {\n if (isOtelAttributeValueType(value)) {\n return [`${ASSOCIATION_PROPERTIES}.metadata.${key}`, value];\n } else {\n return [`${ASSOCIATION_PROPERTIES}.metadata.${key}`, JSON.stringify(value)];\n }\n }),\n);\n","import { SpanAttributes } from '@traceloop/ai-semantic-conventions';\n\nexport const SPAN_INPUT = \"lmnr.span.input\";\nexport const SPAN_OUTPUT = \"lmnr.span.output\";\nexport const SPAN_TYPE = \"lmnr.span.type\";\nexport const SPAN_PATH = \"lmnr.span.path\";\nexport const SPAN_IDS_PATH = \"lmnr.span.ids_path\";\nexport const PARENT_SPAN_PATH = \"lmnr.span.parent_path\";\nexport const PARENT_SPAN_IDS_PATH = \"lmnr.span.parent_ids_path\";\nexport const SPAN_INSTRUMENTATION_SOURCE = \"lmnr.span.instrumentation_source\";\nexport const SPAN_SDK_VERSION = \"lmnr.span.sdk_version\";\nexport const SPAN_LANGUAGE_VERSION = \"lmnr.span.language_version\";\nexport const OVERRIDE_PARENT_SPAN = \"lmnr.internal.override_parent_span\";\nexport const TRACE_HAS_BROWSER_SESSION = \"lmnr.internal.has_browser_session\";\nexport const EXTRACTED_FROM_NEXT_JS = \"lmnr.span.extracted_from.next_js\";\nexport const HUMAN_EVALUATOR_OPTIONS = 'lmnr.span.human_evaluator_options';\nexport const ASSOCIATION_PROPERTIES = \"lmnr.association.properties\";\nexport const SESSION_ID = \"lmnr.association.properties.session_id\";\nexport const USER_ID = \"lmnr.association.properties.user_id\";\nexport const TRACE_TYPE = \"lmnr.association.properties.trace_type\";\n\nexport const ASSOCIATION_PROPERTIES_OVERRIDES: Record<string, string> = {\n \"span_type\": SPAN_TYPE,\n};\n\nexport const LaminarAttributes = {\n // == This is the minimum set of attributes for a proper LLM span ==\n //\n // not SpanAttributes.LLM_USAGE_PROMPT_TOKENS\n INPUT_TOKEN_COUNT: \"gen_ai.usage.input_tokens\",\n // not SpanAttributes.LLM_USAGE_COMPLETION_TOKENS\n OUTPUT_TOKEN_COUNT: \"gen_ai.usage.output_tokens\",\n TOTAL_TOKEN_COUNT: SpanAttributes.LLM_USAGE_TOTAL_TOKENS,\n PROVIDER: SpanAttributes.LLM_SYSTEM,\n REQUEST_MODEL: SpanAttributes.LLM_REQUEST_MODEL,\n RESPONSE_MODEL: SpanAttributes.LLM_RESPONSE_MODEL,\n //\n // == End of minimum set ==\n // == Additional attributes ==\n //\n INPUT_COST: \"gen_ai.usage.input_cost\",\n OUTPUT_COST: \"gen_ai.usage.output_cost\",\n TOTAL_COST: \"gen_ai.usage.cost\",\n //\n // == End of additional attributes ==\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAEA,sBAA+B;AAC/B,cAAyB;AACzB,SAAoB;AACpB,WAAsB;;;ACHpB,cAAW;;;ACFb,iBAAwD;AACxD,WAAsB;AACtB,kBAA4B;AAC5B,yBAA2B;AAC3B,iBAA8B;AAC9B,kBAA6B;;;ACL7B,qCAA+B;AAyBxB,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA,EAI/B,mBAAmB;AAAA;AAAA,EAEnB,oBAAoB;AAAA,EACpB,mBAAmB,8CAAe;AAAA,EAClC,UAAU,8CAAe;AAAA,EACzB,eAAe,8CAAe;AAAA,EAC9B,gBAAgB,8CAAe;AAAA;AAAA;AAAA;AAAA;AAAA,EAK/B,YAAY;AAAA,EACZ,aAAa;AAAA,EACb,YAAY;AAAA;AAAA;AAGd;;;AD7CA;AAUO,SAAS,iBAAiB,SAAiD;AAChF,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,QAAQ,SAAS,SACjB,QAAQ,IAAI,gBAAgB,YAAY,GAAG,KAAK,KACjD;AAEL,aAAO,YAAAA,aAAK,+BAAW;AAAA,IACrB;AAAA,IACA,cAAc;AAAA,EAChB,CAAC,CAAC;AACJ;AAEA,IAAM,SAAS,iBAAiB;AA0KzB,IAAM,aAAa,MAAM;AAC9B,MAAI,OAAO,cAAc,aAAa;AACpC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,aAAa,QAAQ,aAAa;AAC3C,WAAY,iBAAQ,0BAAc,YAAY,GAAG,CAAC;AAAA,EACpD;AAEA,SAAO,QAAQ,IAAI;AACrB;;;AF/LA,IAAMC,UAAS,iBAAiB;AAGhC,IAAM,iCAAiC,CAAC,iBAA2C;AAAA,EACjF,MAAM;AAAA,EACN,MAAMC,QAAO;AACX,QAAI,CAAC,eAAe,YAAY,WAAW,EAAG;AAE9C,IAAAA,OAAM,UAAU,EAAE,QAAQ,KAAK,GAAG,CAAC,SAAS;AAE1C,UAAI,KAAK,SAAS,oBAAoB,YAAY,SAAS,KAAK,IAAI,GAAG;AACrE,QAAAD,QAAO,KAAK,4BAA4B,KAAK,IAAI,EAAE;AAEnD,eAAO;AAAA,UACL,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,QACb;AAAA,MACF;AAAA,IACF,CAAC;AAID,IAAAC,OAAM,OAAO,EAAE,QAAQ,MAAM,WAAW,2BAA2B,GAAG,CAAC,UAAU;AAAA,MAC/E,UAAU;AAAA,MACV,QAAQ;AAAA,IACV,EAAE;AAAA,EACJ;AACF;AAOO,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGgC;AAC9B,aAAW,eAAe,CAAC;AAC3B,aAAW,yBAAyB;AAEpC,QAAM,aAAa;AACnB,QAAMC,aAAY,WAAW;AAK7B,MAAI;AAAA,IACF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACAA;AAAA,EACF;AAIA,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,+BAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,QAAQ,CAAC;AAErE,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,SAAS;AAAA,IAC/B,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa,iBAAiB;AAAA,IACvC,MAAM;AAAA,IACN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,uBAAuB;AAAA,IAC7C,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,6BAA6B;AAAA,IACnD,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,UAAI;AACJ,UAAIA,MAAK,SAASA,MAAK,MAAM,SAAS,GAAG;AACvC,gBAAQA,MAAK,MAAM,QAAQ,CAAC,SAAsB,UAAK,IAAI,CAAC;AAAA,MAC9D,OAAO;AAEL,gBAAa,UAAK,yBAAyB;AAAA,MAC7C;AAEA,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,QAAAH,QAAO,MAAM,qKAEmC;AAChD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAIG,MAAK,MAAM,WAAW,GAAG;AAC3B,QAAAH,QAAO,KAAK,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE,OAAO;AACL,QAAAA,QAAO,KAAK,WAAW,MAAM,MAAM,oBAAoB;AAAA,MACzD;AAEA,YAAM,SAA6D,CAAC;AAEpE,iBAAW,QAAQ,OAAO;AACxB,QAAAA,QAAO,KAAK,WAAW,IAAI,KAAK;AAChC,cAAM,eAAe;AAAA,UACnB,QAAQ;AAAA,UACR,UAAU;AAAA,UACV,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,GAAIG,MAAK,oBAAoBA,MAAK,oBAAoB,CAAC;AAAA,UACzD;AAAA,UACA,SAAS;AAAA,YACP,+BAA+BA,MAAK,2BAA2B,CAAC,CAAC;AAAA,UACnE;AAAA,UACA,aAAa;AAAA,QACf;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,UAAAH,QAAO,MAAM,2HACqE;AAClF,cAAIG,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,cAAc,WAAW;AAAA,UAC7B,UAAU;AAAA,UACV,YAAY;AAAA,QACd,CAAC;AAED,QAAAH,QAAO,KAAK,UAAU,YAAY,MAAM,qBAAqB,IAAI,EAAE;AAEnE,mBAAW,cAAc,aAAa;AACpC,cAAI,CAAC,YAAY,KAAK;AACpB,YAAAA,QAAO,MAAM,cAAc,IAAI,oCAAoC;AACnE,gBAAIG,MAAK,eAAe;AACtB,sBAAQ,KAAK,CAAC;AAAA,YAChB;AACA;AAAA,UACF;AAEA,gBAAM,aAAa,MAAM,WAAW,IAAI;AACxC,iBAAO,KAAK,EAAE,MAAM,QAAQ,WAAW,CAAC;AAAA,QAC1C;AAAA,MACF;AAEA,UAAIA,MAAK,aAAa;AACpB,QAAG,iBAAcA,MAAK,aAAa,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,MACpE;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,MAAM;AACV,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI,EAAE,MAAM,CAAC,QAAQ;AACnB,EAAAH,QAAO,MAAM,eAAe,QAAQ,IAAI,UAAU,GAAG;AACrD,QAAM;AACR,CAAC;","names":["pino","logger","build","__dirname","args"]}
|
package/dist/cli.mjs
CHANGED
package/dist/cli.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/cli.ts","../package.json","../src/utils.ts","../src/opentelemetry-lib/tracing/attributes.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as fs from \"fs\";\nimport * as glob from \"glob\";\n\nimport { version } from \"../package.json\";\nimport { Evaluation } from \"./evaluations\";\nimport { getDirname, initializeLogger } from \"./utils\";\n\nconst logger = initializeLogger();\n\n// esbuild plugin to skip dynamic imports\nconst createSkipDynamicImportsPlugin = (skipModules: string[]): esbuild.Plugin => ({\n name: 'skip-dynamic-imports',\n setup(build) {\n if (!skipModules || skipModules.length === 0) return;\n\n build.onResolve({ filter: /.*/ }, (args) => {\n // Only handle dynamic imports\n if (args.kind === 'dynamic-import' && skipModules.includes(args.path)) {\n logger.warn(`Skipping dynamic import: ${args.path}`);\n // Return a virtual module that exports an empty object\n return {\n path: args.path,\n namespace: 'lmnr-skip-dynamic-import',\n };\n }\n });\n\n // Provide empty module content for skipped dynamic imports\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n build.onLoad({ filter: /.*/, namespace: 'lmnr-skip-dynamic-import' }, (args) => ({\n contents: 'export default {};',\n loader: 'js',\n }));\n },\n});\n\ndeclare global {\n var _evaluations: Evaluation<any, any, any>[] | undefined;\n var _set_global_evaluation: boolean;\n}\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}): Evaluation<any, any, any>[] {\n globalThis._evaluations = [];\n globalThis._set_global_evaluation = true;\n\n const __filename = filename;\n const __dirname = getDirname();\n\n // add some arguments for proper cjs/esm interop\n\n /* eslint-disable @typescript-eslint/no-implied-eval */\n new Function(\n \"require\",\n \"module\",\n \"__filename\",\n \"__dirname\",\n moduleText,\n )(\n require,\n module,\n __filename,\n __dirname,\n );\n /* eslint-enable @typescript-eslint/no-implied-eval */\n\n // Return the modified _evals global variable\n return globalThis._evaluations;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"files\", {\n help: \"A file or files containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory. \" +\n \"If multiple files are provided, the evaluation will run each file in order.\",\n nargs: \"*\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.add_argument(\"--output-file\", {\n help: \"Output file to write the results to. Outputs are written in JSON format.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--external-packages\", {\n help: \"[ADVANCED] List of packages to pass as external to esbuild. This will not link \" +\n \"the packages directly into the eval file, but will instead require them at runtime. \" +\n \"Read more: https://esbuild.github.io/api/#external\",\n nargs: \"*\",\n });\n\n parserEval.add_argument(\"--dynamic-imports-to-skip\", {\n help: \"[ADVANCED] List of module names to skip when encountered as dynamic imports. \" +\n \"These dynamic imports will resolve to an empty module to prevent build failures. \" +\n \"This is meant to skip the imports that are not used in the evaluation itself.\",\n nargs: \"*\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n let files: string[];\n if (args.files && args.files.length > 0) {\n files = args.files.flatMap((file: string) => glob.sync(file));\n } else {\n // No files provided, use default pattern\n files = glob.sync('evals/**/*.eval.{ts,js}');\n }\n\n files.sort();\n\n if (files.length === 0) {\n logger.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are eval files that are named like `*.eval.{ts,js}` in\" +\n \"the `evals` directory or its subdirectories.\");\n process.exit(1);\n }\n\n if (args.files.length === 0) {\n logger.info(`Located ${files.length} evaluation files in evals/`);\n } else {\n logger.info(`Running ${files.length} evaluation files.`);\n }\n\n const scores: { file: string, scores: Record<string, number> }[] = [];\n\n for (const file of files) {\n logger.info(`Loading ${file}...`);\n const buildOptions = {\n bundle: true,\n platform: \"node\" as esbuild.Platform,\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n external: [\n \"node_modules/*\",\n \"playwright\",\n \"puppeteer\",\n \"puppeteer-core\",\n \"playwright-core\",\n \"fsevents\",\n ...(args.external_packages ? args.external_packages : []),\n ],\n plugins: [\n createSkipDynamicImportsPlugin(args.dynamic_imports_to_skip || []),\n ],\n treeShaking: true,\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n logger.error(\"Error when building: No output files found \" +\n \"it is likely that all eval files are not valid TypeScript or JavaScript files.\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluations = loadModule({\n filename: file,\n moduleText: outputFileText,\n });\n\n logger.info(`Loaded ${evaluations.length} evaluations from ${file}`);\n\n for (const evaluation of evaluations) {\n if (!evaluation?.run) {\n logger.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const evalScores = await evaluation.run();\n scores.push({ file, scores: evalScores });\n }\n }\n\n if (args.output_file) {\n fs.writeFileSync(args.output_file, JSON.stringify(scores, null, 2));\n }\n },\n });\n\n parser.set_defaults({\n func: () => {\n parser.print_help();\n process.exit(0);\n },\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli().catch((err) => {\n logger.error(err instanceof Error ? err.message : err);\n throw err;\n});\n","{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.7.0\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup\",\n \"test\": \"tsx --test test/*.test.ts\",\n \"lint\": \"eslint\",\n \"lint:fix\": \"eslint --fix\"\n },\n \"files\": [\n \"dist\"\n ],\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.13.4\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/context-async-hooks\": \"^1.30.1\",\n \"@opentelemetry/core\": \"^1.30.1\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.2\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.2\",\n \"@opentelemetry/instrumentation\": \"^0.57.2\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/resources\": \"^1.30.1\",\n \"@opentelemetry/sdk-node\": \"^0.57.2\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.1\",\n \"@opentelemetry/sdk-trace-base-v2\": \"npm:@opentelemetry/sdk-trace-base@2.0.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.36.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.12.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.12.0\",\n \"@traceloop/instrumentation-azure\": \"^0.12.0\",\n \"@traceloop/instrumentation-bedrock\": \"^0.12.0\",\n \"@traceloop/instrumentation-chromadb\": \"^0.12.0\",\n \"@traceloop/instrumentation-cohere\": \"^0.12.0\",\n \"@traceloop/instrumentation-langchain\": \"^0.12.0\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.12.0\",\n \"@traceloop/instrumentation-openai\": \"^0.12.0\",\n \"@traceloop/instrumentation-pinecone\": \"^0.12.0\",\n \"@traceloop/instrumentation-qdrant\": \"^0.12.0\",\n \"@traceloop/instrumentation-together\": \"^0.12.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.12.0\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.25.8\",\n \"glob\": \"^11.0.3\",\n \"pino\": \"^9.7.0\",\n \"pino-pretty\": \"^13.1.1\",\n \"uuid\": \"^11.1.0\",\n \"zod\": \"^3.25.76\"\n },\n \"devDependencies\": {\n \"@ai-sdk/openai\": \"^2.0.22\",\n \"@anthropic-ai/sdk\": \"^0.57.0\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.859.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^2.4.2\",\n \"@eslint/eslintrc\": \"^3.3.1\",\n \"@eslint/js\": \"^9.32.0\",\n \"@google-cloud/aiplatform\": \"^5.1.0\",\n \"@google-cloud/vertexai\": \"^1.10.0\",\n \"@langchain/core\": \"^0.3.66\",\n \"@pinecone-database/pinecone\": \"^5.1.2\",\n \"@playwright/test\": \"^1.54.2\",\n \"@qdrant/js-client-rest\": \"^1.15.0\",\n \"@stylistic/eslint-plugin\": \"^5.2.2\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^24.2.0\",\n \"@types/semver\": \"^7.7.0\",\n \"@types/uuid\": \"^10.0.0\",\n \"ai\": \"^5.0.26\",\n \"chromadb\": \"^3.0.10\",\n \"cohere-ai\": \"^7.18.0\",\n \"eslint\": \"^9.32.0\",\n \"eslint-plugin-simple-import-sort\": \"^12.1.1\",\n \"eslint-plugin-unused-imports\": \"^4.1.4\",\n \"langchain\": \"^0.3.30\",\n \"llamaindex\": \"^0.11.25\",\n \"nock\": \"^14.0.8\",\n \"openai\": \"^5.11.0\",\n \"playwright\": \"^1.54.2\",\n \"puppeteer\": \"^24.15.0\",\n \"puppeteer-core\": \"^24.15.0\",\n \"together-ai\": \"^0.21.0\",\n \"tsup\": \"^8.5.0\",\n \"tsx\": \"^4.20.3\",\n \"typescript\": \"^5.8.3\",\n \"typescript-eslint\": \"^8.38.0\"\n }\n}","import { AttributeValue, SpanContext, TraceFlags } from '@opentelemetry/api';\nimport * as path from \"path\";\nimport pino, { Level } from 'pino';\nimport { PinoPretty } from 'pino-pretty';\nimport { fileURLToPath } from \"url\";\nimport { v4 as uuidv4 } from 'uuid';\n\nimport { ASSOCIATION_PROPERTIES } from './opentelemetry-lib/tracing/attributes';\nimport { LaminarSpanContext } from './types';\n\nexport function initializeLogger(options?: { colorize?: boolean, level?: Level }) {\n const colorize = options?.colorize ?? true;\n const level = options?.level\n ?? (process.env.LMNR_LOG_LEVEL?.toLowerCase()?.trim() as Level)\n ?? 'info';\n\n return pino(PinoPretty({\n colorize,\n minimumLevel: level,\n }));\n}\n\nconst logger = initializeLogger();\n\nexport type StringUUID = `${string}-${string}-${string}-${string}-${string}`;\n\nexport const isStringUUID = (id: string): id is StringUUID =>\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id);\n\nexport const NIL_UUID: StringUUID = '00000000-0000-0000-0000-000000000000';\n\nexport const newUUID = (): StringUUID => {\n // crypto.randomUUID is available in most of the modern browsers and node,\n // but is not available in \"insecure\" contexts, e.g. not https, not localhost\n // so we fallback to uuidv4 in those cases, which is less secure, but works\n // just fine.\n if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {\n return crypto.randomUUID();\n } else {\n return uuidv4() as `${string}-${string}-${string}-${string}-${string}`;\n }\n};\n\nexport const otelSpanIdToUUID = (spanId: string): string => {\n let id = spanId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 16) {\n logger.warn(`Span ID ${spanId} is not 16 hex chars long. ` +\n 'This is not a valid OpenTelemetry span ID.');\n }\n\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Span ID ${spanId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.padStart(32, '0').replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5',\n );\n};\n\nexport const otelTraceIdToUUID = (traceId: string): StringUUID => {\n let id = traceId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 32) {\n logger.warn(`Trace ID ${traceId} is not 32 hex chars long. ` +\n 'This is not a valid OpenTelemetry trace ID.');\n }\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Trace ID ${traceId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5',\n ) as StringUUID;\n};\n\nexport const uuidToOtelTraceId = (uuid: string): string => uuid.replace(/-/g, '');\nexport const uuidToOtelSpanId = (uuid: string): string => uuid.replace(/-/g, '').slice(16);\n\n/**\n * This is a simple implementation of a semaphore to replicate\n * the behavior of the `asyncio.Semaphore` in Python.\n */\nexport class Semaphore {\n /**\n * Number of permits available.\n */\n private _value: number;\n /**\n * List of promises that will be resolved when a permit becomes available.\n */\n private _waiters: ((...args: any[]) => any)[] = [];\n\n constructor(value = 1) {\n if (value < 0) {\n throw new Error(\"Semaphore value must be >= 0\");\n }\n this._value = value;\n this._waiters = [];\n }\n\n async acquire() {\n if (this._value > 0) {\n this._value--;\n return;\n }\n\n // Create a promise that will be resolved when a permit becomes available\n return new Promise(resolve => {\n this._waiters.push(resolve);\n });\n }\n\n release() {\n if (this._waiters.length > 0) {\n // If there are waiters, wake up the first one\n const resolve = this._waiters.shift();\n resolve?.();\n } else {\n this._value++;\n }\n }\n\n // Python-like context manager functionality\n async using<T>(fn: (...args: any[]) => Promise<T>) {\n try {\n await this.acquire();\n return await fn();\n } finally {\n this.release();\n }\n }\n}\n\nexport const tryToOtelSpanContext = (\n spanContext: LaminarSpanContext | Record<string, unknown> | string | SpanContext,\n): SpanContext => {\n if (typeof spanContext === 'string') {\n try {\n const record = JSON.parse(spanContext) as Record<string, unknown>;\n return recordToOtelSpanContext(record);\n } catch (e) {\n throw new Error(`Failed to parse span context ${spanContext}. ` +\n 'The string must be a json representation of a LaminarSpanContext.'\n + `Error: ${e instanceof Error ? e.message : String(e)}`);\n }\n } else if (isRecord(spanContext)) {\n // This covers the `LaminarSpanContext` case too.\n return recordToOtelSpanContext(spanContext);\n } else if (typeof spanContext.traceId === 'string'\n && typeof spanContext.spanId === 'string'\n && spanContext.traceId.length === 32\n && spanContext.spanId.length === 16) {\n logger.warn('The span context is already an OpenTelemetry SpanContext. ' +\n 'Returning it as is. Please use `LaminarSpanContext` objects instead.');\n return spanContext;\n }\n else {\n throw new Error(`Invalid span context ${JSON.stringify(spanContext)}. ` +\n 'Must be a LaminarSpanContext or its json representation.');\n }\n};\n\nconst recordToOtelSpanContext = (record: Record<string, unknown>): SpanContext => {\n if ((typeof record.spanId === 'string' && typeof record.traceId === 'string') ||\n (typeof record.span_id === 'string' && typeof record.trace_id === 'string')) {\n return {\n spanId: uuidToOtelSpanId(record?.spanId as string ?? record?.['span_id'] as string),\n traceId: uuidToOtelTraceId(record?.traceId as string ?? record?.['trace_id'] as string),\n isRemote: record?.isRemote ?? record?.['is_remote'] ?? false,\n traceFlags: record?.traceFlags ?? TraceFlags.SAMPLED,\n } as SpanContext;\n } else {\n throw new Error(`Invalid span context ${JSON.stringify(record)}. ` +\n 'Must be a json representation of a LaminarSpanContext.');\n }\n};\n\nconst isRecord = (value: unknown): value is Record<string, unknown> =>\n typeof value === 'object' && !Array.isArray(value) && value !== null;\n\n\nexport const getDirname = () => {\n if (typeof __dirname !== 'undefined') {\n return __dirname;\n }\n\n if (typeof import.meta?.url !== 'undefined') {\n return path.dirname(fileURLToPath(import.meta.url));\n }\n\n return process.cwd();\n};\n\nexport const slicePayload = <T>(value: T, length: number) => {\n if (value === null || value === undefined) {\n return value;\n }\n\n const str = JSON.stringify(value);\n if (str.length <= length) {\n return value;\n }\n\n return (str.slice(0, length) + '...');\n};\n\nexport const isOtelAttributeValueType = (value: unknown): value is AttributeValue => {\n if (typeof value === 'string'\n || typeof value === 'number'\n || typeof value === 'boolean') {\n return true;\n }\n\n if (Array.isArray(value)) {\n const allStrings = value.every(value => (value == null) || typeof value === 'string');\n const allNumbers = value.every(value => (value == null) || typeof value === 'number');\n const allBooleans = value.every(value => (value == null) || typeof value === 'boolean');\n return allStrings || allNumbers || allBooleans;\n }\n return false;\n};\n\nexport const metadataToAttributes = (\n metadata: Record<string, unknown>,\n): Record<string, AttributeValue> => Object.fromEntries(\n Object.entries(metadata).map(([key, value]) => {\n if (isOtelAttributeValueType(value)) {\n return [`${ASSOCIATION_PROPERTIES}.metadata.${key}`, value];\n } else {\n return [`${ASSOCIATION_PROPERTIES}.metadata.${key}`, JSON.stringify(value)];\n }\n }),\n);\n","import { SpanAttributes } from '@traceloop/ai-semantic-conventions';\n\nexport const SPAN_INPUT = \"lmnr.span.input\";\nexport const SPAN_OUTPUT = \"lmnr.span.output\";\nexport const SPAN_TYPE = \"lmnr.span.type\";\nexport const SPAN_PATH = \"lmnr.span.path\";\nexport const SPAN_IDS_PATH = \"lmnr.span.ids_path\";\nexport const PARENT_SPAN_PATH = \"lmnr.span.parent_path\";\nexport const PARENT_SPAN_IDS_PATH = \"lmnr.span.parent_ids_path\";\nexport const SPAN_INSTRUMENTATION_SOURCE = \"lmnr.span.instrumentation_source\";\nexport const SPAN_SDK_VERSION = \"lmnr.span.sdk_version\";\nexport const SPAN_LANGUAGE_VERSION = \"lmnr.span.language_version\";\nexport const OVERRIDE_PARENT_SPAN = \"lmnr.internal.override_parent_span\";\nexport const TRACE_HAS_BROWSER_SESSION = \"lmnr.internal.has_browser_session\";\nexport const EXTRACTED_FROM_NEXT_JS = \"lmnr.span.extracted_from.next_js\";\nexport const HUMAN_EVALUATOR_OPTIONS = 'lmnr.span.human_evaluator_options';\nexport const ASSOCIATION_PROPERTIES = \"lmnr.association.properties\";\nexport const SESSION_ID = \"lmnr.association.properties.session_id\";\nexport const USER_ID = \"lmnr.association.properties.user_id\";\nexport const TRACE_TYPE = \"lmnr.association.properties.trace_type\";\n\nexport const ASSOCIATION_PROPERTIES_OVERRIDES: Record<string, string> = {\n \"span_type\": SPAN_TYPE,\n};\n\nexport const LaminarAttributes = {\n // == This is the minimum set of attributes for a proper LLM span ==\n //\n // not SpanAttributes.LLM_USAGE_PROMPT_TOKENS\n INPUT_TOKEN_COUNT: \"gen_ai.usage.input_tokens\",\n // not SpanAttributes.LLM_USAGE_COMPLETION_TOKENS\n OUTPUT_TOKEN_COUNT: \"gen_ai.usage.output_tokens\",\n TOTAL_TOKEN_COUNT: SpanAttributes.LLM_USAGE_TOTAL_TOKENS,\n PROVIDER: SpanAttributes.LLM_SYSTEM,\n REQUEST_MODEL: SpanAttributes.LLM_REQUEST_MODEL,\n RESPONSE_MODEL: SpanAttributes.LLM_RESPONSE_MODEL,\n //\n // == End of minimum set ==\n // == Additional attributes ==\n //\n INPUT_COST: \"gen_ai.usage.input_cost\",\n OUTPUT_COST: \"gen_ai.usage.output_cost\",\n TOTAL_COST: \"gen_ai.usage.cost\",\n //\n // == End of additional attributes ==\n};\n"],"mappings":";;;;;;;;;AAEA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AACzB,YAAY,QAAQ;AACpB,YAAY,UAAU;;;ACHpB,cAAW;;;ACFb,SAAsC,kBAAkB;AACxD,YAAY,UAAU;AACtB,OAAO,UAAqB;AAC5B,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAC9B,SAAS,MAAM,cAAc;;;ACL7B,SAAS,sBAAsB;AAyBxB,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA,EAI/B,mBAAmB;AAAA;AAAA,EAEnB,oBAAoB;AAAA,EACpB,mBAAmB,eAAe;AAAA,EAClC,UAAU,eAAe;AAAA,EACzB,eAAe,eAAe;AAAA,EAC9B,gBAAgB,eAAe;AAAA;AAAA;AAAA;AAAA;AAAA,EAK/B,YAAY;AAAA,EACZ,aAAa;AAAA,EACb,YAAY;AAAA;AAAA;AAGd;;;ADnCO,SAAS,iBAAiB,SAAiD;AAChF,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,QAAQ,SAAS,SACjB,QAAQ,IAAI,gBAAgB,YAAY,GAAG,KAAK,KACjD;AAEL,SAAO,KAAK,WAAW;AAAA,IACrB;AAAA,IACA,cAAc;AAAA,EAChB,CAAC,CAAC;AACJ;AAEA,IAAM,SAAS,iBAAiB;AA0KzB,IAAM,aAAa,MAAM;AAC9B,MAAI,OAAO,cAAc,aAAa;AACpC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,aAAa,QAAQ,aAAa;AAC3C,WAAY,aAAQ,cAAc,YAAY,GAAG,CAAC;AAAA,EACpD;AAEA,SAAO,QAAQ,IAAI;AACrB;;;AF/LA,IAAMA,UAAS,iBAAiB;AAGhC,IAAM,iCAAiC,CAAC,iBAA2C;AAAA,EACjF,MAAM;AAAA,EACN,MAAMC,QAAO;AACX,QAAI,CAAC,eAAe,YAAY,WAAW,EAAG;AAE9C,IAAAA,OAAM,UAAU,EAAE,QAAQ,KAAK,GAAG,CAAC,SAAS;AAE1C,UAAI,KAAK,SAAS,oBAAoB,YAAY,SAAS,KAAK,IAAI,GAAG;AACrE,QAAAD,QAAO,KAAK,4BAA4B,KAAK,IAAI,EAAE;AAEnD,eAAO;AAAA,UACL,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,QACb;AAAA,MACF;AAAA,IACF,CAAC;AAID,IAAAC,OAAM,OAAO,EAAE,QAAQ,MAAM,WAAW,2BAA2B,GAAG,CAAC,UAAU;AAAA,MAC/E,UAAU;AAAA,MACV,QAAQ;AAAA,IACV,EAAE;AAAA,EACJ;AACF;AAOO,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGgC;AAC9B,aAAW,eAAe,CAAC;AAC3B,aAAW,yBAAyB;AAEpC,QAAM,aAAa;AACnB,QAAMC,aAAY,WAAW;AAK7B,MAAI;AAAA,IACF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACAA;AAAA,EACF;AAIA,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,QAAQ,CAAC;AAErE,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,SAAS;AAAA,IAC/B,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa,iBAAiB;AAAA,IACvC,MAAM;AAAA,IACN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,uBAAuB;AAAA,IAC7C,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,6BAA6B;AAAA,IACnD,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,UAAI;AACJ,UAAIA,MAAK,SAASA,MAAK,MAAM,SAAS,GAAG;AACvC,gBAAQA,MAAK,MAAM,QAAQ,CAAC,SAAsB,UAAK,IAAI,CAAC;AAAA,MAC9D,OAAO;AAEL,gBAAa,UAAK,yBAAyB;AAAA,MAC7C;AAEA,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,QAAAH,QAAO,MAAM,qKAEmC;AAChD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAIG,MAAK,MAAM,WAAW,GAAG;AAC3B,QAAAH,QAAO,KAAK,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE,OAAO;AACL,QAAAA,QAAO,KAAK,WAAW,MAAM,MAAM,oBAAoB;AAAA,MACzD;AAEA,YAAM,SAA6D,CAAC;AAEpE,iBAAW,QAAQ,OAAO;AACxB,QAAAA,QAAO,KAAK,WAAW,IAAI,KAAK;AAChC,cAAM,eAAe;AAAA,UACnB,QAAQ;AAAA,UACR,UAAU;AAAA,UACV,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,GAAIG,MAAK,oBAAoBA,MAAK,oBAAoB,CAAC;AAAA,UACzD;AAAA,UACA,SAAS;AAAA,YACP,+BAA+BA,MAAK,2BAA2B,CAAC,CAAC;AAAA,UACnE;AAAA,UACA,aAAa;AAAA,QACf;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,UAAAH,QAAO,MAAM,2HACqE;AAClF,cAAIG,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,cAAc,WAAW;AAAA,UAC7B,UAAU;AAAA,UACV,YAAY;AAAA,QACd,CAAC;AAED,QAAAH,QAAO,KAAK,UAAU,YAAY,MAAM,qBAAqB,IAAI,EAAE;AAEnE,mBAAW,cAAc,aAAa;AACpC,cAAI,CAAC,YAAY,KAAK;AACpB,YAAAA,QAAO,MAAM,cAAc,IAAI,oCAAoC;AACnE,gBAAIG,MAAK,eAAe;AACtB,sBAAQ,KAAK,CAAC;AAAA,YAChB;AACA;AAAA,UACF;AAEA,gBAAM,aAAa,MAAM,WAAW,IAAI;AACxC,iBAAO,KAAK,EAAE,MAAM,QAAQ,WAAW,CAAC;AAAA,QAC1C;AAAA,MACF;AAEA,UAAIA,MAAK,aAAa;AACpB,QAAG,iBAAcA,MAAK,aAAa,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,MACpE;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,MAAM;AACV,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI,EAAE,MAAM,CAAC,QAAQ;AACnB,EAAAH,QAAO,MAAM,eAAe,QAAQ,IAAI,UAAU,GAAG;AACrD,QAAM;AACR,CAAC;","names":["logger","build","__dirname","args"]}
|
|
1
|
+
{"version":3,"sources":["../src/cli.ts","../package.json","../src/utils.ts","../src/opentelemetry-lib/tracing/attributes.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { ArgumentParser } from \"argparse\";\nimport * as esbuild from \"esbuild\";\nimport * as fs from \"fs\";\nimport * as glob from \"glob\";\n\nimport { version } from \"../package.json\";\nimport { Evaluation } from \"./evaluations\";\nimport { getDirname, initializeLogger } from \"./utils\";\n\nconst logger = initializeLogger();\n\n// esbuild plugin to skip dynamic imports\nconst createSkipDynamicImportsPlugin = (skipModules: string[]): esbuild.Plugin => ({\n name: 'skip-dynamic-imports',\n setup(build) {\n if (!skipModules || skipModules.length === 0) return;\n\n build.onResolve({ filter: /.*/ }, (args) => {\n // Only handle dynamic imports\n if (args.kind === 'dynamic-import' && skipModules.includes(args.path)) {\n logger.warn(`Skipping dynamic import: ${args.path}`);\n // Return a virtual module that exports an empty object\n return {\n path: args.path,\n namespace: 'lmnr-skip-dynamic-import',\n };\n }\n });\n\n // Provide empty module content for skipped dynamic imports\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n build.onLoad({ filter: /.*/, namespace: 'lmnr-skip-dynamic-import' }, (args) => ({\n contents: 'export default {};',\n loader: 'js',\n }));\n },\n});\n\ndeclare global {\n var _evaluations: Evaluation<any, any, any>[] | undefined;\n var _set_global_evaluation: boolean;\n}\n\nexport function loadModule({\n filename,\n moduleText,\n}: {\n filename: string;\n moduleText: string;\n}): Evaluation<any, any, any>[] {\n globalThis._evaluations = [];\n globalThis._set_global_evaluation = true;\n\n const __filename = filename;\n const __dirname = getDirname();\n\n // add some arguments for proper cjs/esm interop\n\n /* eslint-disable @typescript-eslint/no-implied-eval */\n new Function(\n \"require\",\n \"module\",\n \"__filename\",\n \"__dirname\",\n moduleText,\n )(\n require,\n module,\n __filename,\n __dirname,\n );\n /* eslint-enable @typescript-eslint/no-implied-eval */\n\n // Return the modified _evals global variable\n return globalThis._evaluations;\n}\n\nasync function cli() {\n const [, , ...args] = process.argv;\n\n // Use argparse, which is the port of the python library\n const parser = new ArgumentParser({\n prog: \"lmnr\",\n description: \"CLI for Laminar. Use `lmnr <subcommand> --help` for more information.\",\n });\n\n parser.add_argument(\"-v\", \"--version\", { action: \"version\", version });\n\n const subparsers = parser.add_subparsers({\n title: \"subcommands\",\n dest: \"subcommand\",\n });\n\n const parserEval = subparsers.add_parser(\"eval\", {\n description: \"Run an evaluation\",\n help: \"Run an evaluation\",\n });\n\n parserEval.add_argument(\"files\", {\n help: \"A file or files containing the evaluation to run. If no file is provided, \" +\n \"the evaluation will run all `*.eval.ts|js` files in the `evals` directory. \" +\n \"If multiple files are provided, the evaluation will run each file in order.\",\n nargs: \"*\",\n });\n\n parserEval.add_argument(\"--fail-on-error\", {\n help: \"Fail on error. If specified, will fail if encounters a file that cannot be run\",\n action: \"store_true\",\n });\n\n parserEval.add_argument(\"--output-file\", {\n help: \"Output file to write the results to. Outputs are written in JSON format.\",\n nargs: \"?\",\n });\n\n parserEval.add_argument(\"--external-packages\", {\n help: \"[ADVANCED] List of packages to pass as external to esbuild. This will not link \" +\n \"the packages directly into the eval file, but will instead require them at runtime. \" +\n \"Read more: https://esbuild.github.io/api/#external\",\n nargs: \"*\",\n });\n\n parserEval.add_argument(\"--dynamic-imports-to-skip\", {\n help: \"[ADVANCED] List of module names to skip when encountered as dynamic imports. \" +\n \"These dynamic imports will resolve to an empty module to prevent build failures. \" +\n \"This is meant to skip the imports that are not used in the evaluation itself.\",\n nargs: \"*\",\n });\n\n parserEval.set_defaults({\n func: async (args: any) => {\n let files: string[];\n if (args.files && args.files.length > 0) {\n files = args.files.flatMap((file: string) => glob.sync(file));\n } else {\n // No files provided, use default pattern\n files = glob.sync('evals/**/*.eval.{ts,js}');\n }\n\n files.sort();\n\n if (files.length === 0) {\n logger.error(\"No evaluation files found. Please provide a file or \" +\n \"ensure there are eval files that are named like `*.eval.{ts,js}` in\" +\n \"the `evals` directory or its subdirectories.\");\n process.exit(1);\n }\n\n if (args.files.length === 0) {\n logger.info(`Located ${files.length} evaluation files in evals/`);\n } else {\n logger.info(`Running ${files.length} evaluation files.`);\n }\n\n const scores: { file: string, scores: Record<string, number> }[] = [];\n\n for (const file of files) {\n logger.info(`Loading ${file}...`);\n const buildOptions = {\n bundle: true,\n platform: \"node\" as esbuild.Platform,\n entryPoints: [file],\n outfile: `tmp_out_${file}.js`,\n write: false, // will be loaded in memory as a temp file\n external: [\n \"node_modules/*\",\n \"playwright\",\n \"puppeteer\",\n \"puppeteer-core\",\n \"playwright-core\",\n \"fsevents\",\n ...(args.external_packages ? args.external_packages : []),\n ],\n plugins: [\n createSkipDynamicImportsPlugin(args.dynamic_imports_to_skip || []),\n ],\n treeShaking: true,\n };\n\n const result = await esbuild.build(buildOptions);\n\n if (!result.outputFiles) {\n logger.error(\"Error when building: No output files found \" +\n \"it is likely that all eval files are not valid TypeScript or JavaScript files.\");\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const outputFileText = result.outputFiles[0].text;\n\n const evaluations = loadModule({\n filename: file,\n moduleText: outputFileText,\n });\n\n logger.info(`Loaded ${evaluations.length} evaluations from ${file}`);\n\n for (const evaluation of evaluations) {\n if (!evaluation?.run) {\n logger.error(`Evaluation ${file} does not properly call evaluate()`);\n if (args.fail_on_error) {\n process.exit(1);\n }\n continue;\n }\n\n const evalScores = await evaluation.run();\n scores.push({ file, scores: evalScores });\n }\n }\n\n if (args.output_file) {\n fs.writeFileSync(args.output_file, JSON.stringify(scores, null, 2));\n }\n },\n });\n\n parser.set_defaults({\n func: () => {\n parser.print_help();\n process.exit(0);\n },\n });\n\n const parsed = parser.parse_args(args);\n await parsed.func(parsed);\n}\n\ncli().catch((err) => {\n logger.error(err instanceof Error ? err.message : err);\n throw err;\n});\n","{\n \"name\": \"@lmnr-ai/lmnr\",\n \"version\": \"0.7.2\",\n \"description\": \"TypeScript SDK for Laminar AI\",\n \"main\": \"dist/index.js\",\n \"types\": \"dist/index.d.ts\",\n \"scripts\": {\n \"build\": \"tsup\",\n \"test\": \"tsx --test test/*.test.ts\",\n \"lint\": \"eslint\",\n \"lint:fix\": \"eslint --fix\"\n },\n \"files\": [\n \"dist\"\n ],\n \"bin\": {\n \"lmnr\": \"./dist/cli.js\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/lmnr-ai/lmnr-ts.git\"\n },\n \"keywords\": [\n \"laminar\",\n \"lmnr\",\n \"sdk\",\n \"lmnr.ai\"\n ],\n \"author\": \"founders@lmnr.ai\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/lmnr-ai/lmnr-ts/issues\"\n },\n \"homepage\": \"https://github.com/lmnr-ai/lmnr-ts#README\",\n \"dependencies\": {\n \"@grpc/grpc-js\": \"^1.13.4\",\n \"@opentelemetry/api\": \"^1.9.0\",\n \"@opentelemetry/context-async-hooks\": \"^1.30.1\",\n \"@opentelemetry/core\": \"^1.30.1\",\n \"@opentelemetry/exporter-trace-otlp-grpc\": \"^0.57.2\",\n \"@opentelemetry/exporter-trace-otlp-proto\": \"^0.57.2\",\n \"@opentelemetry/instrumentation\": \"^0.57.2\",\n \"@opentelemetry/otlp-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/otlp-grpc-exporter-base\": \"^0.57.2\",\n \"@opentelemetry/resources\": \"^1.30.1\",\n \"@opentelemetry/sdk-node\": \"^0.57.2\",\n \"@opentelemetry/sdk-trace-base\": \"^1.30.1\",\n \"@opentelemetry/sdk-trace-base-v2\": \"npm:@opentelemetry/sdk-trace-base@2.0.0\",\n \"@opentelemetry/sdk-trace-node\": \"^1.30.1\",\n \"@opentelemetry/semantic-conventions\": \"^1.36.0\",\n \"@traceloop/ai-semantic-conventions\": \"^0.12.0\",\n \"@traceloop/instrumentation-anthropic\": \"^0.12.0\",\n \"@traceloop/instrumentation-azure\": \"^0.12.0\",\n \"@traceloop/instrumentation-bedrock\": \"^0.12.0\",\n \"@traceloop/instrumentation-chromadb\": \"^0.12.0\",\n \"@traceloop/instrumentation-cohere\": \"^0.12.0\",\n \"@traceloop/instrumentation-langchain\": \"^0.12.0\",\n \"@traceloop/instrumentation-llamaindex\": \"^0.12.0\",\n \"@traceloop/instrumentation-openai\": \"^0.12.0\",\n \"@traceloop/instrumentation-pinecone\": \"^0.12.0\",\n \"@traceloop/instrumentation-qdrant\": \"^0.12.0\",\n \"@traceloop/instrumentation-together\": \"^0.12.1\",\n \"@traceloop/instrumentation-vertexai\": \"^0.12.0\",\n \"argparse\": \"^2.0.1\",\n \"cli-progress\": \"^3.12.0\",\n \"esbuild\": \"^0.25.8\",\n \"glob\": \"^11.0.3\",\n \"pino\": \"^9.7.0\",\n \"pino-pretty\": \"^13.1.1\",\n \"uuid\": \"^11.1.0\",\n \"zod\": \"^3.25.76\"\n },\n \"devDependencies\": {\n \"@ai-sdk/openai\": \"^2.0.22\",\n \"@anthropic-ai/sdk\": \"^0.57.0\",\n \"@aws-sdk/client-bedrock-runtime\": \"^3.859.0\",\n \"@azure/openai\": \"^2.0.0\",\n \"@browserbasehq/stagehand\": \"^2.4.2\",\n \"@eslint/eslintrc\": \"^3.3.1\",\n \"@eslint/js\": \"^9.32.0\",\n \"@google-cloud/aiplatform\": \"^5.1.0\",\n \"@google-cloud/vertexai\": \"^1.10.0\",\n \"@langchain/core\": \"^0.3.66\",\n \"@pinecone-database/pinecone\": \"^5.1.2\",\n \"@playwright/test\": \"^1.54.2\",\n \"@qdrant/js-client-rest\": \"^1.15.0\",\n \"@stylistic/eslint-plugin\": \"^5.2.2\",\n \"@types/argparse\": \"^2.0.17\",\n \"@types/cli-progress\": \"^3.11.6\",\n \"@types/node\": \"^24.2.0\",\n \"@types/semver\": \"^7.7.0\",\n \"@types/uuid\": \"^10.0.0\",\n \"ai\": \"^5.0.26\",\n \"chromadb\": \"^3.0.10\",\n \"cohere-ai\": \"^7.18.0\",\n \"eslint\": \"^9.32.0\",\n \"eslint-plugin-simple-import-sort\": \"^12.1.1\",\n \"eslint-plugin-unused-imports\": \"^4.1.4\",\n \"langchain\": \"^0.3.30\",\n \"llamaindex\": \"^0.11.25\",\n \"nock\": \"^14.0.8\",\n \"openai\": \"^5.11.0\",\n \"playwright\": \"^1.54.2\",\n \"puppeteer\": \"^24.15.0\",\n \"puppeteer-core\": \"^24.15.0\",\n \"together-ai\": \"^0.21.0\",\n \"tsup\": \"^8.5.0\",\n \"tsx\": \"^4.20.3\",\n \"typescript\": \"^5.8.3\",\n \"typescript-eslint\": \"^8.38.0\"\n }\n}","import { AttributeValue, SpanContext, TraceFlags } from '@opentelemetry/api';\nimport * as path from \"path\";\nimport pino, { Level } from 'pino';\nimport { PinoPretty } from 'pino-pretty';\nimport { fileURLToPath } from \"url\";\nimport { v4 as uuidv4 } from 'uuid';\n\nimport { ASSOCIATION_PROPERTIES } from './opentelemetry-lib/tracing/attributes';\nimport { LaminarSpanContext } from './types';\n\nexport function initializeLogger(options?: { colorize?: boolean, level?: Level }) {\n const colorize = options?.colorize ?? true;\n const level = options?.level\n ?? (process.env.LMNR_LOG_LEVEL?.toLowerCase()?.trim() as Level)\n ?? 'info';\n\n return pino(PinoPretty({\n colorize,\n minimumLevel: level,\n }));\n}\n\nconst logger = initializeLogger();\n\nexport type StringUUID = `${string}-${string}-${string}-${string}-${string}`;\n\nexport const isStringUUID = (id: string): id is StringUUID =>\n /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(id);\n\nexport const NIL_UUID: StringUUID = '00000000-0000-0000-0000-000000000000';\n\nexport const newUUID = (): StringUUID => {\n // crypto.randomUUID is available in most of the modern browsers and node,\n // but is not available in \"insecure\" contexts, e.g. not https, not localhost\n // so we fallback to uuidv4 in those cases, which is less secure, but works\n // just fine.\n if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') {\n return crypto.randomUUID();\n } else {\n return uuidv4() as `${string}-${string}-${string}-${string}-${string}`;\n }\n};\n\nexport const otelSpanIdToUUID = (spanId: string): string => {\n let id = spanId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 16) {\n logger.warn(`Span ID ${spanId} is not 16 hex chars long. ` +\n 'This is not a valid OpenTelemetry span ID.');\n }\n\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Span ID ${spanId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.padStart(32, '0').replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5',\n );\n};\n\nexport const otelTraceIdToUUID = (traceId: string): StringUUID => {\n let id = traceId.toLowerCase();\n if (id.startsWith('0x')) {\n id = id.slice(2);\n }\n if (id.length !== 32) {\n logger.warn(`Trace ID ${traceId} is not 32 hex chars long. ` +\n 'This is not a valid OpenTelemetry trace ID.');\n }\n if (!/^[0-9a-f]+$/.test(id)) {\n logger.error(`Trace ID ${traceId} is not a valid hex string. ` +\n 'Generating a random UUID instead.');\n return newUUID();\n }\n\n return id.replace(\n /^([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{12})$/,\n '$1-$2-$3-$4-$5',\n ) as StringUUID;\n};\n\nexport const uuidToOtelTraceId = (uuid: string): string => uuid.replace(/-/g, '');\nexport const uuidToOtelSpanId = (uuid: string): string => uuid.replace(/-/g, '').slice(16);\n\n/**\n * This is a simple implementation of a semaphore to replicate\n * the behavior of the `asyncio.Semaphore` in Python.\n */\nexport class Semaphore {\n /**\n * Number of permits available.\n */\n private _value: number;\n /**\n * List of promises that will be resolved when a permit becomes available.\n */\n private _waiters: ((...args: any[]) => any)[] = [];\n\n constructor(value = 1) {\n if (value < 0) {\n throw new Error(\"Semaphore value must be >= 0\");\n }\n this._value = value;\n this._waiters = [];\n }\n\n async acquire() {\n if (this._value > 0) {\n this._value--;\n return;\n }\n\n // Create a promise that will be resolved when a permit becomes available\n return new Promise(resolve => {\n this._waiters.push(resolve);\n });\n }\n\n release() {\n if (this._waiters.length > 0) {\n // If there are waiters, wake up the first one\n const resolve = this._waiters.shift();\n resolve?.();\n } else {\n this._value++;\n }\n }\n\n // Python-like context manager functionality\n async using<T>(fn: (...args: any[]) => Promise<T>) {\n try {\n await this.acquire();\n return await fn();\n } finally {\n this.release();\n }\n }\n}\n\nexport const tryToOtelSpanContext = (\n spanContext: LaminarSpanContext | Record<string, unknown> | string | SpanContext,\n): SpanContext => {\n if (typeof spanContext === 'string') {\n try {\n const record = JSON.parse(spanContext) as Record<string, unknown>;\n return recordToOtelSpanContext(record);\n } catch (e) {\n throw new Error(`Failed to parse span context ${spanContext}. ` +\n 'The string must be a json representation of a LaminarSpanContext.'\n + `Error: ${e instanceof Error ? e.message : String(e)}`);\n }\n } else if (isRecord(spanContext)) {\n // This covers the `LaminarSpanContext` case too.\n return recordToOtelSpanContext(spanContext);\n } else if (typeof spanContext.traceId === 'string'\n && typeof spanContext.spanId === 'string'\n && spanContext.traceId.length === 32\n && spanContext.spanId.length === 16) {\n logger.warn('The span context is already an OpenTelemetry SpanContext. ' +\n 'Returning it as is. Please use `LaminarSpanContext` objects instead.');\n return spanContext;\n }\n else {\n throw new Error(`Invalid span context ${JSON.stringify(spanContext)}. ` +\n 'Must be a LaminarSpanContext or its json representation.');\n }\n};\n\nconst recordToOtelSpanContext = (record: Record<string, unknown>): SpanContext => {\n if ((typeof record.spanId === 'string' && typeof record.traceId === 'string') ||\n (typeof record.span_id === 'string' && typeof record.trace_id === 'string')) {\n return {\n spanId: uuidToOtelSpanId(record?.spanId as string ?? record?.['span_id'] as string),\n traceId: uuidToOtelTraceId(record?.traceId as string ?? record?.['trace_id'] as string),\n isRemote: record?.isRemote ?? record?.['is_remote'] ?? false,\n traceFlags: record?.traceFlags ?? TraceFlags.SAMPLED,\n } as SpanContext;\n } else {\n throw new Error(`Invalid span context ${JSON.stringify(record)}. ` +\n 'Must be a json representation of a LaminarSpanContext.');\n }\n};\n\nconst isRecord = (value: unknown): value is Record<string, unknown> =>\n typeof value === 'object' && !Array.isArray(value) && value !== null;\n\n\nexport const getDirname = () => {\n if (typeof __dirname !== 'undefined') {\n return __dirname;\n }\n\n if (typeof import.meta?.url !== 'undefined') {\n return path.dirname(fileURLToPath(import.meta.url));\n }\n\n return process.cwd();\n};\n\nexport const slicePayload = <T>(value: T, length: number) => {\n if (value === null || value === undefined) {\n return value;\n }\n\n const str = JSON.stringify(value);\n if (str.length <= length) {\n return value;\n }\n\n return (str.slice(0, length) + '...');\n};\n\nexport const isOtelAttributeValueType = (value: unknown): value is AttributeValue => {\n if (typeof value === 'string'\n || typeof value === 'number'\n || typeof value === 'boolean') {\n return true;\n }\n\n if (Array.isArray(value)) {\n const allStrings = value.every(value => (value == null) || typeof value === 'string');\n const allNumbers = value.every(value => (value == null) || typeof value === 'number');\n const allBooleans = value.every(value => (value == null) || typeof value === 'boolean');\n return allStrings || allNumbers || allBooleans;\n }\n return false;\n};\n\nexport const metadataToAttributes = (\n metadata: Record<string, unknown>,\n): Record<string, AttributeValue> => Object.fromEntries(\n Object.entries(metadata).map(([key, value]) => {\n if (isOtelAttributeValueType(value)) {\n return [`${ASSOCIATION_PROPERTIES}.metadata.${key}`, value];\n } else {\n return [`${ASSOCIATION_PROPERTIES}.metadata.${key}`, JSON.stringify(value)];\n }\n }),\n);\n","import { SpanAttributes } from '@traceloop/ai-semantic-conventions';\n\nexport const SPAN_INPUT = \"lmnr.span.input\";\nexport const SPAN_OUTPUT = \"lmnr.span.output\";\nexport const SPAN_TYPE = \"lmnr.span.type\";\nexport const SPAN_PATH = \"lmnr.span.path\";\nexport const SPAN_IDS_PATH = \"lmnr.span.ids_path\";\nexport const PARENT_SPAN_PATH = \"lmnr.span.parent_path\";\nexport const PARENT_SPAN_IDS_PATH = \"lmnr.span.parent_ids_path\";\nexport const SPAN_INSTRUMENTATION_SOURCE = \"lmnr.span.instrumentation_source\";\nexport const SPAN_SDK_VERSION = \"lmnr.span.sdk_version\";\nexport const SPAN_LANGUAGE_VERSION = \"lmnr.span.language_version\";\nexport const OVERRIDE_PARENT_SPAN = \"lmnr.internal.override_parent_span\";\nexport const TRACE_HAS_BROWSER_SESSION = \"lmnr.internal.has_browser_session\";\nexport const EXTRACTED_FROM_NEXT_JS = \"lmnr.span.extracted_from.next_js\";\nexport const HUMAN_EVALUATOR_OPTIONS = 'lmnr.span.human_evaluator_options';\nexport const ASSOCIATION_PROPERTIES = \"lmnr.association.properties\";\nexport const SESSION_ID = \"lmnr.association.properties.session_id\";\nexport const USER_ID = \"lmnr.association.properties.user_id\";\nexport const TRACE_TYPE = \"lmnr.association.properties.trace_type\";\n\nexport const ASSOCIATION_PROPERTIES_OVERRIDES: Record<string, string> = {\n \"span_type\": SPAN_TYPE,\n};\n\nexport const LaminarAttributes = {\n // == This is the minimum set of attributes for a proper LLM span ==\n //\n // not SpanAttributes.LLM_USAGE_PROMPT_TOKENS\n INPUT_TOKEN_COUNT: \"gen_ai.usage.input_tokens\",\n // not SpanAttributes.LLM_USAGE_COMPLETION_TOKENS\n OUTPUT_TOKEN_COUNT: \"gen_ai.usage.output_tokens\",\n TOTAL_TOKEN_COUNT: SpanAttributes.LLM_USAGE_TOTAL_TOKENS,\n PROVIDER: SpanAttributes.LLM_SYSTEM,\n REQUEST_MODEL: SpanAttributes.LLM_REQUEST_MODEL,\n RESPONSE_MODEL: SpanAttributes.LLM_RESPONSE_MODEL,\n //\n // == End of minimum set ==\n // == Additional attributes ==\n //\n INPUT_COST: \"gen_ai.usage.input_cost\",\n OUTPUT_COST: \"gen_ai.usage.output_cost\",\n TOTAL_COST: \"gen_ai.usage.cost\",\n //\n // == End of additional attributes ==\n};\n"],"mappings":";;;;;;;;;AAEA,SAAS,sBAAsB;AAC/B,YAAY,aAAa;AACzB,YAAY,QAAQ;AACpB,YAAY,UAAU;;;ACHpB,cAAW;;;ACFb,SAAsC,kBAAkB;AACxD,YAAY,UAAU;AACtB,OAAO,UAAqB;AAC5B,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAC9B,SAAS,MAAM,cAAc;;;ACL7B,SAAS,sBAAsB;AAyBxB,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA,EAI/B,mBAAmB;AAAA;AAAA,EAEnB,oBAAoB;AAAA,EACpB,mBAAmB,eAAe;AAAA,EAClC,UAAU,eAAe;AAAA,EACzB,eAAe,eAAe;AAAA,EAC9B,gBAAgB,eAAe;AAAA;AAAA;AAAA;AAAA;AAAA,EAK/B,YAAY;AAAA,EACZ,aAAa;AAAA,EACb,YAAY;AAAA;AAAA;AAGd;;;ADnCO,SAAS,iBAAiB,SAAiD;AAChF,QAAM,WAAW,SAAS,YAAY;AACtC,QAAM,QAAQ,SAAS,SACjB,QAAQ,IAAI,gBAAgB,YAAY,GAAG,KAAK,KACjD;AAEL,SAAO,KAAK,WAAW;AAAA,IACrB;AAAA,IACA,cAAc;AAAA,EAChB,CAAC,CAAC;AACJ;AAEA,IAAM,SAAS,iBAAiB;AA0KzB,IAAM,aAAa,MAAM;AAC9B,MAAI,OAAO,cAAc,aAAa;AACpC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,aAAa,QAAQ,aAAa;AAC3C,WAAY,aAAQ,cAAc,YAAY,GAAG,CAAC;AAAA,EACpD;AAEA,SAAO,QAAQ,IAAI;AACrB;;;AF/LA,IAAMA,UAAS,iBAAiB;AAGhC,IAAM,iCAAiC,CAAC,iBAA2C;AAAA,EACjF,MAAM;AAAA,EACN,MAAMC,QAAO;AACX,QAAI,CAAC,eAAe,YAAY,WAAW,EAAG;AAE9C,IAAAA,OAAM,UAAU,EAAE,QAAQ,KAAK,GAAG,CAAC,SAAS;AAE1C,UAAI,KAAK,SAAS,oBAAoB,YAAY,SAAS,KAAK,IAAI,GAAG;AACrE,QAAAD,QAAO,KAAK,4BAA4B,KAAK,IAAI,EAAE;AAEnD,eAAO;AAAA,UACL,MAAM,KAAK;AAAA,UACX,WAAW;AAAA,QACb;AAAA,MACF;AAAA,IACF,CAAC;AAID,IAAAC,OAAM,OAAO,EAAE,QAAQ,MAAM,WAAW,2BAA2B,GAAG,CAAC,UAAU;AAAA,MAC/E,UAAU;AAAA,MACV,QAAQ;AAAA,IACV,EAAE;AAAA,EACJ;AACF;AAOO,SAAS,WAAW;AAAA,EACzB;AAAA,EACA;AACF,GAGgC;AAC9B,aAAW,eAAe,CAAC;AAC3B,aAAW,yBAAyB;AAEpC,QAAM,aAAa;AACnB,QAAMC,aAAY,WAAW;AAK7B,MAAI;AAAA,IACF;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAAA,IACE;AAAA,IACA;AAAA,IACA;AAAA,IACAA;AAAA,EACF;AAIA,SAAO,WAAW;AACpB;AAEA,eAAe,MAAM;AACnB,QAAM,CAAC,EAAE,EAAE,GAAG,IAAI,IAAI,QAAQ;AAG9B,QAAM,SAAS,IAAI,eAAe;AAAA,IAChC,MAAM;AAAA,IACN,aAAa;AAAA,EACf,CAAC;AAED,SAAO,aAAa,MAAM,aAAa,EAAE,QAAQ,WAAW,QAAQ,CAAC;AAErE,QAAM,aAAa,OAAO,eAAe;AAAA,IACvC,OAAO;AAAA,IACP,MAAM;AAAA,EACR,CAAC;AAED,QAAM,aAAa,WAAW,WAAW,QAAQ;AAAA,IAC/C,aAAa;AAAA,IACb,MAAM;AAAA,EACR,CAAC;AAED,aAAW,aAAa,SAAS;AAAA,IAC/B,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,mBAAmB;AAAA,IACzC,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AAED,aAAW,aAAa,iBAAiB;AAAA,IACvC,MAAM;AAAA,IACN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,uBAAuB;AAAA,IAC7C,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa,6BAA6B;AAAA,IACnD,MAAM;AAAA,IAGN,OAAO;AAAA,EACT,CAAC;AAED,aAAW,aAAa;AAAA,IACtB,MAAM,OAAOC,UAAc;AACzB,UAAI;AACJ,UAAIA,MAAK,SAASA,MAAK,MAAM,SAAS,GAAG;AACvC,gBAAQA,MAAK,MAAM,QAAQ,CAAC,SAAsB,UAAK,IAAI,CAAC;AAAA,MAC9D,OAAO;AAEL,gBAAa,UAAK,yBAAyB;AAAA,MAC7C;AAEA,YAAM,KAAK;AAEX,UAAI,MAAM,WAAW,GAAG;AACtB,QAAAH,QAAO,MAAM,qKAEmC;AAChD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAIG,MAAK,MAAM,WAAW,GAAG;AAC3B,QAAAH,QAAO,KAAK,WAAW,MAAM,MAAM,6BAA6B;AAAA,MAClE,OAAO;AACL,QAAAA,QAAO,KAAK,WAAW,MAAM,MAAM,oBAAoB;AAAA,MACzD;AAEA,YAAM,SAA6D,CAAC;AAEpE,iBAAW,QAAQ,OAAO;AACxB,QAAAA,QAAO,KAAK,WAAW,IAAI,KAAK;AAChC,cAAM,eAAe;AAAA,UACnB,QAAQ;AAAA,UACR,UAAU;AAAA,UACV,aAAa,CAAC,IAAI;AAAA,UAClB,SAAS,WAAW,IAAI;AAAA,UACxB,OAAO;AAAA;AAAA,UACP,UAAU;AAAA,YACR;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,GAAIG,MAAK,oBAAoBA,MAAK,oBAAoB,CAAC;AAAA,UACzD;AAAA,UACA,SAAS;AAAA,YACP,+BAA+BA,MAAK,2BAA2B,CAAC,CAAC;AAAA,UACnE;AAAA,UACA,aAAa;AAAA,QACf;AAEA,cAAM,SAAS,MAAc,cAAM,YAAY;AAE/C,YAAI,CAAC,OAAO,aAAa;AACvB,UAAAH,QAAO,MAAM,2HACqE;AAClF,cAAIG,MAAK,eAAe;AACtB,oBAAQ,KAAK,CAAC;AAAA,UAChB;AACA;AAAA,QACF;AAEA,cAAM,iBAAiB,OAAO,YAAY,CAAC,EAAE;AAE7C,cAAM,cAAc,WAAW;AAAA,UAC7B,UAAU;AAAA,UACV,YAAY;AAAA,QACd,CAAC;AAED,QAAAH,QAAO,KAAK,UAAU,YAAY,MAAM,qBAAqB,IAAI,EAAE;AAEnE,mBAAW,cAAc,aAAa;AACpC,cAAI,CAAC,YAAY,KAAK;AACpB,YAAAA,QAAO,MAAM,cAAc,IAAI,oCAAoC;AACnE,gBAAIG,MAAK,eAAe;AACtB,sBAAQ,KAAK,CAAC;AAAA,YAChB;AACA;AAAA,UACF;AAEA,gBAAM,aAAa,MAAM,WAAW,IAAI;AACxC,iBAAO,KAAK,EAAE,MAAM,QAAQ,WAAW,CAAC;AAAA,QAC1C;AAAA,MACF;AAEA,UAAIA,MAAK,aAAa;AACpB,QAAG,iBAAcA,MAAK,aAAa,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,MACpE;AAAA,IACF;AAAA,EACF,CAAC;AAED,SAAO,aAAa;AAAA,IAClB,MAAM,MAAM;AACV,aAAO,WAAW;AAClB,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAED,QAAM,SAAS,OAAO,WAAW,IAAI;AACrC,QAAM,OAAO,KAAK,MAAM;AAC1B;AAEA,IAAI,EAAE,MAAM,CAAC,QAAQ;AACnB,EAAAH,QAAO,MAAM,eAAe,QAAQ,IAAI,UAAU,GAAG;AACrD,QAAM;AACR,CAAC;","names":["logger","build","__dirname","args"]}
|
package/dist/index.js
CHANGED
|
@@ -675,7 +675,7 @@ var AgentResource = class extends BaseResource {
|
|
|
675
675
|
};
|
|
676
676
|
|
|
677
677
|
// package.json
|
|
678
|
-
var version = "0.7.
|
|
678
|
+
var version = "0.7.2";
|
|
679
679
|
|
|
680
680
|
// src/version.ts
|
|
681
681
|
var getLangVersion = () => {
|
|
@@ -2318,13 +2318,23 @@ var PlaywrightInstrumentation = class extends import_instrumentation.Instrumenta
|
|
|
2318
2318
|
}
|
|
2319
2319
|
);
|
|
2320
2320
|
this._patchedBrowsers = /* @__PURE__ */ new Set();
|
|
2321
|
-
this.
|
|
2321
|
+
this._parentSpans = /* @__PURE__ */ new Map();
|
|
2322
2322
|
this._client = client;
|
|
2323
2323
|
this._sessionRecordingOptions = sessionRecordingOptions;
|
|
2324
2324
|
}
|
|
2325
2325
|
// It's the caller's responsibility to ensure the span is ended
|
|
2326
|
-
|
|
2327
|
-
this.
|
|
2326
|
+
setParentSpanForSession(sessionId, span) {
|
|
2327
|
+
this._parentSpans.set(sessionId, span);
|
|
2328
|
+
}
|
|
2329
|
+
removeAndEndParentSpanForSession(sessionId) {
|
|
2330
|
+
const span = this._parentSpans.get(sessionId);
|
|
2331
|
+
if (span && span.isRecording()) {
|
|
2332
|
+
span.end();
|
|
2333
|
+
}
|
|
2334
|
+
this._parentSpans.delete(sessionId);
|
|
2335
|
+
}
|
|
2336
|
+
getParentSpanForSession(sessionId) {
|
|
2337
|
+
return this._parentSpans.get(sessionId);
|
|
2328
2338
|
}
|
|
2329
2339
|
init() {
|
|
2330
2340
|
const module2 = new import_instrumentation.InstrumentationNodeModuleDefinition(
|
|
@@ -2447,9 +2457,10 @@ var PlaywrightInstrumentation = class extends import_instrumentation.Instrumenta
|
|
|
2447
2457
|
await this._patchPage(page, sessionId);
|
|
2448
2458
|
});
|
|
2449
2459
|
};
|
|
2450
|
-
|
|
2460
|
+
const parentSpan = this.getParentSpanForSession(sessionId);
|
|
2461
|
+
if (parentSpan) {
|
|
2451
2462
|
return await Laminar.withSpan(
|
|
2452
|
-
|
|
2463
|
+
parentSpan,
|
|
2453
2464
|
wrapped
|
|
2454
2465
|
);
|
|
2455
2466
|
}
|
|
@@ -2512,6 +2523,9 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
|
|
|
2512
2523
|
enabled: true
|
|
2513
2524
|
}
|
|
2514
2525
|
);
|
|
2526
|
+
this.globalLLMClientOptions = /* @__PURE__ */ new WeakMap();
|
|
2527
|
+
this.globalAgentOptions = /* @__PURE__ */ new WeakMap();
|
|
2528
|
+
this.stagehandInstanceToSessionId = /* @__PURE__ */ new WeakMap();
|
|
2515
2529
|
this.playwrightInstrumentation = playwrightInstrumentation;
|
|
2516
2530
|
}
|
|
2517
2531
|
init() {
|
|
@@ -2617,43 +2631,47 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
|
|
|
2617
2631
|
}
|
|
2618
2632
|
patchStagehandInit() {
|
|
2619
2633
|
const instrumentation = this;
|
|
2620
|
-
return (original) => async function method(
|
|
2634
|
+
return (original) => async function method() {
|
|
2621
2635
|
const sessionId = newUUID();
|
|
2622
|
-
|
|
2623
|
-
instrumentation._parentSpan = Laminar.startSpan({
|
|
2636
|
+
const parentSpan = Laminar.startSpan({
|
|
2624
2637
|
name: "Stagehand"
|
|
2625
2638
|
});
|
|
2626
|
-
instrumentation.playwrightInstrumentation.
|
|
2627
|
-
const result = await original.bind(this).apply(this
|
|
2639
|
+
instrumentation.playwrightInstrumentation.setParentSpanForSession(sessionId, parentSpan);
|
|
2640
|
+
const result = await original.bind(this).apply(this);
|
|
2628
2641
|
await instrumentation.playwrightInstrumentation.patchPage(this.page, sessionId);
|
|
2629
2642
|
instrumentation._wrap(
|
|
2630
2643
|
this,
|
|
2631
2644
|
"agent",
|
|
2632
|
-
instrumentation.patchStagehandAgentInitializer()
|
|
2633
|
-
);
|
|
2634
|
-
instrumentation.patchStagehandPage(this.stagehandPage);
|
|
2635
|
-
instrumentation.globalLLMClientOptions = {
|
|
2636
|
-
provider: this.llmClient.type,
|
|
2637
|
-
model: this.llmClient.modelName
|
|
2638
|
-
};
|
|
2639
|
-
instrumentation._wrap(
|
|
2640
|
-
this.llmClient,
|
|
2641
|
-
"createChatCompletion",
|
|
2642
|
-
instrumentation.patchStagehandLLMClientCreateChatCompletion()
|
|
2645
|
+
instrumentation.patchStagehandAgentInitializer(sessionId)
|
|
2643
2646
|
);
|
|
2647
|
+
instrumentation.patchStagehandPage(this.stagehandPage, sessionId);
|
|
2648
|
+
if (this.llmClient) {
|
|
2649
|
+
instrumentation.globalLLMClientOptions.set(this.llmClient, {
|
|
2650
|
+
provider: this.llmClient.type,
|
|
2651
|
+
model: this.llmClient.modelName
|
|
2652
|
+
});
|
|
2653
|
+
instrumentation._wrap(
|
|
2654
|
+
this.llmClient,
|
|
2655
|
+
"createChatCompletion",
|
|
2656
|
+
instrumentation.patchStagehandLLMClientCreateChatCompletion()
|
|
2657
|
+
);
|
|
2658
|
+
}
|
|
2659
|
+
instrumentation.stagehandInstanceToSessionId.set(this, sessionId);
|
|
2644
2660
|
return result;
|
|
2645
2661
|
};
|
|
2646
2662
|
}
|
|
2647
2663
|
patchStagehandClose() {
|
|
2648
2664
|
const instrumentation = this;
|
|
2649
2665
|
return (original) => async function method(...args) {
|
|
2650
|
-
|
|
2651
|
-
|
|
2666
|
+
const sessionId = instrumentation.stagehandInstanceToSessionId.get(this);
|
|
2667
|
+
if (sessionId) {
|
|
2668
|
+
instrumentation.playwrightInstrumentation.removeAndEndParentSpanForSession(sessionId);
|
|
2669
|
+
instrumentation.stagehandInstanceToSessionId.delete(this);
|
|
2652
2670
|
}
|
|
2653
2671
|
await original.bind(this).apply(this, args);
|
|
2654
2672
|
};
|
|
2655
2673
|
}
|
|
2656
|
-
patchStagehandPage(page) {
|
|
2674
|
+
patchStagehandPage(page, sessionId) {
|
|
2657
2675
|
const actHandler = page.actHandler;
|
|
2658
2676
|
if (actHandler) {
|
|
2659
2677
|
if (actHandler.act) {
|
|
@@ -2706,20 +2724,20 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
|
|
|
2706
2724
|
this._wrap(
|
|
2707
2725
|
page,
|
|
2708
2726
|
"act",
|
|
2709
|
-
this.patchStagehandGlobalMethod("act")
|
|
2727
|
+
this.patchStagehandGlobalMethod("act", sessionId)
|
|
2710
2728
|
);
|
|
2711
2729
|
this._wrap(
|
|
2712
2730
|
page,
|
|
2713
2731
|
"extract",
|
|
2714
|
-
this.patchStagehandGlobalMethod("extract")
|
|
2732
|
+
this.patchStagehandGlobalMethod("extract", sessionId)
|
|
2715
2733
|
);
|
|
2716
2734
|
this._wrap(
|
|
2717
2735
|
page,
|
|
2718
2736
|
"observe",
|
|
2719
|
-
this.patchStagehandGlobalMethod("observe")
|
|
2737
|
+
this.patchStagehandGlobalMethod("observe", sessionId)
|
|
2720
2738
|
);
|
|
2721
2739
|
}
|
|
2722
|
-
patchStagehandGlobalMethod(methodName) {
|
|
2740
|
+
patchStagehandGlobalMethod(methodName, sessionId) {
|
|
2723
2741
|
const instrumentation = this;
|
|
2724
2742
|
return (original) => async function method(...args) {
|
|
2725
2743
|
const input = nameArgsOrCopy(args);
|
|
@@ -2734,7 +2752,7 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
|
|
|
2734
2752
|
input[0] = { ...rest, schema: prettySchema };
|
|
2735
2753
|
}
|
|
2736
2754
|
return await Laminar.withSpan(
|
|
2737
|
-
instrumentation.
|
|
2755
|
+
instrumentation.playwrightInstrumentation.getParentSpanForSession(sessionId),
|
|
2738
2756
|
async () => await observe(
|
|
2739
2757
|
{
|
|
2740
2758
|
name: `stagehand.${methodName}`,
|
|
@@ -2890,8 +2908,8 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
|
|
|
2890
2908
|
const currentSpan = import_api6.trace.getSpan(LaminarContextManager.getContext()) ?? import_api6.trace.getActiveSpan();
|
|
2891
2909
|
const span = currentSpan;
|
|
2892
2910
|
const innerOptions = options.options;
|
|
2893
|
-
const recordedProvider = instrumentation.globalLLMClientOptions?.provider;
|
|
2894
|
-
const provider = recordedProvider === "aisdk" && instrumentation.globalLLMClientOptions?.model ? modelToProviderMap[instrumentation.globalLLMClientOptions.model] ?? "aisdk" : recordedProvider;
|
|
2911
|
+
const recordedProvider = instrumentation.globalLLMClientOptions.get(this)?.provider;
|
|
2912
|
+
const provider = recordedProvider === "aisdk" && instrumentation.globalLLMClientOptions.get(this)?.model ? modelToProviderMap[instrumentation.globalLLMClientOptions.get(this).model] ?? "aisdk" : recordedProvider;
|
|
2895
2913
|
span.setAttributes({
|
|
2896
2914
|
[SPAN_TYPE]: "LLM",
|
|
2897
2915
|
...innerOptions.temperature ? {
|
|
@@ -2909,8 +2927,8 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
|
|
|
2909
2927
|
...innerOptions.maxTokens !== void 0 ? {
|
|
2910
2928
|
"gen_ai.request.max_tokens": innerOptions.maxTokens
|
|
2911
2929
|
} : {},
|
|
2912
|
-
...instrumentation.globalLLMClientOptions ? {
|
|
2913
|
-
"gen_ai.request.model": instrumentation.globalLLMClientOptions.model,
|
|
2930
|
+
...instrumentation.globalLLMClientOptions.get(this) ? {
|
|
2931
|
+
"gen_ai.request.model": instrumentation.globalLLMClientOptions.get(this)?.model,
|
|
2914
2932
|
"gen_ai.system": provider
|
|
2915
2933
|
} : {}
|
|
2916
2934
|
});
|
|
@@ -2967,30 +2985,30 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
|
|
|
2967
2985
|
});
|
|
2968
2986
|
};
|
|
2969
2987
|
}
|
|
2970
|
-
patchStagehandAgentInitializer() {
|
|
2988
|
+
patchStagehandAgentInitializer(sessionId) {
|
|
2971
2989
|
const instrumentation = this;
|
|
2972
2990
|
return (original) => function agent(...args) {
|
|
2973
2991
|
if (args.length > 0 && typeof args[0] === "object") {
|
|
2974
|
-
instrumentation.globalAgentOptions
|
|
2992
|
+
instrumentation.globalAgentOptions.set(this, args[0]);
|
|
2975
2993
|
}
|
|
2976
2994
|
const agent2 = original.bind(this).apply(this, args);
|
|
2977
|
-
instrumentation.patchStagehandAgent(agent2);
|
|
2995
|
+
instrumentation.patchStagehandAgent(agent2, sessionId);
|
|
2978
2996
|
return agent2;
|
|
2979
2997
|
};
|
|
2980
2998
|
}
|
|
2981
|
-
patchStagehandAgent(agent) {
|
|
2999
|
+
patchStagehandAgent(agent, sessionId) {
|
|
2982
3000
|
this._wrap(
|
|
2983
3001
|
agent,
|
|
2984
3002
|
"execute",
|
|
2985
|
-
this.patchStagehandAgentExecute()
|
|
3003
|
+
this.patchStagehandAgentExecute(sessionId)
|
|
2986
3004
|
);
|
|
2987
3005
|
}
|
|
2988
|
-
patchStagehandAgentExecute() {
|
|
3006
|
+
patchStagehandAgentExecute(sessionId) {
|
|
2989
3007
|
const instrumentation = this;
|
|
2990
3008
|
return (original) => async function execute(...args) {
|
|
2991
3009
|
const input = nameArgsOrCopy(args);
|
|
2992
3010
|
return await Laminar.withSpan(
|
|
2993
|
-
instrumentation.
|
|
3011
|
+
instrumentation.playwrightInstrumentation.getParentSpanForSession(sessionId),
|
|
2994
3012
|
async () => await observe(
|
|
2995
3013
|
{
|
|
2996
3014
|
name: "stagehand.agent.execute",
|
|
@@ -3006,16 +3024,16 @@ var StagehandInstrumentation = class extends import_instrumentation2.Instrumenta
|
|
|
3006
3024
|
},
|
|
3007
3025
|
async () => {
|
|
3008
3026
|
const span = import_api6.trace.getSpan(LaminarContextManager.getContext()) ?? import_api6.trace.getActiveSpan();
|
|
3009
|
-
const provider = instrumentation.globalAgentOptions?.provider ?? instrumentation.globalLLMClientOptions?.provider;
|
|
3010
|
-
const model = instrumentation.globalAgentOptions?.model ?? instrumentation.globalLLMClientOptions?.model;
|
|
3027
|
+
const provider = instrumentation.globalAgentOptions.get(this)?.provider ?? instrumentation.globalLLMClientOptions.get(this)?.provider;
|
|
3028
|
+
const model = instrumentation.globalAgentOptions.get(this)?.model ?? instrumentation.globalLLMClientOptions.get(this)?.model;
|
|
3011
3029
|
span?.setAttributes({
|
|
3012
3030
|
...provider ? { "gen_ai.system": provider } : {},
|
|
3013
3031
|
...model ? { "gen_ai.request.model": model } : {}
|
|
3014
3032
|
});
|
|
3015
3033
|
let promptIndex = 0;
|
|
3016
|
-
if (instrumentation.globalAgentOptions?.instructions) {
|
|
3034
|
+
if (instrumentation.globalAgentOptions.get(this)?.instructions) {
|
|
3017
3035
|
span?.setAttributes({
|
|
3018
|
-
"gen_ai.prompt.0.content": instrumentation.globalAgentOptions.instructions,
|
|
3036
|
+
"gen_ai.prompt.0.content": instrumentation.globalAgentOptions.get(this)?.instructions,
|
|
3019
3037
|
"gen_ai.prompt.0.role": "system"
|
|
3020
3038
|
});
|
|
3021
3039
|
promptIndex++;
|
|
@@ -3659,7 +3677,12 @@ var LaminarTracer = class {
|
|
|
3659
3677
|
return this._tracer.startActiveSpan(name, {}, contextToUse, wrapped(optionsOrFn));
|
|
3660
3678
|
}
|
|
3661
3679
|
if (typeof contextOrFn === "function") {
|
|
3662
|
-
return this._tracer.startActiveSpan(
|
|
3680
|
+
return this._tracer.startActiveSpan(
|
|
3681
|
+
name,
|
|
3682
|
+
optionsOrFn,
|
|
3683
|
+
contextToUse,
|
|
3684
|
+
wrapped(contextOrFn)
|
|
3685
|
+
);
|
|
3663
3686
|
}
|
|
3664
3687
|
return this._tracer.startActiveSpan(
|
|
3665
3688
|
name,
|
|
@@ -4301,7 +4324,7 @@ var Evaluation = class {
|
|
|
4301
4324
|
async evaluateDatapoint(evalId, datapoint, index) {
|
|
4302
4325
|
return observe({ name: "evaluation", traceType: "EVALUATION" }, async () => {
|
|
4303
4326
|
import_api15.trace.getSpan(LaminarContextManager.getContext()).setAttribute(SPAN_TYPE, "EVALUATION");
|
|
4304
|
-
const executorSpan = Laminar.
|
|
4327
|
+
const executorSpan = Laminar.startSpan({
|
|
4305
4328
|
name: "executor",
|
|
4306
4329
|
input: datapoint.data
|
|
4307
4330
|
});
|
|
@@ -4326,7 +4349,11 @@ var Evaluation = class {
|
|
|
4326
4349
|
});
|
|
4327
4350
|
const output = await Laminar.withSpan(
|
|
4328
4351
|
executorSpan,
|
|
4329
|
-
async () =>
|
|
4352
|
+
async () => {
|
|
4353
|
+
const result = await this.executor(datapoint.data);
|
|
4354
|
+
Laminar.setSpanOutput(result);
|
|
4355
|
+
return result;
|
|
4356
|
+
},
|
|
4330
4357
|
true
|
|
4331
4358
|
);
|
|
4332
4359
|
const target = datapoint.target;
|