@lmnr-ai/lmnr 0.8.47 → 0.8.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +2 -2
- package/dist/cli.d.cts +1 -1
- package/dist/cli.d.mts +1 -1
- package/dist/cli.mjs +2 -2
- package/dist/{evaluations-C3hpKJYm.d.mts → evaluations-4bQ4UuTj.d.mts} +13 -1
- package/dist/{evaluations-NLPG-oag.d.cts → evaluations-CbHnDCAY.d.cts} +13 -1
- package/dist/{file-utils-BHkRwkjp.mjs → file-utils-Ct8PiWkV.mjs} +2 -2
- package/dist/{file-utils-BHkRwkjp.mjs.map → file-utils-Ct8PiWkV.mjs.map} +1 -1
- package/dist/{file-utils-D8HN2emo.cjs → file-utils-N65ihgSU.cjs} +2 -2
- package/dist/{file-utils-D8HN2emo.cjs.map → file-utils-N65ihgSU.cjs.map} +1 -1
- package/dist/index.cjs +223 -75
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.mjs +223 -75
- package/dist/index.mjs.map +1 -1
- package/dist/{utils-BaY7eFxS.mjs → utils-CbvnF9B9.mjs} +3 -3
- package/dist/{utils-BaY7eFxS.mjs.map → utils-CbvnF9B9.mjs.map} +1 -1
- package/dist/{utils-Cibdqbyh.cjs → utils-D8kgcFxb.cjs} +3 -3
- package/dist/{utils-Cibdqbyh.cjs.map → utils-D8kgcFxb.cjs.map} +1 -1
- package/package.json +4 -3
package/dist/index.d.cts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { C as __exportAll, S as TracingLevel, _ as MaskInputOptions, a as HumanEvaluator, b as SpanType, c as InitializeOptions, d as LaminarClient, f as Dataset, g as LaminarSpanContext, h as Event, i as EvaluatorFunctionReturn, l as EvaluationDataset, m as EvaluationDatapointDatasetLink, o as evaluate, p as EvaluationDatapoint, r as EvaluatorFunction, s as StringUUID, t as Datapoint, u as LaminarDataset, v as PushDatapointsResponse, x as TraceType, y as SessionRecordingOptions } from "./evaluations-
|
|
1
|
+
import { C as __exportAll, S as TracingLevel, _ as MaskInputOptions, a as HumanEvaluator, b as SpanType, c as InitializeOptions, d as LaminarClient, f as Dataset, g as LaminarSpanContext, h as Event, i as EvaluatorFunctionReturn, l as EvaluationDataset, m as EvaluationDatapointDatasetLink, o as evaluate, p as EvaluationDatapoint, r as EvaluatorFunction, s as StringUUID, t as Datapoint, u as LaminarDataset, v as PushDatapointsResponse, x as TraceType, y as SessionRecordingOptions } from "./evaluations-CbHnDCAY.cjs";
|
|
2
2
|
import { ServerResponse } from "node:http";
|
|
3
3
|
import { ReadableSpan, SpanExporter, SpanProcessor, TimedEvent } from "@opentelemetry/sdk-trace-base";
|
|
4
4
|
import { AttributeValue, Attributes, Context, Exception, HrTime, Link, Span, Span as Span$1, SpanContext, SpanKind, SpanStatus, TimeInput, Tracer, TracerProvider } from "@opentelemetry/api";
|
package/dist/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { S as TracingLevel, _ as MaskInputOptions, a as HumanEvaluator, b as SpanType, c as InitializeOptions, d as LaminarClient, f as Dataset, g as LaminarSpanContext, h as Event, i as EvaluatorFunctionReturn, l as EvaluationDataset, m as EvaluationDatapointDatasetLink, o as evaluate, p as EvaluationDatapoint, r as EvaluatorFunction, s as StringUUID, t as Datapoint, u as LaminarDataset, v as PushDatapointsResponse, x as TraceType, y as SessionRecordingOptions } from "./evaluations-
|
|
1
|
+
import { S as TracingLevel, _ as MaskInputOptions, a as HumanEvaluator, b as SpanType, c as InitializeOptions, d as LaminarClient, f as Dataset, g as LaminarSpanContext, h as Event, i as EvaluatorFunctionReturn, l as EvaluationDataset, m as EvaluationDatapointDatasetLink, o as evaluate, p as EvaluationDatapoint, r as EvaluatorFunction, s as StringUUID, t as Datapoint, u as LaminarDataset, v as PushDatapointsResponse, x as TraceType, y as SessionRecordingOptions } from "./evaluations-4bQ4UuTj.mjs";
|
|
2
2
|
import { AttributeValue, Attributes, Context, Exception, HrTime, Link, Span, Span as Span$1, SpanContext, SpanKind, SpanStatus, TimeInput, Tracer, TracerProvider } from "@opentelemetry/api";
|
|
3
3
|
import { Instrumentation, InstrumentationBase } from "@opentelemetry/instrumentation";
|
|
4
4
|
import { ReadableSpan, SpanExporter, SpanProcessor, TimedEvent } from "@opentelemetry/sdk-trace-base";
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { r as __require } from "./rolldown-runtime-2rV9d50f.mjs";
|
|
2
|
-
import { A as SPAN_INSTRUMENTATION_SOURCE, B as LaminarClient, C as PARENT_SPAN_IDS_PATH, D as SPAN_INPUT, E as SPAN_IDS_PATH, F as SPAN_TYPE, H as DEBUG_SESSION_FILE, I as TRACE_HAS_BROWSER_SESSION, L as TRACE_TYPE, M as SPAN_OUTPUT, N as SPAN_PATH, O as SPAN_INSTRUMENTATION_SCOPE_NAME, P as SPAN_SDK_VERSION, R as USER_ID, S as LaminarAttributes, T as SESSION_ID, U as TracingLevel, V as DEBUG_SESSION_DIR, W as errorMessage, _ as tryToOtelSpanContext, a as getFrontendUrl, b as ASSOCIATION_PROPERTIES_OVERRIDES, c as loadEnv, d as normalizeOtelSpanId, f as normalizeOtelTraceId, g as truncateSpanPayload, h as parseOtelHeaders, j as SPAN_LANGUAGE_VERSION, k as SPAN_INSTRUMENTATION_SCOPE_VERSION, l as metadataToAttributes, m as otelTraceIdToUUID, n as Semaphore, o as getOtelEnvVar, p as otelSpanIdToUUID, r as deserializeLaminarSpanContext, s as initializeLogger, t as NIL_UUID, u as newUUID, v as validateTracingConfig, w as PARENT_SPAN_PATH, x as HUMAN_EVALUATOR_OPTIONS, y as ASSOCIATION_PROPERTIES, z as version } from "./utils-
|
|
2
|
+
import { A as SPAN_INSTRUMENTATION_SOURCE, B as LaminarClient, C as PARENT_SPAN_IDS_PATH, D as SPAN_INPUT, E as SPAN_IDS_PATH, F as SPAN_TYPE, H as DEBUG_SESSION_FILE, I as TRACE_HAS_BROWSER_SESSION, L as TRACE_TYPE, M as SPAN_OUTPUT, N as SPAN_PATH, O as SPAN_INSTRUMENTATION_SCOPE_NAME, P as SPAN_SDK_VERSION, R as USER_ID, S as LaminarAttributes, T as SESSION_ID, U as TracingLevel, V as DEBUG_SESSION_DIR, W as errorMessage, _ as tryToOtelSpanContext, a as getFrontendUrl, b as ASSOCIATION_PROPERTIES_OVERRIDES, c as loadEnv, d as normalizeOtelSpanId, f as normalizeOtelTraceId, g as truncateSpanPayload, h as parseOtelHeaders, j as SPAN_LANGUAGE_VERSION, k as SPAN_INSTRUMENTATION_SCOPE_VERSION, l as metadataToAttributes, m as otelTraceIdToUUID, n as Semaphore, o as getOtelEnvVar, p as otelSpanIdToUUID, r as deserializeLaminarSpanContext, s as initializeLogger, t as NIL_UUID, u as newUUID, v as validateTracingConfig, w as PARENT_SPAN_PATH, x as HUMAN_EVALUATOR_OPTIONS, y as ASSOCIATION_PROPERTIES, z as version } from "./utils-CbvnF9B9.mjs";
|
|
3
3
|
import { n as TRACEPARENT_HEADER, t as LAMINAR_SPAN_CONTEXT_HEADER } from "./consts-CTTu3jhI.mjs";
|
|
4
4
|
import * as path from "path";
|
|
5
5
|
import { DiagConsoleLogger, DiagLogLevel, ROOT_CONTEXT, SpanKind, SpanStatusCode, TraceFlags, context, createContextKey, diag, isSpanContextValid, trace } from "@opentelemetry/api";
|
|
@@ -176,7 +176,7 @@ var LaminarDataset = class extends EvaluationDataset {
|
|
|
176
176
|
*/
|
|
177
177
|
async push(paths, recursive = false) {
|
|
178
178
|
if (!this.client) throw new Error("Client not set");
|
|
179
|
-
const { loadFromPaths } = await import("./file-utils-
|
|
179
|
+
const { loadFromPaths } = await import("./file-utils-Ct8PiWkV.mjs").then((n) => n.t);
|
|
180
180
|
const data = await loadFromPaths(Array.isArray(paths) ? paths : [paths], recursive);
|
|
181
181
|
if (data.length === 0) {
|
|
182
182
|
console.warn("No data to push. Skipping");
|
|
@@ -4586,7 +4586,7 @@ const LLM_USAGE_TOTAL_TOKENS = "llm.usage.total_tokens";
|
|
|
4586
4586
|
/**
|
|
4587
4587
|
* Set an attribute on a span only if the value is non-null, non-undefined, and non-empty.
|
|
4588
4588
|
*/
|
|
4589
|
-
function safeSetAttribute$
|
|
4589
|
+
function safeSetAttribute$2(span, key, value) {
|
|
4590
4590
|
if (value === null || value === void 0) return;
|
|
4591
4591
|
if (typeof value === "string" && value.length === 0) return;
|
|
4592
4592
|
if (Array.isArray(value) && value.length === 0) return;
|
|
@@ -4648,44 +4648,44 @@ function extractToolDefinitions(config) {
|
|
|
4648
4648
|
/**
|
|
4649
4649
|
* Set request attributes on a span from generateContent params.
|
|
4650
4650
|
*/
|
|
4651
|
-
function setRequestAttributes$
|
|
4652
|
-
safeSetAttribute$
|
|
4653
|
-
safeSetAttribute$
|
|
4651
|
+
function setRequestAttributes$2(span, params, traceContent) {
|
|
4652
|
+
safeSetAttribute$2(span, LLM_SYSTEM, "gemini");
|
|
4653
|
+
safeSetAttribute$2(span, LLM_REQUEST_MODEL, params?.model);
|
|
4654
4654
|
const config = params?.config;
|
|
4655
4655
|
if (config) {
|
|
4656
|
-
safeSetAttribute$
|
|
4657
|
-
safeSetAttribute$
|
|
4658
|
-
safeSetAttribute$
|
|
4659
|
-
safeSetAttribute$
|
|
4660
|
-
safeSetAttribute$
|
|
4661
|
-
safeSetAttribute$
|
|
4662
|
-
safeSetAttribute$
|
|
4663
|
-
safeSetAttribute$
|
|
4664
|
-
if (config.stopSequences && config.stopSequences.length > 0) safeSetAttribute$
|
|
4656
|
+
safeSetAttribute$2(span, "gen_ai.request.temperature", config.temperature);
|
|
4657
|
+
safeSetAttribute$2(span, "gen_ai.request.top_p", config.topP);
|
|
4658
|
+
safeSetAttribute$2(span, "gen_ai.request.top_k", config.topK);
|
|
4659
|
+
safeSetAttribute$2(span, "gen_ai.request.max_tokens", config.maxOutputTokens);
|
|
4660
|
+
safeSetAttribute$2(span, "gen_ai.request.frequency_penalty", config.frequencyPenalty);
|
|
4661
|
+
safeSetAttribute$2(span, "gen_ai.request.presence_penalty", config.presencePenalty);
|
|
4662
|
+
safeSetAttribute$2(span, "gen_ai.request.seed", config.seed);
|
|
4663
|
+
safeSetAttribute$2(span, "gen_ai.request.choice_count", config.candidateCount);
|
|
4664
|
+
if (config.stopSequences && config.stopSequences.length > 0) safeSetAttribute$2(span, "gen_ai.request.stop_sequences", JSON.stringify(config.stopSequences));
|
|
4665
4665
|
const outputSchema = config.responseSchema ?? config.responseJsonSchema;
|
|
4666
|
-
if (outputSchema) safeSetAttribute$
|
|
4666
|
+
if (outputSchema) safeSetAttribute$2(span, "gen_ai.request.structured_output_schema", JSON.stringify(toDict(outputSchema)));
|
|
4667
4667
|
const toolDefs = extractToolDefinitions(config);
|
|
4668
|
-
if (toolDefs.length > 0) safeSetAttribute$
|
|
4668
|
+
if (toolDefs.length > 0) safeSetAttribute$2(span, "gen_ai.tool.definitions", JSON.stringify(toolDefs));
|
|
4669
4669
|
}
|
|
4670
4670
|
if (traceContent) {
|
|
4671
4671
|
const messages = contentToMessages(params?.contents, config?.systemInstruction);
|
|
4672
|
-
if (messages.length > 0) safeSetAttribute$
|
|
4672
|
+
if (messages.length > 0) safeSetAttribute$2(span, "gen_ai.input.messages", JSON.stringify(messages));
|
|
4673
4673
|
}
|
|
4674
4674
|
}
|
|
4675
4675
|
/**
|
|
4676
4676
|
* Set response attributes on a span from a GenerateContentResponse.
|
|
4677
4677
|
*/
|
|
4678
|
-
function setResponseAttributes$
|
|
4678
|
+
function setResponseAttributes$2(span, response, traceContent) {
|
|
4679
4679
|
if (!response) return;
|
|
4680
|
-
safeSetAttribute$
|
|
4680
|
+
safeSetAttribute$2(span, LLM_RESPONSE_MODEL, response.modelVersion);
|
|
4681
4681
|
const usage = response.usageMetadata;
|
|
4682
4682
|
if (usage) {
|
|
4683
|
-
safeSetAttribute$
|
|
4683
|
+
safeSetAttribute$2(span, "gen_ai.usage.input_tokens", usage.promptTokenCount);
|
|
4684
4684
|
const outputTokens = (usage.candidatesTokenCount ?? 0) + (usage.thoughtsTokenCount ?? 0);
|
|
4685
|
-
if (outputTokens > 0) safeSetAttribute$
|
|
4686
|
-
safeSetAttribute$
|
|
4687
|
-
safeSetAttribute$
|
|
4688
|
-
safeSetAttribute$
|
|
4685
|
+
if (outputTokens > 0) safeSetAttribute$2(span, "gen_ai.usage.output_tokens", outputTokens);
|
|
4686
|
+
safeSetAttribute$2(span, LLM_USAGE_TOTAL_TOKENS, usage.totalTokenCount);
|
|
4687
|
+
safeSetAttribute$2(span, "gen_ai.usage.reasoning_tokens", usage.thoughtsTokenCount);
|
|
4688
|
+
safeSetAttribute$2(span, "gen_ai.usage.cache_read_input_tokens", usage.cachedContentTokenCount);
|
|
4689
4689
|
}
|
|
4690
4690
|
if (traceContent && response.candidates) {
|
|
4691
4691
|
const outputMessages = [];
|
|
@@ -4712,7 +4712,7 @@ function setResponseAttributes$1(span, response, traceContent) {
|
|
|
4712
4712
|
...toolCalls.length > 0 ? { tool_calls: toolCalls } : {}
|
|
4713
4713
|
});
|
|
4714
4714
|
}
|
|
4715
|
-
if (outputMessages.length > 0) safeSetAttribute$
|
|
4715
|
+
if (outputMessages.length > 0) safeSetAttribute$2(span, "gen_ai.output.messages", JSON.stringify(outputMessages));
|
|
4716
4716
|
}
|
|
4717
4717
|
}
|
|
4718
4718
|
/**
|
|
@@ -4777,7 +4777,7 @@ function wrapStreamingResponse(span, asyncGen, traceContent) {
|
|
|
4777
4777
|
try {
|
|
4778
4778
|
if (modelVersion !== void 0 || role !== void 0 || accumulatedParts.length > 0 || sawUsageMetadata) {
|
|
4779
4779
|
const mergedParts = mergeTextParts(accumulatedParts);
|
|
4780
|
-
setResponseAttributes$
|
|
4780
|
+
setResponseAttributes$2(span, {
|
|
4781
4781
|
modelVersion,
|
|
4782
4782
|
candidates: [{ content: {
|
|
4783
4783
|
role: role ?? "model",
|
|
@@ -4902,11 +4902,11 @@ var GoogleGenAiInstrumentation = class extends InstrumentationBase {
|
|
|
4902
4902
|
name: "gemini.generate_content",
|
|
4903
4903
|
spanType: "LLM"
|
|
4904
4904
|
});
|
|
4905
|
-
setRequestAttributes$
|
|
4905
|
+
setRequestAttributes$2(span, params, classPatchData.traceContent);
|
|
4906
4906
|
return Laminar.withSpan(span, async () => {
|
|
4907
4907
|
try {
|
|
4908
4908
|
const response = await originalGenerateContent(params);
|
|
4909
|
-
setResponseAttributes$
|
|
4909
|
+
setResponseAttributes$2(span, response, classPatchData.traceContent);
|
|
4910
4910
|
return response;
|
|
4911
4911
|
} catch (error) {
|
|
4912
4912
|
span.setAttribute("error.type", error.constructor?.name ?? "Error");
|
|
@@ -4921,7 +4921,7 @@ var GoogleGenAiInstrumentation = class extends InstrumentationBase {
|
|
|
4921
4921
|
name: "gemini.generate_content_stream",
|
|
4922
4922
|
spanType: "LLM"
|
|
4923
4923
|
});
|
|
4924
|
-
setRequestAttributes$
|
|
4924
|
+
setRequestAttributes$2(span, params, classPatchData.traceContent);
|
|
4925
4925
|
try {
|
|
4926
4926
|
return wrapStreamingResponse(span, await originalGenerateContentStream(params), classPatchData.traceContent);
|
|
4927
4927
|
} catch (error) {
|
|
@@ -5754,12 +5754,12 @@ const TERMINAL_RESPONSE_EVENTS = new Set([
|
|
|
5754
5754
|
"response.failed"
|
|
5755
5755
|
]);
|
|
5756
5756
|
const ERROR_RESPONSE_STATUSES = new Set(["failed", "incomplete"]);
|
|
5757
|
-
const safeSetAttribute = (span, key, value) => {
|
|
5757
|
+
const safeSetAttribute$1 = (span, key, value) => {
|
|
5758
5758
|
if (value === null || value === void 0) return;
|
|
5759
5759
|
if (typeof value === "string" && value.length === 0) return;
|
|
5760
5760
|
span.setAttribute(key, value);
|
|
5761
5761
|
};
|
|
5762
|
-
const recordError = (span, error) => {
|
|
5762
|
+
const recordError$1 = (span, error) => {
|
|
5763
5763
|
span.setAttribute("error.type", error?.constructor?.name ?? "Error");
|
|
5764
5764
|
span.recordException(error);
|
|
5765
5765
|
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
@@ -5790,49 +5790,49 @@ const structuredOutputSchema = (kind, request) => {
|
|
|
5790
5790
|
const embeddingsInputMessages = (input) => {
|
|
5791
5791
|
return (Array.isArray(input) && !(input.length > 0 && typeof input[0] === "number") ? input : [input]).map((content) => ({ content }));
|
|
5792
5792
|
};
|
|
5793
|
-
const setRequestAttributes = (span, kind, request, traceContent) => {
|
|
5794
|
-
safeSetAttribute(span, "gen_ai.request.model", request?.model);
|
|
5795
|
-
safeSetAttribute(span, "llm.user", request?.user);
|
|
5793
|
+
const setRequestAttributes$1 = (span, kind, request, traceContent) => {
|
|
5794
|
+
safeSetAttribute$1(span, "gen_ai.request.model", request?.model);
|
|
5795
|
+
safeSetAttribute$1(span, "llm.user", request?.user);
|
|
5796
5796
|
if (kind === "embeddings") {
|
|
5797
|
-
if (traceContent && request?.input !== void 0) safeSetAttribute(span, "gen_ai.input.messages", JSON.stringify(embeddingsInputMessages(request.input)));
|
|
5797
|
+
if (traceContent && request?.input !== void 0) safeSetAttribute$1(span, "gen_ai.input.messages", JSON.stringify(embeddingsInputMessages(request.input)));
|
|
5798
5798
|
return;
|
|
5799
5799
|
}
|
|
5800
|
-
safeSetAttribute(span, "gen_ai.request.temperature", request?.temperature);
|
|
5801
|
-
safeSetAttribute(span, "gen_ai.request.top_p", request?.topP);
|
|
5802
|
-
safeSetAttribute(span, "gen_ai.request.frequency_penalty", request?.frequencyPenalty);
|
|
5803
|
-
safeSetAttribute(span, "gen_ai.request.presence_penalty", request?.presencePenalty);
|
|
5804
|
-
safeSetAttribute(span, "gen_ai.request.reasoning_effort", request?.reasoningEffort ?? request?.reasoning?.effort);
|
|
5805
|
-
safeSetAttribute(span, "gen_ai.request.max_tokens", kind === "chat" ? request?.maxTokens : request?.maxOutputTokens);
|
|
5800
|
+
safeSetAttribute$1(span, "gen_ai.request.temperature", request?.temperature);
|
|
5801
|
+
safeSetAttribute$1(span, "gen_ai.request.top_p", request?.topP);
|
|
5802
|
+
safeSetAttribute$1(span, "gen_ai.request.frequency_penalty", request?.frequencyPenalty);
|
|
5803
|
+
safeSetAttribute$1(span, "gen_ai.request.presence_penalty", request?.presencePenalty);
|
|
5804
|
+
safeSetAttribute$1(span, "gen_ai.request.reasoning_effort", request?.reasoningEffort ?? request?.reasoning?.effort);
|
|
5805
|
+
safeSetAttribute$1(span, "gen_ai.request.max_tokens", kind === "chat" ? request?.maxTokens : request?.maxOutputTokens);
|
|
5806
5806
|
const schema = structuredOutputSchema(kind, request);
|
|
5807
|
-
if (schema) safeSetAttribute(span, "gen_ai.request.structured_output_schema", JSON.stringify(schema));
|
|
5808
|
-
if (request?.stream) safeSetAttribute(span, "llm.is_streaming", true);
|
|
5807
|
+
if (schema) safeSetAttribute$1(span, "gen_ai.request.structured_output_schema", JSON.stringify(schema));
|
|
5808
|
+
if (request?.stream) safeSetAttribute$1(span, "llm.is_streaming", true);
|
|
5809
5809
|
if (!traceContent) return;
|
|
5810
|
-
if (Array.isArray(request?.tools) && request.tools.length > 0) safeSetAttribute(span, "gen_ai.tool.definitions", JSON.stringify(request.tools));
|
|
5810
|
+
if (Array.isArray(request?.tools) && request.tools.length > 0) safeSetAttribute$1(span, "gen_ai.tool.definitions", JSON.stringify(request.tools));
|
|
5811
5811
|
const messages = kind === "chat" ? request?.messages : responsesInputMessages(request);
|
|
5812
|
-
if (Array.isArray(messages) && messages.length > 0) safeSetAttribute(span, "gen_ai.input.messages", JSON.stringify(messages));
|
|
5812
|
+
if (Array.isArray(messages) && messages.length > 0) safeSetAttribute$1(span, "gen_ai.input.messages", JSON.stringify(messages));
|
|
5813
5813
|
};
|
|
5814
5814
|
const setUsageAttributes = (span, usage, inputKey, outputKey, inputCostKey, outputCostKey) => {
|
|
5815
5815
|
if (!usage) return;
|
|
5816
|
-
safeSetAttribute(span, "gen_ai.usage.input_tokens", usage[inputKey]);
|
|
5817
|
-
safeSetAttribute(span, "gen_ai.usage.output_tokens", usage[outputKey]);
|
|
5818
|
-
safeSetAttribute(span, "llm.usage.total_tokens", usage.totalTokens);
|
|
5816
|
+
safeSetAttribute$1(span, "gen_ai.usage.input_tokens", usage[inputKey]);
|
|
5817
|
+
safeSetAttribute$1(span, "gen_ai.usage.output_tokens", usage[outputKey]);
|
|
5818
|
+
safeSetAttribute$1(span, "llm.usage.total_tokens", usage.totalTokens);
|
|
5819
5819
|
const inputDetails = usage[`${inputKey}Details`];
|
|
5820
|
-
safeSetAttribute(span, "gen_ai.usage.cache_read_input_tokens", inputDetails?.cachedTokens);
|
|
5821
|
-
safeSetAttribute(span, "gen_ai.usage.cache_creation_input_tokens", inputDetails?.cacheWriteTokens);
|
|
5822
|
-
safeSetAttribute(span, "gen_ai.usage.reasoning_tokens", usage[`${outputKey}Details`]?.reasoningTokens);
|
|
5823
|
-
safeSetAttribute(span, "gen_ai.usage.cost", usage.cost);
|
|
5824
|
-
safeSetAttribute(span, "gen_ai.usage.input_cost", usage.costDetails?.[inputCostKey]);
|
|
5825
|
-
safeSetAttribute(span, "gen_ai.usage.output_cost", usage.costDetails?.[outputCostKey]);
|
|
5820
|
+
safeSetAttribute$1(span, "gen_ai.usage.cache_read_input_tokens", inputDetails?.cachedTokens);
|
|
5821
|
+
safeSetAttribute$1(span, "gen_ai.usage.cache_creation_input_tokens", inputDetails?.cacheWriteTokens);
|
|
5822
|
+
safeSetAttribute$1(span, "gen_ai.usage.reasoning_tokens", usage[`${outputKey}Details`]?.reasoningTokens);
|
|
5823
|
+
safeSetAttribute$1(span, "gen_ai.usage.cost", usage.cost);
|
|
5824
|
+
safeSetAttribute$1(span, "gen_ai.usage.input_cost", usage.costDetails?.[inputCostKey]);
|
|
5825
|
+
safeSetAttribute$1(span, "gen_ai.usage.output_cost", usage.costDetails?.[outputCostKey]);
|
|
5826
5826
|
};
|
|
5827
5827
|
/** Message to fail the span with, or `undefined` if the response succeeded. */
|
|
5828
5828
|
const responsesErrorMessage = (response) => {
|
|
5829
5829
|
if (!ERROR_RESPONSE_STATUSES.has(response?.status)) return void 0;
|
|
5830
5830
|
return response.error?.message ?? response.incompleteDetails?.reason ?? response.status;
|
|
5831
5831
|
};
|
|
5832
|
-
const setResponseAttributes = (span, kind, response, traceContent) => {
|
|
5832
|
+
const setResponseAttributes$1 = (span, kind, response, traceContent) => {
|
|
5833
5833
|
if (!response) return;
|
|
5834
|
-
safeSetAttribute(span, "gen_ai.response.id", response.id);
|
|
5835
|
-
safeSetAttribute(span, "gen_ai.response.model", response.model);
|
|
5834
|
+
safeSetAttribute$1(span, "gen_ai.response.id", response.id);
|
|
5835
|
+
safeSetAttribute$1(span, "gen_ai.response.model", response.model);
|
|
5836
5836
|
if (kind === "responses") {
|
|
5837
5837
|
setUsageAttributes(span, response.usage, "inputTokens", "outputTokens", "upstreamInferenceInputCost", "upstreamInferenceOutputCost");
|
|
5838
5838
|
const error = responsesErrorMessage(response);
|
|
@@ -5846,7 +5846,7 @@ const setResponseAttributes = (span, kind, response, traceContent) => {
|
|
|
5846
5846
|
} else setUsageAttributes(span, response.usage, "promptTokens", "completionTokens", "upstreamInferencePromptCost", "upstreamInferenceCompletionsCost");
|
|
5847
5847
|
if (!traceContent || kind === "embeddings") return;
|
|
5848
5848
|
const output = kind === "chat" ? response.choices : response.output;
|
|
5849
|
-
if (Array.isArray(output) && output.length > 0) safeSetAttribute(span, "gen_ai.output.messages", JSON.stringify(output));
|
|
5849
|
+
if (Array.isArray(output) && output.length > 0) safeSetAttribute$1(span, "gen_ai.output.messages", JSON.stringify(output));
|
|
5850
5850
|
};
|
|
5851
5851
|
const aggregateChatChunks = (chunks) => {
|
|
5852
5852
|
const result = {
|
|
@@ -5915,11 +5915,11 @@ const wrapStream = (span, kind, stream, traceContent) => {
|
|
|
5915
5915
|
yield chunk;
|
|
5916
5916
|
}
|
|
5917
5917
|
} catch (error) {
|
|
5918
|
-
recordError(span, error);
|
|
5918
|
+
recordError$1(span, error);
|
|
5919
5919
|
throw error;
|
|
5920
5920
|
} finally {
|
|
5921
5921
|
try {
|
|
5922
|
-
setResponseAttributes(span, kind, kind === "chat" ? aggregateChatChunks(chunks) : responseFromStreamEvents(chunks), traceContent);
|
|
5922
|
+
setResponseAttributes$1(span, kind, kind === "chat" ? aggregateChatChunks(chunks) : responseFromStreamEvents(chunks), traceContent);
|
|
5923
5923
|
} finally {
|
|
5924
5924
|
span.end();
|
|
5925
5925
|
}
|
|
@@ -5931,7 +5931,7 @@ const wrapStream = (span, kind, stream, traceContent) => {
|
|
|
5931
5931
|
//#endregion
|
|
5932
5932
|
//#region src/opentelemetry-lib/instrumentation/openrouter/index.ts
|
|
5933
5933
|
const WRAPPED_SYMBOL = Symbol("lmnr.openrouter.wrapped");
|
|
5934
|
-
const PATCH_STATE_SYMBOL = Symbol("lmnr.openrouter.patch-state");
|
|
5934
|
+
const PATCH_STATE_SYMBOL$1 = Symbol("lmnr.openrouter.patch-state");
|
|
5935
5935
|
/** The method each resource sends on, and the key its request is wrapped in. */
|
|
5936
5936
|
const RESOURCE_METHODS = {
|
|
5937
5937
|
chat: {
|
|
@@ -5979,7 +5979,7 @@ var OpenRouterInstrumentation = class extends InstrumentationBase {
|
|
|
5979
5979
|
}
|
|
5980
5980
|
patchOpenRouterClass(OpenRouterClass) {
|
|
5981
5981
|
const proto = OpenRouterClass.prototype;
|
|
5982
|
-
const existing = proto[PATCH_STATE_SYMBOL];
|
|
5982
|
+
const existing = proto[PATCH_STATE_SYMBOL$1];
|
|
5983
5983
|
if (existing) {
|
|
5984
5984
|
existing.traceContent = this.traceContent;
|
|
5985
5985
|
return;
|
|
@@ -5991,18 +5991,18 @@ var OpenRouterInstrumentation = class extends InstrumentationBase {
|
|
|
5991
5991
|
embeddings: Object.getOwnPropertyDescriptor(proto, "embeddings"),
|
|
5992
5992
|
callModel: proto.callModel
|
|
5993
5993
|
};
|
|
5994
|
-
proto[PATCH_STATE_SYMBOL] = state;
|
|
5994
|
+
proto[PATCH_STATE_SYMBOL$1] = state;
|
|
5995
5995
|
for (const kind of RESOURCE_KINDS) patchResourceGetter(proto, kind, state);
|
|
5996
5996
|
if (typeof state.callModel === "function") patchCallModel(proto, state);
|
|
5997
5997
|
}
|
|
5998
5998
|
unpatchOpenRouterClass(OpenRouterClass) {
|
|
5999
5999
|
const proto = OpenRouterClass.prototype;
|
|
6000
|
-
const state = proto[PATCH_STATE_SYMBOL];
|
|
6000
|
+
const state = proto[PATCH_STATE_SYMBOL$1];
|
|
6001
6001
|
if (!state) return;
|
|
6002
6002
|
for (const kind of RESOURCE_KINDS) if (state[kind]) Object.defineProperty(proto, kind, state[kind]);
|
|
6003
6003
|
else delete proto[kind];
|
|
6004
6004
|
if (state.callModel) proto.callModel = state.callModel;
|
|
6005
|
-
delete proto[PATCH_STATE_SYMBOL];
|
|
6005
|
+
delete proto[PATCH_STATE_SYMBOL$1];
|
|
6006
6006
|
}
|
|
6007
6007
|
};
|
|
6008
6008
|
const patchResourceGetter = (proto, kind, state) => {
|
|
@@ -6035,15 +6035,15 @@ const patchResourceMethod = (resource, kind, state) => {
|
|
|
6035
6035
|
spanType: "LLM"
|
|
6036
6036
|
});
|
|
6037
6037
|
span.setAttribute("gen_ai.system", "openrouter");
|
|
6038
|
-
setRequestAttributes(span, kind, request?.[requestKey], traceContent);
|
|
6038
|
+
setRequestAttributes$1(span, kind, request?.[requestKey], traceContent);
|
|
6039
6039
|
try {
|
|
6040
6040
|
const response = await Laminar.withSpan(span, () => original(request, options));
|
|
6041
6041
|
if (isAsyncIterable(response)) return wrapStream(span, kind, response, traceContent);
|
|
6042
|
-
setResponseAttributes(span, kind, response, traceContent);
|
|
6042
|
+
setResponseAttributes$1(span, kind, response, traceContent);
|
|
6043
6043
|
span.end();
|
|
6044
6044
|
return response;
|
|
6045
6045
|
} catch (error) {
|
|
6046
|
-
recordError(span, error);
|
|
6046
|
+
recordError$1(span, error);
|
|
6047
6047
|
span.end();
|
|
6048
6048
|
throw error;
|
|
6049
6049
|
}
|
|
@@ -6068,7 +6068,7 @@ const patchCallModel = (proto, state) => {
|
|
|
6068
6068
|
instrumentModelResult(result, span, traceContent);
|
|
6069
6069
|
return result;
|
|
6070
6070
|
} catch (error) {
|
|
6071
|
-
recordError(span, error);
|
|
6071
|
+
recordError$1(span, error);
|
|
6072
6072
|
span.end();
|
|
6073
6073
|
throw error;
|
|
6074
6074
|
}
|
|
@@ -6094,7 +6094,7 @@ const wrapTool = (tool, parentSpan, traceContent) => {
|
|
|
6094
6094
|
if (traceContent) span.setOutput(result);
|
|
6095
6095
|
return result;
|
|
6096
6096
|
} catch (error) {
|
|
6097
|
-
recordError(span, error);
|
|
6097
|
+
recordError$1(span, error);
|
|
6098
6098
|
throw error;
|
|
6099
6099
|
}
|
|
6100
6100
|
}, true);
|
|
@@ -6124,11 +6124,11 @@ const instrumentModelResult = (result, parentSpan, traceContent) => {
|
|
|
6124
6124
|
return Laminar.withSpan(span, async () => {
|
|
6125
6125
|
try {
|
|
6126
6126
|
const response = await original(...args);
|
|
6127
|
-
setRequestAttributes(span, "responses", result.resolvedRequest, traceContent);
|
|
6128
|
-
setResponseAttributes(span, "responses", response, traceContent);
|
|
6127
|
+
setRequestAttributes$1(span, "responses", result.resolvedRequest, traceContent);
|
|
6128
|
+
setResponseAttributes$1(span, "responses", response, traceContent);
|
|
6129
6129
|
return response;
|
|
6130
6130
|
} catch (error) {
|
|
6131
|
-
recordError(span, error);
|
|
6131
|
+
recordError$1(span, error);
|
|
6132
6132
|
throw error;
|
|
6133
6133
|
}
|
|
6134
6134
|
}, true);
|
|
@@ -6144,7 +6144,7 @@ const instrumentModelResult = (result, parentSpan, traceContent) => {
|
|
|
6144
6144
|
if (traceContent && result.finalResponse?.output) parentSpan.setOutput(result.finalResponse.output);
|
|
6145
6145
|
parentSpan.end();
|
|
6146
6146
|
}, (error) => {
|
|
6147
|
-
recordError(parentSpan, error);
|
|
6147
|
+
recordError$1(parentSpan, error);
|
|
6148
6148
|
parentSpan.end();
|
|
6149
6149
|
});
|
|
6150
6150
|
return execution;
|
|
@@ -6995,6 +6995,140 @@ const LaminarTemporalInterceptors = {
|
|
|
6995
6995
|
ActivityInterceptorFactory
|
|
6996
6996
|
};
|
|
6997
6997
|
//#endregion
|
|
6998
|
+
//#region src/opentelemetry-lib/instrumentation/typesafe/utils.ts
|
|
6999
|
+
const safeSetAttribute = (span, key, value) => {
|
|
7000
|
+
if (value === null || value === void 0) return;
|
|
7001
|
+
if (typeof value === "string" && value.length === 0) return;
|
|
7002
|
+
span.setAttribute(key, value);
|
|
7003
|
+
};
|
|
7004
|
+
const recordError = (span, error) => {
|
|
7005
|
+
span.setAttribute("error.type", error?.constructor?.name ?? "Error");
|
|
7006
|
+
span.recordException(error);
|
|
7007
|
+
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
7008
|
+
};
|
|
7009
|
+
const setRequestAttributes = (span, request, defaultModel, traceContent) => {
|
|
7010
|
+
safeSetAttribute(span, "gen_ai.request.model", request?.model ?? defaultModel);
|
|
7011
|
+
if (!traceContent) return;
|
|
7012
|
+
const state = request?.state;
|
|
7013
|
+
if (state !== null && state !== void 0) {
|
|
7014
|
+
const content = typeof state === "string" ? state : JSON.stringify(state);
|
|
7015
|
+
safeSetAttribute(span, "gen_ai.input.messages", JSON.stringify([{
|
|
7016
|
+
role: "user",
|
|
7017
|
+
content
|
|
7018
|
+
}]));
|
|
7019
|
+
}
|
|
7020
|
+
const questions = request?.questions;
|
|
7021
|
+
if (questions && Object.keys(questions).length > 0) safeSetAttribute(span, "gen_ai.request.structured_output_schema", JSON.stringify(questions));
|
|
7022
|
+
};
|
|
7023
|
+
/** `body` is the raw `/v1/systemone` JSON: `{ model, usage, answers }`. */
|
|
7024
|
+
const setResponseAttributes = (span, body, traceContent) => {
|
|
7025
|
+
if (!body) return;
|
|
7026
|
+
safeSetAttribute(span, "gen_ai.response.model", body.model);
|
|
7027
|
+
safeSetAttribute(span, "gen_ai.usage.input_tokens", body.usage?.input_tokens);
|
|
7028
|
+
safeSetAttribute(span, "gen_ai.usage.output_tokens", body.usage?.output_tokens);
|
|
7029
|
+
if (body.answers && traceContent) safeSetAttribute(span, "gen_ai.output.messages", JSON.stringify([{
|
|
7030
|
+
role: "assistant",
|
|
7031
|
+
content: JSON.stringify(body.answers)
|
|
7032
|
+
}]));
|
|
7033
|
+
};
|
|
7034
|
+
//#endregion
|
|
7035
|
+
//#region src/opentelemetry-lib/instrumentation/typesafe/index.ts
|
|
7036
|
+
const PATCH_STATE_SYMBOL = Symbol("lmnr.typesafe.patch-state");
|
|
7037
|
+
var TypeSafeInstrumentation = class extends InstrumentationBase {
|
|
7038
|
+
constructor(config) {
|
|
7039
|
+
super("@lmnr/typesafe-instrumentation", version, { enabled: true });
|
|
7040
|
+
this.traceContent = config?.traceContent ?? true;
|
|
7041
|
+
}
|
|
7042
|
+
init() {
|
|
7043
|
+
return new InstrumentationNodeModuleDefinition("@typesafe-ai/sdk", [">=0.1.0"], this.patch.bind(this), this.unpatch.bind(this));
|
|
7044
|
+
}
|
|
7045
|
+
manuallyInstrument(typeSafeModule) {
|
|
7046
|
+
diag.debug("Manually instrumenting @typesafe-ai/sdk");
|
|
7047
|
+
const TypeSafeClientClass = this.resolveClientClass(typeSafeModule);
|
|
7048
|
+
if (TypeSafeClientClass) this.patchClientClass(TypeSafeClientClass);
|
|
7049
|
+
else diag.warn("Could not find TypeSafeClient class in typesafe manual instrumentation input. Pass either the TypeSafeClient class or the module object with a TypeSafeClient export.");
|
|
7050
|
+
}
|
|
7051
|
+
patch(moduleExports) {
|
|
7052
|
+
diag.debug("Patching @typesafe-ai/sdk");
|
|
7053
|
+
const TypeSafeClientClass = this.resolveClientClass(moduleExports);
|
|
7054
|
+
if (TypeSafeClientClass) this.patchClientClass(TypeSafeClientClass);
|
|
7055
|
+
return moduleExports;
|
|
7056
|
+
}
|
|
7057
|
+
unpatch(moduleExports) {
|
|
7058
|
+
diag.debug("Unpatching @typesafe-ai/sdk");
|
|
7059
|
+
const TypeSafeClientClass = this.resolveClientClass(moduleExports);
|
|
7060
|
+
if (TypeSafeClientClass) this.unpatchClientClass(TypeSafeClientClass);
|
|
7061
|
+
}
|
|
7062
|
+
resolveClientClass(moduleOrClass) {
|
|
7063
|
+
if (typeof moduleOrClass === "function") return moduleOrClass;
|
|
7064
|
+
if (typeof moduleOrClass?.TypeSafeClient === "function") return moduleOrClass.TypeSafeClient;
|
|
7065
|
+
}
|
|
7066
|
+
patchClientClass(TypeSafeClientClass) {
|
|
7067
|
+
const proto = TypeSafeClientClass.prototype;
|
|
7068
|
+
const existing = proto[PATCH_STATE_SYMBOL];
|
|
7069
|
+
if (existing) {
|
|
7070
|
+
existing.traceContent = this.traceContent;
|
|
7071
|
+
return;
|
|
7072
|
+
}
|
|
7073
|
+
if (typeof proto.systemOne !== "function") {
|
|
7074
|
+
diag.warn("Could not patch TypeSafeClient.prototype.systemOne.");
|
|
7075
|
+
return;
|
|
7076
|
+
}
|
|
7077
|
+
const state = {
|
|
7078
|
+
traceContent: this.traceContent,
|
|
7079
|
+
systemOne: proto.systemOne
|
|
7080
|
+
};
|
|
7081
|
+
proto[PATCH_STATE_SYMBOL] = state;
|
|
7082
|
+
proto.systemOne = wrapSystemOne(state);
|
|
7083
|
+
}
|
|
7084
|
+
unpatchClientClass(TypeSafeClientClass) {
|
|
7085
|
+
const proto = TypeSafeClientClass.prototype;
|
|
7086
|
+
const state = proto[PATCH_STATE_SYMBOL];
|
|
7087
|
+
if (!state) return;
|
|
7088
|
+
proto.systemOne = state.systemOne;
|
|
7089
|
+
delete proto[PATCH_STATE_SYMBOL];
|
|
7090
|
+
}
|
|
7091
|
+
};
|
|
7092
|
+
const wrapSystemOne = (state) => function(request, options) {
|
|
7093
|
+
const original = state.systemOne;
|
|
7094
|
+
if (isTracingSuppressed(context.active())) return original.call(this, request, options);
|
|
7095
|
+
const { traceContent } = state;
|
|
7096
|
+
const span = Laminar.startSpan({
|
|
7097
|
+
name: "typesafe.system_one",
|
|
7098
|
+
spanType: "LLM"
|
|
7099
|
+
});
|
|
7100
|
+
span.setAttribute("gen_ai.system", "typesafe");
|
|
7101
|
+
setRequestAttributes(span, request, this?.defaultModel, traceContent);
|
|
7102
|
+
let promise;
|
|
7103
|
+
try {
|
|
7104
|
+
promise = context.with(trace.setSpan(context.active(), span), () => original.call(this, request, options));
|
|
7105
|
+
} catch (error) {
|
|
7106
|
+
recordError(span, error);
|
|
7107
|
+
span.end();
|
|
7108
|
+
throw error;
|
|
7109
|
+
}
|
|
7110
|
+
const onError = (error) => {
|
|
7111
|
+
recordError(span, error);
|
|
7112
|
+
span.end();
|
|
7113
|
+
};
|
|
7114
|
+
try {
|
|
7115
|
+
if (typeof promise?.asResponse === "function") promise.asResponse().then(async (response) => {
|
|
7116
|
+
try {
|
|
7117
|
+
setResponseAttributes(span, await response.clone().json(), traceContent);
|
|
7118
|
+
} catch {}
|
|
7119
|
+
span.end();
|
|
7120
|
+
}, onError);
|
|
7121
|
+
else if (typeof promise?.then === "function") promise.then((result) => {
|
|
7122
|
+
setResponseAttributes(span, result, traceContent);
|
|
7123
|
+
span.end();
|
|
7124
|
+
}, onError);
|
|
7125
|
+
else span.end();
|
|
7126
|
+
} catch {
|
|
7127
|
+
span.end();
|
|
7128
|
+
}
|
|
7129
|
+
return promise;
|
|
7130
|
+
};
|
|
7131
|
+
//#endregion
|
|
6998
7132
|
//#region src/opentelemetry-lib/tracing/instrumentations.ts
|
|
6999
7133
|
const logger$7 = initializeLogger();
|
|
7000
7134
|
/**
|
|
@@ -7078,6 +7212,7 @@ const initInstrumentations = (client, suppressContentTracing, sessionRecordingOp
|
|
|
7078
7212
|
instrumentations.push(new OpencodeInstrumentation());
|
|
7079
7213
|
instrumentations.push(new OpenAIAgentsInstrumentation());
|
|
7080
7214
|
instrumentations.push(new OpenRouterInstrumentation({ traceContent: !suppressContentTracing }));
|
|
7215
|
+
instrumentations.push(new TypeSafeInstrumentation({ traceContent: !suppressContentTracing }));
|
|
7081
7216
|
return instrumentations;
|
|
7082
7217
|
};
|
|
7083
7218
|
const manuallyInitInstrumentations = (client, instrumentModules, suppressContentTracing, sessionRecordingOptions) => {
|
|
@@ -7172,6 +7307,11 @@ const manuallyInitInstrumentations = (client, instrumentModules, suppressContent
|
|
|
7172
7307
|
instrumentations.push(openRouterInstrumentation);
|
|
7173
7308
|
openRouterInstrumentation.manuallyInstrument(instrumentModules.openrouter);
|
|
7174
7309
|
}
|
|
7310
|
+
if (instrumentModules?.typesafe) {
|
|
7311
|
+
const typeSafeInstrumentation = new TypeSafeInstrumentation({ traceContent: !suppressContentTracing });
|
|
7312
|
+
instrumentations.push(typeSafeInstrumentation);
|
|
7313
|
+
typeSafeInstrumentation.manuallyInstrument(instrumentModules.typesafe);
|
|
7314
|
+
}
|
|
7175
7315
|
if (instrumentModules?.kernel) {
|
|
7176
7316
|
const kernelInstrumentation = new KernelInstrumentation();
|
|
7177
7317
|
instrumentations.push(kernelInstrumentation);
|
|
@@ -9546,6 +9686,14 @@ var LaminarAiSdkTelemetry = class {
|
|
|
9546
9686
|
}
|
|
9547
9687
|
if (typeof event.functionId === "string") span.setAttribute("ai.functionId", event.functionId);
|
|
9548
9688
|
if (event.operationId === "ai.embed" || event.operationId === "ai.embedMany" || event.operationId === "ai.rerank") span.setAttribute(SPAN_TYPE, "LLM");
|
|
9689
|
+
const { userId, sessionId, tags, ...metadata } = {
|
|
9690
|
+
...event.toolsContext ?? {},
|
|
9691
|
+
...event.runtimeContext ?? {}
|
|
9692
|
+
};
|
|
9693
|
+
if (sessionId) span.setAttribute(SESSION_ID, sessionId);
|
|
9694
|
+
if (userId) span.setAttribute(USER_ID, userId);
|
|
9695
|
+
if (tags && Array.isArray(tags) && tags.every((tag) => typeof tag === "string")) span.setAttribute(`${ASSOCIATION_PROPERTIES}.tags`, tags);
|
|
9696
|
+
span.setAttributes(metadataToAttributes(metadata));
|
|
9549
9697
|
if (this.recordInputs) {
|
|
9550
9698
|
if (Array.isArray(event.messages) || event.system !== void 0 || event.instructions !== void 0) {
|
|
9551
9699
|
const msgs = verbatimStandardizedMessages(event);
|