@langfuse/core 5.4.1 → 5.5.0-beta.0

package/dist/index.mjs

@@ -276,7 +276,7 @@ var resetGlobalLogger = () => {
 // package.json
 var package_default = {
   name: "@langfuse/core",
-  version: "5.4.1",
+  version: "5.5.0-beta.0",
   description: "Core functions and utilities for Langfuse packages",
   type: "module",
   sideEffects: false,
@@ -878,6 +878,9 @@ var scim_exports = {};
 // src/api/api/resources/scoreConfigs/index.ts
 var scoreConfigs_exports = {};
 
+// src/api/api/resources/scoresV3/index.ts
+var scoresV3_exports = {};
+
 // src/api/api/resources/scores/index.ts
 var scores_exports = {};
 
@@ -892,6 +895,7 @@ var unstable_exports = {};
 __export(unstable_exports, {
   AccessDeniedError: () => AccessDeniedError2,
   BadRequestError: () => BadRequestError,
+  CodeEvaluatorSourceCodeLanguage: () => CodeEvaluatorSourceCodeLanguage,
   ConflictError: () => ConflictError,
   EvaluationRuleArrayOptionsFilterOperator: () => EvaluationRuleArrayOptionsFilterOperator,
   EvaluationRuleBooleanFilterOperator: () => EvaluationRuleBooleanFilterOperator,
@@ -906,6 +910,7 @@ __export(unstable_exports, {
   EvaluatorScope: () => EvaluatorScope,
   EvaluatorType: () => EvaluatorType,
   InternalServerError: () => InternalServerError,
+  LlmAsJudgeEvaluatorType: () => LlmAsJudgeEvaluatorType,
   MethodNotAllowedError: () => MethodNotAllowedError2,
   NotFoundError: () => NotFoundError2,
   PublicApiErrorCode: () => PublicApiErrorCode,
@@ -921,6 +926,7 @@ __export(unstable_exports, {
 // src/api/api/resources/unstable/resources/commons/index.ts
 var commons_exports2 = {};
 __export(commons_exports2, {
+  CodeEvaluatorSourceCodeLanguage: () => CodeEvaluatorSourceCodeLanguage,
   EvaluationRuleArrayOptionsFilterOperator: () => EvaluationRuleArrayOptionsFilterOperator,
   EvaluationRuleBooleanFilterOperator: () => EvaluationRuleBooleanFilterOperator,
   EvaluationRuleMappingSource: () => EvaluationRuleMappingSource,
@@ -937,7 +943,14 @@ __export(commons_exports2, {
 
 // src/api/api/resources/unstable/resources/commons/types/EvaluatorType.ts
 var EvaluatorType = {
-  LlmAsJudge: "llm_as_judge"
+  LlmAsJudge: "llm_as_judge",
+  Code: "code"
+};
+
+// src/api/api/resources/unstable/resources/commons/types/CodeEvaluatorSourceCodeLanguage.ts
+var CodeEvaluatorSourceCodeLanguage = {
+  Python: "PYTHON",
+  Typescript: "TYPESCRIPT"
 };
 
 // src/api/api/resources/unstable/resources/commons/types/EvaluatorScope.ts
@@ -1174,6 +1187,14 @@ var InternalServerError = class _InternalServerError extends LangfuseAPIError {
 
 // src/api/api/resources/unstable/resources/evaluationRules/index.ts
 var evaluationRules_exports = {};
+__export(evaluationRules_exports, {
+  LlmAsJudgeEvaluatorType: () => LlmAsJudgeEvaluatorType
+});
+
+// src/api/api/resources/unstable/resources/evaluationRules/types/LlmAsJudgeEvaluatorType.ts
+var LlmAsJudgeEvaluatorType = {
+  LlmAsJudge: "llm_as_judge"
+};
 
 // src/api/api/resources/unstable/resources/evaluators/index.ts
 var evaluators_exports = {};
@@ -11417,6 +11438,220 @@ var ScoreConfigs = class {
   }
 };
 
+// src/api/api/resources/scoresV3/client/Client.ts
+var ScoresV3 = class {
+  constructor(_options) {
+    this._options = _options;
+  }
+  /**
+   * Get a list of scores with a polymorphic `value` field (v3).
+   *
+   * This endpoint requires Langfuse v4 or later.
+   *
+   * The `value` field type depends on `dataType`:
+   * - `NUMERIC` → number
+   * - `BOOLEAN` → boolean
+   * - `CATEGORICAL`, `TEXT`, `CORRECTION` → string
+   *
+   * Use the `fields` parameter to include optional field groups beyond the
+   * default `core`. Unknown group names return HTTP 400.
+   *
+   * @param {LangfuseAPI.GetScoresV3Request} request
+   * @param {ScoresV3.RequestOptions} requestOptions - Request-specific configuration.
+   *
+   * @throws {@link LangfuseAPI.Error}
+   * @throws {@link LangfuseAPI.UnauthorizedError}
+   * @throws {@link LangfuseAPI.AccessDeniedError}
+   * @throws {@link LangfuseAPI.MethodNotAllowedError}
+   * @throws {@link LangfuseAPI.NotFoundError}
+   *
+   * @example
+   *     await client.scoresV3.getManyV3()
+   */
+  getManyV3(request = {}, requestOptions) {
+    return HttpResponsePromise.fromPromise(
+      this.__getManyV3(request, requestOptions)
+    );
+  }
+  async __getManyV3(request = {}, requestOptions) {
+    var _a2, _b, _c, _d, _e, _f, _g, _h;
+    const {
+      limit,
+      cursor,
+      fields,
+      id,
+      name,
+      source,
+      dataType,
+      environment,
+      configId,
+      queueId,
+      authorUserId,
+      value,
+      valueMin,
+      valueMax,
+      traceId,
+      sessionId,
+      observationId,
+      experimentId,
+      fromTimestamp,
+      toTimestamp
+    } = request;
+    const _queryParams = {};
+    if (limit != null) {
+      _queryParams["limit"] = limit.toString();
+    }
+    if (cursor != null) {
+      _queryParams["cursor"] = cursor;
+    }
+    if (fields != null) {
+      _queryParams["fields"] = fields;
+    }
+    if (id != null) {
+      _queryParams["id"] = id;
+    }
+    if (name != null) {
+      _queryParams["name"] = name;
+    }
+    if (source != null) {
+      _queryParams["source"] = source;
+    }
+    if (dataType != null) {
+      _queryParams["dataType"] = dataType;
+    }
+    if (environment != null) {
+      _queryParams["environment"] = environment;
+    }
+    if (configId != null) {
+      _queryParams["configId"] = configId;
+    }
+    if (queueId != null) {
+      _queryParams["queueId"] = queueId;
+    }
+    if (authorUserId != null) {
+      _queryParams["authorUserId"] = authorUserId;
+    }
+    if (value != null) {
+      _queryParams["value"] = value;
+    }
+    if (valueMin != null) {
+      _queryParams["valueMin"] = valueMin.toString();
+    }
+    if (valueMax != null) {
+      _queryParams["valueMax"] = valueMax.toString();
+    }
+    if (traceId != null) {
+      _queryParams["traceId"] = traceId;
+    }
+    if (sessionId != null) {
+      _queryParams["sessionId"] = sessionId;
+    }
+    if (observationId != null) {
+      _queryParams["observationId"] = observationId;
+    }
+    if (experimentId != null) {
+      _queryParams["experimentId"] = experimentId;
+    }
+    if (fromTimestamp != null) {
+      _queryParams["fromTimestamp"] = fromTimestamp;
+    }
+    if (toTimestamp != null) {
+      _queryParams["toTimestamp"] = toTimestamp;
+    }
+    let _headers = mergeHeaders(
+      (_a2 = this._options) == null ? void 0 : _a2.headers,
+      mergeOnlyDefinedHeaders({
+        Authorization: await this._getAuthorizationHeader(),
+        "X-Langfuse-Sdk-Name": (_c = requestOptions == null ? void 0 : requestOptions.xLangfuseSdkName) != null ? _c : (_b = this._options) == null ? void 0 : _b.xLangfuseSdkName,
+        "X-Langfuse-Sdk-Version": (_e = requestOptions == null ? void 0 : requestOptions.xLangfuseSdkVersion) != null ? _e : (_d = this._options) == null ? void 0 : _d.xLangfuseSdkVersion,
+        "X-Langfuse-Public-Key": (_g = requestOptions == null ? void 0 : requestOptions.xLangfusePublicKey) != null ? _g : (_f = this._options) == null ? void 0 : _f.xLangfusePublicKey
+      }),
+      requestOptions == null ? void 0 : requestOptions.headers
+    );
+    const _response = await fetcher({
+      url: url_exports.join(
+        (_h = await Supplier.get(this._options.baseUrl)) != null ? _h : await Supplier.get(this._options.environment),
+        "/api/public/v3/scores"
+      ),
+      method: "GET",
+      headers: _headers,
+      queryParameters: { ..._queryParams, ...requestOptions == null ? void 0 : requestOptions.queryParams },
+      timeoutMs: (requestOptions == null ? void 0 : requestOptions.timeoutInSeconds) != null ? requestOptions.timeoutInSeconds * 1e3 : 6e4,
+      maxRetries: requestOptions == null ? void 0 : requestOptions.maxRetries,
+      abortSignal: requestOptions == null ? void 0 : requestOptions.abortSignal
+    });
+    if (_response.ok) {
+      return {
+        data: _response.body,
+        rawResponse: _response.rawResponse
+      };
+    }
+    if (_response.error.reason === "status-code") {
+      switch (_response.error.statusCode) {
+        case 400:
+          throw new Error2(
+            _response.error.body,
+            _response.rawResponse
+          );
+        case 401:
+          throw new UnauthorizedError(
+            _response.error.body,
+            _response.rawResponse
+          );
+        case 403:
+          throw new AccessDeniedError(
+            _response.error.body,
+            _response.rawResponse
+          );
+        case 405:
+          throw new MethodNotAllowedError(
+            _response.error.body,
+            _response.rawResponse
+          );
+        case 404:
+          throw new NotFoundError(
+            _response.error.body,
+            _response.rawResponse
+          );
+        default:
+          throw new LangfuseAPIError({
+            statusCode: _response.error.statusCode,
+            body: _response.error.body,
+            rawResponse: _response.rawResponse
+          });
+      }
+    }
+    switch (_response.error.reason) {
+      case "non-json":
+        throw new LangfuseAPIError({
+          statusCode: _response.error.statusCode,
+          body: _response.error.rawBody,
+          rawResponse: _response.rawResponse
+        });
+      case "timeout":
+        throw new LangfuseAPITimeoutError(
+          "Timeout exceeded when calling GET /api/public/v3/scores."
+        );
+      case "unknown":
+        throw new LangfuseAPIError({
+          message: _response.error.errorMessage,
+          rawResponse: _response.rawResponse
+        });
+    }
+  }
+  async _getAuthorizationHeader() {
+    const username = await Supplier.get(this._options.username);
+    const password = await Supplier.get(this._options.password);
+    if (username != null && password != null) {
+      return BasicAuth.toAuthorizationHeader({
+        username,
+        password
+      });
+    }
+    return void 0;
+  }
+};
+
 // src/api/api/resources/scores/client/Client.ts
 var Scores = class {
   constructor(_options) {
@@ -12507,8 +12742,9 @@ var EvaluationRules = class {
    * - `evaluator.name` + `evaluator.scope` must identify an existing evaluator family returned by the evaluator endpoints
    * - Langfuse resolves that family to its latest version before saving the evaluation rule
    * - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId`
-   * - every evaluator prompt variable must be mapped exactly once
-   * - `expected_output` and `experiment_item_metadata` mappings are only valid for `target=experiment`
+   * - for `llm_as_judge` evaluators, every evaluator prompt variable must be mapped exactly once
+   * - for `code` evaluators, Langfuse uses the fixed code runtime mapping; omit `mapping` in create and update requests
+   * - for user-provided `llm_as_judge` mappings, `expected_output` and `experiment_item_metadata` are only valid for `target=experiment`
    * - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run
    * - at most 50 evaluation rules can be effectively active in one project at the same time
    *
@@ -12525,9 +12761,9 @@ var EvaluationRules = class {
    * Recovery guidance:
    * - `400 invalid_filter_value`: fix the filter `column` or `value` using `details.column`, `details.invalidValues`, and `details.allowedValues`
    * - `400 invalid_filter_value` with `details.column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response
-   * - `400 missing_variable_mapping`: fetch the evaluator again and make sure every variable in `variables` appears exactly once in `mapping`
+   * - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch the evaluator again and make sure every variable in `variables` appears exactly once in `mapping`
    * - `400 duplicate_variable_mapping`: remove repeated mappings for the same variable
-   * - `400 invalid_variable_mapping`: switch to a valid `source` for the selected `target`, or fix the variable name
+   * - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid `source` for the selected `target`, or fix the variable name
    * - `400 invalid_json_path`: remove or correct the `jsonPath`
    * - `422 evaluator_preflight_failed`: the selected evaluator cannot run with the resolved model configuration. Fix the evaluator/default model setup, then retry the create request.
    *
@@ -12554,7 +12790,8 @@ var EvaluationRules = class {
    *         name: "answer-correctness-live",
    *         evaluator: {
    *             name: "answer-correctness",
-   *             scope: "project"
+   *             scope: "project",
+   *             type: "llm_as_judge"
    *         },
    *         target: "observation",
    *         enabled: true,
@@ -12576,10 +12813,30 @@ var EvaluationRules = class {
    *
    * @example
    *     await client.unstable.evaluationRules.create({
+   *         name: "toxicity-code-live",
+   *         evaluator: {
+   *             name: "toxicity-detector",
+   *             scope: "project",
+   *             type: "code"
+   *         },
+   *         target: "observation",
+   *         enabled: true,
+   *         sampling: 1,
+   *         filter: [{
+   *                 type: "stringOptions",
+   *                 column: "type",
+   *                 operator: "any of",
+   *                 value: ["GENERATION"]
+   *             }]
+   *     })
+   *
+   * @example
+   *     await client.unstable.evaluationRules.create({
    *         name: "experiment-expected-output-match",
    *         evaluator: {
    *             name: "expected-output-match",
-   *             scope: "project"
+   *             scope: "project",
+   *             type: "llm_as_judge"
    *         },
    *         target: "experiment",
    *         enabled: true,
@@ -13039,18 +13296,19 @@ var EvaluationRules = class {
    * - switch to another evaluator
    * - adjust sampling
    * - change filters
-   * - update variable mappings
+   * - update LLM-as-judge variable mappings
    *
    * Important behavior:
    * - provide only the fields you want to change
    * - if you provide `evaluator`, Langfuse resolves that evaluator family to its latest version before saving
-   * - changing `target`, `filter`, or `mapping` must still produce a valid target-specific configuration
-   * - if you change `target`, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target
+   * - changing `target`, `filter`, or an LLM-as-judge `mapping` must still produce a valid target-specific configuration
+   * - if you change `target` for an LLM-as-judge rule, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target
+   * - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically
    * - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run
    * - if the update would move a non-active evaluation rule into the active state and the project already has 50 active evaluation rules, the API returns `409`
    *
    * Recovery guidance:
-   * - if the update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping`
+   * - if an LLM-as-judge update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping`
    * - if the update fails with `invalid_filter_value` after changing `target`, resend the request with a target-compatible `filter`
    *
    * @param {string} evaluationRuleId - Evaluation rule identifier.
@@ -13381,7 +13639,9 @@ var Evaluators = class {
   /**
    * Create an evaluator in the authenticated project.
    *
-   * Use evaluators to define **how** Langfuse should score data: the prompt, the expected structured output, and the optional model configuration.
+   * Use evaluators to define **how** Langfuse should score data.
+   * LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration.
+   * Code evaluators define source code and a runtime language.
    *
    * Naming behavior:
    * - If this is a new evaluator name in your project, Langfuse creates version `1`.
@@ -13394,10 +13654,15 @@ var Evaluators = class {
    * 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical.
    * 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`.
    *
+   * Code evaluator validation:
+   * - At creation, Langfuse only validates the request shape
+   * - The `sourceCode` itself is not executed here. It is first run (preflight-tested against a sample observation) when you link the evaluator to an evaluation rule, so runtime errors in the code surface at evaluation-rule creation, not at evaluator creation.
+   *
    * Recovery guidance:
    * - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request.
    * - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry.
-   * - `400` with `code=invalid_body` on `outputDefinition`: send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape.
+   * - `400` with `code=invalid_body` on `outputDefinition`: for `type=llm_as_judge`, send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape.
+   * - If `type` is omitted, Langfuse treats the request as `type=llm_as_judge` for backwards compatibility. New clients should send `type` explicitly.
    *
    * Unstable API note:
    * - This surface may evolve while the underlying evaluation data model is being redesigned.
@@ -13421,6 +13686,7 @@ var Evaluators = class {
    *
    * @example
    *     await client.unstable.evaluators.create({
+   *         type: "llm_as_judge",
    *         name: "answer-correctness",
    *         prompt: "You are grading an answer.\n\nInput:\n{{input}}\n\nOutput:\n{{output}}\n\nReturn a score between 0 and 1.\n",
    *         outputDefinition: {
@@ -13438,6 +13704,22 @@ var Evaluators = class {
    *             model: "gpt-4.1-mini"
    *         }
    *     })
+   *
+   * @example
+   *     await client.unstable.evaluators.create({
+   *         type: "code",
+   *         name: "exact-match",
+   *         sourceCode: "function evaluate(ctx: EvaluationContext): EvaluationResult {\n  const input = ctx.observation.input;\n  const matchesOutput =\n    input !== undefined && ctx.observation.output === input;\n\n  return {\n    scores: [\n      {\n        name: \"Exact match\",\n        value: matchesOutput,\n        dataType: \"BOOLEAN\",\n        comment: matchesOutput\n          ? \"Output exactly matches the input.\"\n          : \"Output does not match the input.\",\n      },\n    ],\n  };\n}\n",
+   *         sourceCodeLanguage: "TYPESCRIPT"
+   *     })
+   *
+   * @example
+   *     await client.unstable.evaluators.create({
+   *         type: "code",
+   *         name: "exact-match",
+   *         sourceCode: "def evaluate(ctx: EvaluationContext) -> EvaluationResult:\n    \"\"\"Evaluates one observation and returns one or more Langfuse scores.\"\"\"\n    input = ctx.observation.input\n    matches_output = input is not None and ctx.observation.output == input\n\n    return EvaluationResult(\n        scores=[\n            Score(\n                name=\"Exact match\",\n                value=matches_output,\n                data_type=\"BOOLEAN\",\n                comment=(\n                    \"Output exactly matches the input.\"\n                    if matches_output\n                    else \"Output does not match the input.\"\n                ),\n            )\n        ]\n    )\n",
+   *         sourceCodeLanguage: "PYTHON"
+   *     })
    */
   create(request, requestOptions) {
     return HttpResponsePromise.fromPromise(
@@ -13998,6 +14280,10 @@ var LangfuseAPIClient = class {
     var _a2;
     return (_a2 = this._scoreConfigs) != null ? _a2 : this._scoreConfigs = new ScoreConfigs(this._options);
   }
+  get scoresV3() {
+    var _a2;
+    return (_a2 = this._scoresV3) != null ? _a2 : this._scoresV3 = new ScoresV3(this._options);
+  }
   get scores() {
     var _a2;
     return (_a2 = this._scores) != null ? _a2 : this._scores = new Scores(this._options);
@@ -14683,6 +14969,7 @@ export {
   scim_exports as scim,
   scoreConfigs_exports as scoreConfigs,
   scores_exports as scores,
+  scoresV3_exports as scoresV3,
   serializeValue,
   sessions_exports as sessions,
   setLangfuseTraceIdInBaggage,