@kortyx/usage-telemetry 0.3.0 → 0.4.0

package/dist/node.js

@@ -23,8 +23,10 @@ __export(node_exports, {
   configureUsageTelemetry: () => configureUsageTelemetry,
   createNodeUsageTelemetry: () => createNodeUsageTelemetry,
   createUsageTelemetry: () => createUsageTelemetry,
+  flushUsageTelemetry: () => flushUsageTelemetry,
   getUsageTelemetry: () => getUsageTelemetry,
   publicUsageConfiguration: () => publicUsageConfiguration,
+  runtimeUsageContext: () => runtimeUsageContext,
   scale: () => scale,
   usageDisabled: () => usageDisabled
 });
@@ -76,6 +78,10 @@ var operations = [
   "studio.settings",
   "studio.update",
   "studio.ingest",
+  "cli.command",
+  "studio.eval.execution",
+  "studio.diagnostic",
+  "studio.table.layout",
   "cli.topology",
   "cli.eval"
 ];
@@ -90,9 +96,143 @@ var outcomes = [
   "ungraded"
 ];
 var publicCatalog = {
-  products: ["sdk", "oss_studio"],
+  products: ["sdk", "oss_studio", "cli"],
   operations,
   properties: {
+    measurement_version: ["2"],
+    measurement_kind: [
+      "invocation",
+      "stage",
+      "execution",
+      "attempt",
+      "chunk",
+      "output",
+      "update",
+      "poll",
+      "request",
+      "call",
+      "feature",
+      "observation"
+    ],
+    traffic_source: ["internal", "synthetic", "external", "unknown"],
+    layout_action: [
+      "visibility",
+      "show_all",
+      "reset",
+      "apply",
+      "reset_width",
+      "pin",
+      "unpin",
+      "resize",
+      "reorder"
+    ],
+    eval_phase: [
+      "params",
+      "setup",
+      "execute",
+      "responder",
+      "reference",
+      "grading",
+      "verification",
+      "cleanup",
+      "reporting"
+    ],
+    cli_resource: ["evals.runs", "evals.suites"],
+    retryable: "boolean",
+    retried: "boolean",
+    recovered: "boolean",
+    retry_attempt: ["1", "2-5", "6-20", "21-100", "101+"],
+    batch_size: ["1", "2-5", "6-20", "21-100", "101+"],
+    http_status_class: ["1xx", "2xx", "3xx", "4xx", "5xx"],
+    cause_category: [
+      "configuration",
+      "provider",
+      "validation",
+      "domain",
+      "workflow",
+      "persistence",
+      "request",
+      "transport",
+      "internal",
+      "unknown"
+    ],
+    cli_command: [
+      "studio.start",
+      "studio.stop",
+      "studio.restart",
+      "studio.status",
+      "studio.logs",
+      "studio.credentials",
+      "studio.reset",
+      "studio.prompts",
+      "studio.evals",
+      "studio.runs",
+      "studio.sessions",
+      "studio.interrupts",
+      "studio.workflows",
+      "studio.diagnostics",
+      "studio.inspect",
+      "studio.doctor",
+      "studio.catalogs",
+      "connections",
+      "evals.run",
+      "evals.list",
+      "topology.push",
+      "help",
+      "unknown"
+    ],
+    cli_action: [
+      "start",
+      "stop",
+      "restart",
+      "status",
+      "logs",
+      "credentials",
+      "reset",
+      "run",
+      "list",
+      "get",
+      "show",
+      "inspect",
+      "analyze",
+      "download",
+      "doctor",
+      "cancel",
+      "compare",
+      "create",
+      "update",
+      "delete",
+      "import",
+      "export",
+      "pull",
+      "push",
+      "add",
+      "remove",
+      "use",
+      "verify",
+      "versions",
+      "diff",
+      "validate",
+      "action",
+      "tag",
+      "untag",
+      "rollback",
+      "promote",
+      "archive",
+      "restore",
+      "test",
+      "copy",
+      "apply",
+      "wait",
+      "catalogs",
+      "help",
+      "unknown"
+    ],
+    dry_run: "boolean",
+    json_output: "boolean",
+    settings_override: "boolean",
+    case_timeout_override: "boolean",
+    run_timeout_override: "boolean",
     phase: [
       "admission",
       "execution",
@@ -237,6 +377,7 @@ var publicCatalog = {
       "pending",
       "expired"
     ],
+    match: ["all", "any"],
     filter: [
       "search",
       "status",
@@ -269,6 +410,7 @@ var publicCatalog = {
       "401",
       "403",
       "404",
+      "408",
       "409",
       "413",
       "422",
@@ -292,6 +434,47 @@ var publicCatalog = {
   },
   // Built-in contracts only; application codes and provider response text are excluded.
   errorCodes: [
+    "UNCLASSIFIED_FAILURE",
+    "HTTP_UNAUTHORIZED",
+    "HTTP_FORBIDDEN",
+    "HTTP_NOT_FOUND",
+    "HTTP_TIMEOUT",
+    "HTTP_CONFLICT",
+    "HTTP_TOO_LARGE",
+    "HTTP_RATE_LIMITED",
+    "HTTP_SERVER_ERROR",
+    "HTTP_INVALID_REQUEST",
+    "NETWORK_DNS_FAILED",
+    "NETWORK_CONNECTION_REFUSED",
+    "NETWORK_CONNECTION_RESET",
+    "NETWORK_TIMEOUT",
+    "NETWORK_TLS_FAILED",
+    "CLI_ARGUMENT_INVALID",
+    "CLI_EXIT_FAILED",
+    "STUDIO_NOT_CONFIGURED",
+    "STUDIO_RESPONSE_INVALID",
+    "EVAL_CONCURRENCY_CHANGED",
+    "EVAL_CONSUMER_FAILED",
+    "EVAL_CONTRACT_CHANGED",
+    "EVAL_DISCOVERY_REJECTED",
+    "EVAL_EXECUTION_CONFLICT",
+    "EVAL_EXECUTION_STORE_CHANGED",
+    "EVAL_EXECUTION_UNKNOWN",
+    "EVAL_EXECUTOR_BUSY",
+    "EVAL_EXECUTOR_FAILED",
+    "EVAL_EXECUTOR_REJECTED",
+    "EVAL_EXECUTOR_UNAUTHORIZED",
+    "EVAL_GRADING_FAILED",
+    "EVAL_GRADING_TIMEOUT",
+    "EVAL_JUDGE_UNAVAILABLE",
+    "EVAL_LEASE_LOST",
+    "EVAL_RETRY_DELAY_EXCEEDED",
+    "EVAL_SETUP_TIMEOUT",
+    "EVAL_TRANSPORT_UNAVAILABLE",
+    "EVAL_VERIFICATION_TIMEOUT",
+    "EVAL_RESULT_INVALID",
+    "EVAL_RUN_TIMEOUT",
+    "EVAL_SETTINGS_INVALID",
     "EXECUTION_FAILED",
     "INVALID_REQUEST",
     "INVALID_RESUME",
@@ -359,8 +542,8 @@ function scale(value) {
   return value <= 1 ? "1" : value <= 5 ? "2-5" : value <= 20 ? "6-20" : value <= 100 ? "21-100" : "101+";
 }
 
-// src/client.ts
-function property(value, key) {
+// src/failure.ts
+function usageErrorProperty(value, key) {
   try {
     if (!value || typeof value !== "object") return void 0;
     const descriptor = Object.getOwnPropertyDescriptor(value, key);
@@ -369,40 +552,201 @@ function property(value, key) {
     return void 0;
   }
 }
-function classifyUsageError(error, catalog = publicCatalog) {
-  const name = property(error, "name");
-  const code = property(error, "code") ?? property(property(error, "failure"), "code");
-  if (name === "AbortError" || code === "EXECUTION_CANCELLED")
+var networkCodes = {
+  ENOTFOUND: "NETWORK_DNS_FAILED",
+  EAI_AGAIN: "NETWORK_DNS_FAILED",
+  ECONNREFUSED: "NETWORK_CONNECTION_REFUSED",
+  ECONNRESET: "NETWORK_CONNECTION_RESET",
+  EPIPE: "NETWORK_CONNECTION_RESET",
+  ETIMEDOUT: "NETWORK_TIMEOUT",
+  UND_ERR_CONNECT_TIMEOUT: "NETWORK_TIMEOUT",
+  UND_ERR_HEADERS_TIMEOUT: "NETWORK_TIMEOUT",
+  UND_ERR_BODY_TIMEOUT: "NETWORK_TIMEOUT",
+  CERT_HAS_EXPIRED: "NETWORK_TLS_FAILED",
+  DEPTH_ZERO_SELF_SIGNED_CERT: "NETWORK_TLS_FAILED",
+  UNABLE_TO_VERIFY_LEAF_SIGNATURE: "NETWORK_TLS_FAILED"
+};
+function classifyUsageHttpStatus(status) {
+  if (!Number.isInteger(status) || status < 100 || status > 599)
+    return {
+      outcome: "failed",
+      properties: {
+        error_code: "UNCLASSIFIED_FAILURE",
+        error_category: "unknown"
+      }
+    };
+  if (status < 400)
+    return {
+      outcome: "success",
+      properties: { http_status_class: `${Math.floor(status / 100)}xx` }
+    };
+  const code = status === 401 ? "HTTP_UNAUTHORIZED" : status === 403 ? "HTTP_FORBIDDEN" : status === 404 ? "HTTP_NOT_FOUND" : status === 408 || status === 504 ? "HTTP_TIMEOUT" : status === 409 ? "HTTP_CONFLICT" : status === 413 ? "HTTP_TOO_LARGE" : status === 429 ? "HTTP_RATE_LIMITED" : status >= 500 ? "HTTP_SERVER_ERROR" : "HTTP_INVALID_REQUEST";
+  return {
+    outcome: "failed",
+    properties: {
+      error_code: code,
+      error_category: status >= 500 || status === 408 ? "transport" : "request",
+      http_status: String(status),
+      http_status_class: `${Math.floor(status / 100)}xx`,
+      retryable: status === 408 || status === 429 || status >= 500
+    }
+  };
+}
+function nativeExceptionName(value) {
+  try {
+    if (typeof DOMException !== "undefined")
+      return Object.getOwnPropertyDescriptor(
+        DOMException.prototype,
+        "name"
+      )?.get?.call(value);
+  } catch {
+  }
+  return void 0;
+}
+function classifyOne(error, catalog) {
+  const read = (key) => usageErrorProperty(error, key);
+  let name = read("name") ?? nativeExceptionName(error);
+  try {
+    if (name === void 0 && error instanceof SyntaxError)
+      name = "SyntaxError";
+  } catch {
+  }
+  const code = read("code");
+  if (name === "AbortError" || code === "EXECUTION_CANCELLED" || code === "ABORT_ERR")
     return { outcome: "cancelled", properties: {} };
   if (code === "EXECUTION_LIMIT_REACHED" || name === "ExecutionLimitReached")
     return { outcome: "limit_reached", properties: {} };
-  if (name === "GraphInterrupt" || name === "NodeInterrupt" || name === "Interrupt")
+  if (typeof name === "string" && [
+    "GraphInterrupt",
+    "NodeInterrupt",
+    "Interrupt",
+    "ParallelChildWaiting"
+  ].includes(name))
     return { outcome: "suspended", properties: {} };
-  const known = typeof code === "string" && catalog.errorCodes.includes(code);
-  const category = known && code !== "EXECUTION_FAILED" ? property(error, "category") ?? property(property(error, "failure"), "category") ?? "unknown" : "unknown";
-  const status = property(error, "status") ?? property(property(error, "failure"), "status");
-  return {
-    outcome: "failed",
-    properties: {
-      error_code: known ? code : "other",
-      error_category: category,
-      ...typeof status === "number" ? { http_status: String(status) } : {}
+  if (name === "EvalConfigurationError")
+    return {
+      outcome: "failed",
+      properties: {
+        error_code: "EVAL_SETTINGS_INVALID",
+        error_category: "configuration",
+        retryable: false
+      }
+    };
+  if (name === "TimeoutError")
+    return {
+      outcome: "failed",
+      properties: {
+        error_code: "NETWORK_TIMEOUT",
+        error_category: "transport",
+        retryable: true
+      }
+    };
+  if (typeof code === "string" && Object.hasOwn(networkCodes, code))
+    return {
+      outcome: "failed",
+      properties: {
+        error_code: networkCodes[code] ?? "UNCLASSIFIED_FAILURE",
+        error_category: "transport",
+        retryable: true
+      }
+    };
+  if (typeof code === "string" && catalog.errorCodes.includes(code)) {
+    const failure = usageErrorProperty(error, "failure");
+    const category = read("category") ?? usageErrorProperty(failure, "category");
+    const retryable = read("retryable") ?? usageErrorProperty(failure, "retryable");
+    const status2 = read("status") ?? usageErrorProperty(failure, "status");
+    return {
+      outcome: "failed",
+      properties: {
+        ...typeof status2 === "number" ? classifyUsageHttpStatus(status2).properties : {},
+        // The framework code is more specific than its HTTP projection.
+        error_code: code,
+        error_category: code !== "EXECUTION_FAILED" && typeof category === "string" && Array.isArray(catalog.properties.error_category) && catalog.properties.error_category.includes(category) ? category : "unknown",
+        ...typeof retryable === "boolean" ? { retryable } : {}
+      }
+    };
+  }
+  const status = read("status") ?? read("statusCode");
+  if (typeof status === "number" && Number.isInteger(status) && status >= 400 && status <= 599)
+    return classifyUsageHttpStatus(status);
+  if (name === "ZodError" || name === "SyntaxError")
+    return {
+      outcome: "failed",
+      properties: {
+        error_code: "SCHEMA_VALIDATION",
+        error_category: "validation"
+      }
+    };
+  return void 0;
+}
+function classifyUsageError(error, catalog = publicCatalog) {
+  const seen = /* @__PURE__ */ new Set();
+  const pending = [{ value: error, depth: 0 }];
+  let result;
+  while (pending.length && seen.size < 8) {
+    const current = pending.shift();
+    if (!current || seen.has(current.value)) continue;
+    seen.add(current.value);
+    const found = classifyOne(current.value, catalog);
+    if (found) {
+      if (!result) result = found;
+      else if (found.properties.error_code && found.outcome === "failed" && found.properties.error_code !== result.properties.error_code) {
+        result.properties.cause_code = found.properties.error_code;
+        result.properties.cause_category = found.properties.error_category ?? "unknown";
+        if (result.properties.retryable === void 0 && typeof found.properties.retryable === "boolean")
+          result.properties.retryable = found.properties.retryable;
+        break;
+      }
+      if (result === found && found.outcome !== "failed") return found;
     }
+    if (current.depth < 3)
+      for (const key of ["failure", "cause", "error"]) {
+        const value = usageErrorProperty(current.value, key);
+        if (value && typeof value === "object")
+          pending.push({ value, depth: current.depth + 1 });
+      }
+  }
+  return result ?? {
+    outcome: "failed",
+    properties: { error_code: "other", error_category: "unknown" }
   };
 }
+
+// src/measurement.ts
+function usageMeasurement(event, operation, input) {
+  if (event === "usage.feature")
+    return usageErrorProperty(input, "origin") === "background" || usageErrorProperty(input, "origin") === "server" ? "observation" : "feature";
+  if (operation === "cli.command" || operation === "sdk.eval")
+    return "invocation";
+  if (operation === "studio.setup") return "stage";
+  if (operation === "sdk.execution" || operation === "studio.eval.execution")
+    return "execution";
+  if (operation === "sdk.export" || operation === "sdk.model") return "attempt";
+  if (operation === "sdk.structured") {
+    const format = usageErrorProperty(input, "format");
+    return format === "text-delta" ? "chunk" : format === "final" ? "output" : "update";
+  }
+  if (usageErrorProperty(input, "origin") === "background") return "poll";
+  if (operation.startsWith("studio.")) return "request";
+  return "call";
+}
+
+// src/client.ts
 function sanitizeProperties(input, catalog = publicCatalog) {
   const safe = {};
   for (const [key, rule] of Object.entries(catalog.properties)) {
-    const value = property(input, key);
+    const value = usageErrorProperty(input, key);
     if (value === void 0) continue;
     if (rule === "boolean") {
       if (typeof value === "boolean") safe[key] = value;
     } else if (typeof value === "string")
       safe[key] = rule.includes(value) ? value : "other";
   }
-  const code = property(input, "error_code");
-  if (typeof code === "string")
-    safe.error_code = catalog.errorCodes.includes(code) ? code : "other";
+  for (const key of ["error_code", "cause_code"]) {
+    const code = usageErrorProperty(input, key);
+    if (typeof code === "string")
+      safe[key] = catalog.errorCodes.includes(code) ? code : "other";
+  }
   return safe;
 }
 var durationBounds = [10, 100, 1e3, 1e4, 6e4];
@@ -423,6 +767,16 @@ function createUsageTelemetry(options) {
   let disposed = false;
   let dropped = 0;
   let delivered = 0;
+  let lostObservations = 0;
+  let failedBatches = 0;
+  let reportedLoss = 0;
+  let reportedFailures = 0;
+  const lose = (count) => {
+    lostObservations = Math.min(
+      Number.MAX_SAFE_INTEGER,
+      lostObservations + count
+    );
+  };
   let hour = Math.floor(now() / 36e5);
   let sent = 0;
   const active = () => {
@@ -458,6 +812,10 @@ function createUsageTelemetry(options) {
     pending.clear();
     if (!records.length) return;
     let attempted = records.length;
+    let attemptedObservations = records.reduce(
+      (total, record2) => total + record2.count,
+      0
+    );
     flight = (async () => {
       try {
         if (options.debug?.()) {
@@ -475,7 +833,14 @@ function createUsageTelemetry(options) {
         );
         const batch = records.slice(0, allowance);
         dropped += records.length - batch.length;
+        lose(
+          records.slice(allowance).reduce((total, record2) => total + record2.count, 0)
+        );
         attempted = batch.length;
+        attemptedObservations = batch.reduce(
+          (total, record2) => total + record2.count,
+          0
+        );
         if (!batch.length || !active()) return;
         const identified = await options.identity();
         const identity2 = typeof identified === "string" ? identified : identified.id;
@@ -486,13 +851,25 @@ function createUsageTelemetry(options) {
           identity2
         )) {
           dropped += batch.length;
+          lose(attemptedObservations);
           return;
         }
         sent += batch.length;
+        const lossSnapshot = lostObservations;
+        const failureSnapshot = failedBatches;
+        if (batch[0] && (lossSnapshot > reportedLoss || failureSnapshot > reportedFailures))
+          batch[0].delivery_health = {
+            dropped_observations: lossSnapshot - reportedLoss,
+            failed_batches: failureSnapshot - reportedFailures
+          };
         await options.transport(batch, identity2);
+        reportedLoss = lossSnapshot;
+        reportedFailures = failureSnapshot;
         delivered += batch.length;
       } catch {
         dropped += attempted;
+        lose(attemptedObservations);
+        failedBatches = Math.min(Number.MAX_SAFE_INTEGER, failedBatches + 1);
       }
     })().finally(() => {
       flight = void 0;
@@ -516,12 +893,29 @@ function createUsageTelemetry(options) {
       }
       if (!configured || !catalog.operations.includes(operation) || !outcomes.includes(outcome))
         return;
-      const properties = sanitizeProperties(input, catalog);
+      const failed = outcome === "failed";
+      const status = usageErrorProperty(input, "http_status");
+      const http = typeof status === "string" && /^\d{3}$/.test(status) ? classifyUsageHttpStatus(Number(status)).properties : {};
+      const properties = sanitizeProperties(
+        {
+          measurement_version: "2",
+          measurement_kind: usageMeasurement(event, operation, input),
+          ...failed ? {
+            error_code: "UNCLASSIFIED_FAILURE",
+            error_category: "unknown",
+            ...http
+          } : {},
+          ...sanitizeProperties(input, catalog),
+          traffic_source: options.context.trafficSource ?? "unknown"
+        },
+        catalog
+      );
       const key = JSON.stringify([event, operation, outcome, properties]);
       let existing = pending.get(key);
       if (!existing) {
         if (pending.size >= (options.maxGroups ?? 64)) {
           dropped++;
+          lose(1);
           return;
         }
         existing = {
@@ -533,7 +927,9 @@ function createUsageTelemetry(options) {
           component: options.context.component,
           version: options.context.version,
           analytics_environment: options.context.environment ?? "production",
-          runtime_mode: options.context.runtimeMode ?? "unknown",
+          runtime_mode: ["development", "production", "ci"].includes(
+            options.context.runtimeMode ?? ""
+          ) ? options.context.runtimeMode ?? "unknown" : "unknown",
           identity_scope: options.context.identityScope ?? "installation",
           operation,
           outcome,
@@ -578,8 +974,11 @@ function createUsageTelemetry(options) {
           if (ended) return;
           ended = true;
           try {
+            const status = usageErrorProperty(extra, "http_status");
+            const http = outcome === "failed" && typeof status === "string" && /^\d{3}$/.test(status) ? classifyUsageHttpStatus(Number(status)).properties : {};
             record("usage.operation", operation, outcome, now() - began, {
               ...initial,
+              ...http,
               ...sanitizeProperties(extra, catalog)
             });
           } catch {
@@ -666,6 +1065,10 @@ function createPostHogTransport(options) {
         count: record.count,
         duration_total_ms: record.duration_total_ms,
         duration_buckets: record.duration_buckets,
+        ...record.delivery_health ? {
+          usage_dropped_observations: record.delivery_health.dropped_observations,
+          usage_failed_batches: record.delivery_health.failed_batches
+        } : {},
         $process_person_profile: false,
         $geoip_disable: true,
         $ip: null
@@ -761,6 +1164,13 @@ async function installationIdentity() {
   })();
   return identity;
 }
+function runtimeUsageContext() {
+  const traffic = process.env.KORTYX_USAGE_TELEMETRY_TRAFFIC_SOURCE;
+  return {
+    runtimeMode: process.env.CI && !["0", "false"].includes(process.env.CI.toLowerCase()) ? "ci" : process.env.NODE_ENV === "production" ? "production" : process.env.NODE_ENV === "development" ? "development" : "unknown",
+    trafficSource: traffic === "internal" || traffic === "synthetic" || traffic === "external" ? traffic : "unknown"
+  };
+}
 function usageDisabled() {
   return ["1", "true"].includes(process.env.KORTYX_USAGE_TELEMETRY_DISABLED ?? "") || process.env.NODE_ENV === "test";
 }
@@ -784,6 +1194,11 @@ function createNodeUsageTelemetry(options) {
   }
   return createUsageTelemetry({
     ...options,
+    context: {
+      ...options.context,
+      runtimeMode: options.context.runtimeMode ?? runtimeUsageContext().runtimeMode,
+      trafficSource: options.context.trafficSource ?? runtimeUsageContext().trafficSource
+    },
     transport,
     identity: options.identity ?? installationIdentity,
     enabled: () => valid && !usageDisabled() && (options.enabled?.() ?? true),
@@ -792,6 +1207,9 @@ function createNodeUsageTelemetry(options) {
   });
 }
 var clients = /* @__PURE__ */ new Map();
+async function flushUsageTelemetry() {
+  await Promise.all([...clients.values()].map((client) => client.flush()));
+}
 function configureUsageTelemetry(component, client) {
   if (clients.get(component) === client) return;
   clients.get(component)?.dispose();
@@ -806,7 +1224,7 @@ function getUsageTelemetry(component, version = "unknown", product = "sdk") {
         component,
         version,
         environment: process.env.KORTYX_USAGE_TELEMETRY_ENVIRONMENT === "staging" ? "staging" : "production",
-        runtimeMode: process.env.CI ? "ci" : process.env.NODE_ENV === "production" ? "production" : process.env.NODE_ENV === "development" ? "development" : "unknown"
+        ...runtimeUsageContext()
       }
     });
     clients.set(component, client);
@@ -832,6 +1250,7 @@ function publicUsageConfiguration() {
   }
   return {
     enabled: valid && !usageDisabled(),
+    ...runtimeUsageContext(),
     debug: process.env.KORTYX_USAGE_TELEMETRY_DEBUG === "1",
     environment: process.env.KORTYX_USAGE_TELEMETRY_ENVIRONMENT === "staging" ? "staging" : "production",
     // A mistakenly supplied personal token or credential-bearing URL must never reach HTML.
@@ -845,8 +1264,10 @@ function publicUsageConfiguration() {
   configureUsageTelemetry,
   createNodeUsageTelemetry,
   createUsageTelemetry,
+  flushUsageTelemetry,
   getUsageTelemetry,
   publicUsageConfiguration,
+  runtimeUsageContext,
   scale,
   usageDisabled
 });