@autter/runtime-node 1.3.1 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/index.cjs +215 -48
- package/dist/index.d.cts +21 -1
- package/dist/index.d.ts +21 -1
- package/dist/index.js +165 -10
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# @autter/runtime-node
|
|
2
2
|
|
|
3
|
+
The server tracker exports per-instance RSS, heap, memory limit, and GC
|
|
4
|
+
metrics by default. See [memory pressure incidents](../../docs/MEMORY-PRESSURE.md)
|
|
5
|
+
for ECS/Kubernetes OOM event forwarding and optional heap profiles. Set
|
|
6
|
+
`memoryMetrics: false` only if another exporter already emits equivalent
|
|
7
|
+
process metrics.
|
|
8
|
+
|
|
3
9
|
Autter Runtime for Node.js — two halves in one package:
|
|
4
10
|
|
|
5
11
|
1. **Same-origin browser relay** for `@autter/runtime-browser`
|
|
@@ -187,6 +193,9 @@ engine — keep secrets out of attributes in the first place.
|
|
|
187
193
|
|
|
188
194
|
## 3. LLM tracing
|
|
189
195
|
|
|
196
|
+
For the field contract, manual reporting patterns, and provider-agnostic
|
|
197
|
+
OpenTelemetry setup, see the dedicated [LLM instrumentation guide](../../docs/LLM-INSTRUMENTATION.md).
|
|
198
|
+
|
|
190
199
|
`initAutterServer` initialises LLM tracing automatically: any GenAI span —
|
|
191
200
|
`gen_ai.*` semconv attributes or the Vercel AI SDK's `ai.*` spans — bypasses
|
|
192
201
|
head sampling, so **every** model call is recorded with model, tokens,
|
package/dist/index.cjs
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
2
3
|
var __defProp = Object.defineProperty;
|
|
3
4
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
5
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
5
7
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
8
|
var __export = (target, all) => {
|
|
7
9
|
for (var name in all)
|
|
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
15
17
|
}
|
|
16
18
|
return to;
|
|
17
19
|
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
18
28
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
29
|
|
|
20
30
|
// src/index.ts
|
|
@@ -30,7 +40,9 @@ __export(index_exports, {
|
|
|
30
40
|
instrumentLlmClient: () => instrumentLlmClient,
|
|
31
41
|
makeSafeCapture: () => makeSafeCapture,
|
|
32
42
|
redactAttributes: () => redactAttributes,
|
|
43
|
+
reportOutcome: () => reportOutcome,
|
|
33
44
|
sanitizeBrowserPayload: () => sanitizeBrowserPayload,
|
|
45
|
+
startCaughtExceptionSampler: () => startCaughtExceptionSampler,
|
|
34
46
|
trackLlmCall: () => trackLlmCall,
|
|
35
47
|
withLlmCall: () => withLlmCall,
|
|
36
48
|
withProcessSpan: () => withProcessSpan
|
|
@@ -68,7 +80,10 @@ var EVENT_TYPES = /* @__PURE__ */ new Set([
|
|
|
68
80
|
"unhandled_rejection",
|
|
69
81
|
"message",
|
|
70
82
|
"session_start",
|
|
71
|
-
"track_event"
|
|
83
|
+
"track_event",
|
|
84
|
+
"outcome",
|
|
85
|
+
"request_failure",
|
|
86
|
+
"timing"
|
|
72
87
|
]);
|
|
73
88
|
var SEVERITIES = /* @__PURE__ */ new Set(["fatal", "error", "warning", "info"]);
|
|
74
89
|
function sanitizeBrowserPayload(raw) {
|
|
@@ -90,6 +105,7 @@ function sanitizeBrowserPayload(raw) {
|
|
|
90
105
|
...typeof e.severity === "string" && SEVERITIES.has(e.severity) ? { severity: e.severity } : {},
|
|
91
106
|
message: typeof e.message === "string" ? e.message.slice(0, 4e3) : "",
|
|
92
107
|
...typeof e.name === "string" ? { name: e.name.slice(0, 200) } : {},
|
|
108
|
+
...typeof e.durationMs === "number" && Number.isFinite(e.durationMs) ? { durationMs: Math.max(0, Math.min(12e4, e.durationMs)) } : {},
|
|
93
109
|
...typeof e.stack === "string" ? { stack: e.stack.slice(0, 32e3) } : {},
|
|
94
110
|
...typeof e.errorType === "string" ? { errorType: e.errorType.slice(0, 200) } : {},
|
|
95
111
|
...typeof e.filename === "string" ? { filename: e.filename.split("?")[0].slice(0, 1e3) } : {},
|
|
@@ -239,7 +255,7 @@ function createBrowserRelayHandler(opts) {
|
|
|
239
255
|
// src/server.ts
|
|
240
256
|
var import_node_http = require("http");
|
|
241
257
|
var import_node_crypto = require("crypto");
|
|
242
|
-
var
|
|
258
|
+
var import_api2 = require("@opentelemetry/api");
|
|
243
259
|
var import_core2 = require("@opentelemetry/core");
|
|
244
260
|
var import_exporter_metrics_otlp_http = require("@opentelemetry/exporter-metrics-otlp-http");
|
|
245
261
|
var import_exporter_trace_otlp_http = require("@opentelemetry/exporter-trace-otlp-http");
|
|
@@ -323,6 +339,7 @@ function installAutterAutoFlush(options = {}) {
|
|
|
323
339
|
const warnOnUnflushedExit = options.warnOnUnflushedExit !== false;
|
|
324
340
|
const timeoutMs = Math.max(100, options.timeoutMs ?? 3e3);
|
|
325
341
|
const signals = options.signals ?? DEFAULT_SIGNALS;
|
|
342
|
+
const flushOnUncaughtException = options.flushOnUncaughtException !== false;
|
|
326
343
|
let disposed = false;
|
|
327
344
|
let inFlight = null;
|
|
328
345
|
const doFlush = async () => {
|
|
@@ -368,6 +385,19 @@ function installAutterAutoFlush(options = {}) {
|
|
|
368
385
|
return inFlight;
|
|
369
386
|
};
|
|
370
387
|
const signalCounts = /* @__PURE__ */ new Map();
|
|
388
|
+
let handlingUncaughtException = false;
|
|
389
|
+
const onUncaughtException = (error) => {
|
|
390
|
+
if (process.listenerCount("uncaughtException") > 1) return;
|
|
391
|
+
if (handlingUncaughtException) return;
|
|
392
|
+
handlingUncaughtException = true;
|
|
393
|
+
logger.error(error instanceof Error ? error.stack ?? error.message : String(error));
|
|
394
|
+
const keepAlive = setTimeout(() => {
|
|
395
|
+
}, timeoutMs + 50);
|
|
396
|
+
void flush("uncaughtException").finally(() => {
|
|
397
|
+
clearTimeout(keepAlive);
|
|
398
|
+
process.exit(1);
|
|
399
|
+
});
|
|
400
|
+
};
|
|
371
401
|
const onSignal = (signal) => {
|
|
372
402
|
const count = (signalCounts.get(signal) ?? 0) + 1;
|
|
373
403
|
signalCounts.set(signal, count);
|
|
@@ -407,6 +437,9 @@ function installAutterAutoFlush(options = {}) {
|
|
|
407
437
|
};
|
|
408
438
|
const signalHandlers = /* @__PURE__ */ new Map();
|
|
409
439
|
process.on("beforeExit", onBeforeExit);
|
|
440
|
+
if (flushOnUncaughtException) {
|
|
441
|
+
process.on("uncaughtException", onUncaughtException);
|
|
442
|
+
}
|
|
410
443
|
for (const signal of signals) {
|
|
411
444
|
const handler = () => onSignal(signal);
|
|
412
445
|
signalHandlers.set(signal, handler);
|
|
@@ -418,6 +451,7 @@ function installAutterAutoFlush(options = {}) {
|
|
|
418
451
|
dispose() {
|
|
419
452
|
disposed = true;
|
|
420
453
|
process.removeListener("beforeExit", onBeforeExit);
|
|
454
|
+
process.removeListener("uncaughtException", onUncaughtException);
|
|
421
455
|
process.removeListener("exit", onExit);
|
|
422
456
|
for (const [signal, handler] of signalHandlers) {
|
|
423
457
|
process.removeListener(signal, handler);
|
|
@@ -673,6 +707,51 @@ function makeRedactor(options) {
|
|
|
673
707
|
return (attributes) => redactWith(attributes, r);
|
|
674
708
|
}
|
|
675
709
|
|
|
710
|
+
// src/memory.ts
|
|
711
|
+
var import_node_perf_hooks = require("perf_hooks");
|
|
712
|
+
var import_node_v8 = require("v8");
|
|
713
|
+
var import_api = require("@opentelemetry/api");
|
|
714
|
+
function startMemoryMetrics() {
|
|
715
|
+
const meter = import_api.metrics.getMeter("autter-process-memory");
|
|
716
|
+
const rss = meter.createObservableGauge("autter.process.memory.rss", { unit: "By" });
|
|
717
|
+
const heapUsed = meter.createObservableGauge("autter.process.memory.heap.used", { unit: "By" });
|
|
718
|
+
const heapLimit = meter.createObservableGauge("autter.process.memory.heap.limit", { unit: "By" });
|
|
719
|
+
const containerLimit = meter.createObservableGauge("autter.process.memory.limit", { unit: "By" });
|
|
720
|
+
const afterGc = meter.createObservableGauge("autter.process.memory.heap.after_gc", { unit: "By" });
|
|
721
|
+
const gcCount = meter.createObservableCounter("autter.process.gc.count", { unit: "{collection}" });
|
|
722
|
+
const gcDuration = meter.createObservableCounter("autter.process.gc.duration", { unit: "ms" });
|
|
723
|
+
let collections = 0;
|
|
724
|
+
let durationMs = 0;
|
|
725
|
+
let lastAfterGc = null;
|
|
726
|
+
const callbacks = [
|
|
727
|
+
[rss, (result) => result.observe(process.memoryUsage().rss)],
|
|
728
|
+
[heapUsed, (result) => result.observe(process.memoryUsage().heapUsed)],
|
|
729
|
+
[heapLimit, (result) => result.observe((0, import_node_v8.getHeapStatistics)().heap_size_limit)],
|
|
730
|
+
[containerLimit, (result) => {
|
|
731
|
+
const constrained = process.constrainedMemory?.() ?? 0;
|
|
732
|
+
if (constrained > 0) result.observe(constrained);
|
|
733
|
+
}],
|
|
734
|
+
[afterGc, (result) => {
|
|
735
|
+
if (lastAfterGc !== null) result.observe(lastAfterGc);
|
|
736
|
+
}],
|
|
737
|
+
[gcCount, (result) => result.observe(collections)],
|
|
738
|
+
[gcDuration, (result) => result.observe(durationMs)]
|
|
739
|
+
];
|
|
740
|
+
for (const [instrument, callback] of callbacks) instrument.addCallback(callback);
|
|
741
|
+
const observer = new import_node_perf_hooks.PerformanceObserver((list) => {
|
|
742
|
+
for (const entry of list.getEntries()) {
|
|
743
|
+
collections++;
|
|
744
|
+
durationMs += entry.duration;
|
|
745
|
+
lastAfterGc = process.memoryUsage().heapUsed;
|
|
746
|
+
}
|
|
747
|
+
});
|
|
748
|
+
observer.observe({ entryTypes: ["gc"] });
|
|
749
|
+
return () => {
|
|
750
|
+
observer.disconnect();
|
|
751
|
+
for (const [instrument, callback] of callbacks) instrument.removeCallback(callback);
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
|
|
676
755
|
// src/server.ts
|
|
677
756
|
var LLM_MARKER_ATTRIBUTES = [
|
|
678
757
|
"gen_ai.operation.name",
|
|
@@ -735,14 +814,14 @@ var RETENTION_MAX_BUFFERED_SPANS = 5e3;
|
|
|
735
814
|
var RETENTION_SWEEP_INTERVAL_MS = 1e4;
|
|
736
815
|
var RETENTION_TRACE_TTL_MS = 3e4;
|
|
737
816
|
function spanIndicatesError(span) {
|
|
738
|
-
if (span.status.code ===
|
|
817
|
+
if (span.status.code === import_api2.SpanStatusCode.ERROR) return true;
|
|
739
818
|
return span.events.some((event) => event.name === "exception");
|
|
740
819
|
}
|
|
741
820
|
var TAIL_RETAINED_ATTR = "autter.tail_retained";
|
|
742
821
|
function withSampledFlag(span) {
|
|
743
822
|
const spanContext = {
|
|
744
823
|
...span.spanContext(),
|
|
745
|
-
traceFlags: span.spanContext().traceFlags |
|
|
824
|
+
traceFlags: span.spanContext().traceFlags | import_api2.TraceFlags.SAMPLED
|
|
746
825
|
};
|
|
747
826
|
return {
|
|
748
827
|
name: span.name,
|
|
@@ -782,11 +861,11 @@ var ErrorTraceRetentionProcessor = class {
|
|
|
782
861
|
}
|
|
783
862
|
onEnd(span) {
|
|
784
863
|
const ctx = span.spanContext();
|
|
785
|
-
if ((ctx.traceFlags &
|
|
864
|
+
if ((ctx.traceFlags & import_api2.TraceFlags.SAMPLED) !== 0) return;
|
|
786
865
|
const entry = this.entryFor(ctx.traceId);
|
|
787
866
|
if (entry.retained) {
|
|
788
867
|
this.forward(span);
|
|
789
|
-
} else if (this.retainErrors && spanIndicatesError(span) || this.slowThresholdMs > 0 && span.kind ===
|
|
868
|
+
} else if (this.retainErrors && spanIndicatesError(span) || this.slowThresholdMs > 0 && span.kind === import_api2.SpanKind.SERVER && span.duration[0] * 1e3 + span.duration[1] / 1e6 >= this.slowThresholdMs) {
|
|
790
869
|
entry.retained = true;
|
|
791
870
|
this.flush(entry);
|
|
792
871
|
this.forward(span);
|
|
@@ -801,8 +880,8 @@ var ErrorTraceRetentionProcessor = class {
|
|
|
801
880
|
}
|
|
802
881
|
/** Promote the active trace, if any — called on captured errors. */
|
|
803
882
|
retainActiveTrace() {
|
|
804
|
-
const active2 =
|
|
805
|
-
if (!active2 || (active2.traceFlags &
|
|
883
|
+
const active2 = import_api2.trace.getActiveSpan()?.spanContext();
|
|
884
|
+
if (!active2 || (active2.traceFlags & import_api2.TraceFlags.SAMPLED) !== 0) return;
|
|
806
885
|
const entry = this.entryFor(active2.traceId);
|
|
807
886
|
if (entry.retained) return;
|
|
808
887
|
entry.retained = true;
|
|
@@ -891,7 +970,7 @@ function usageAttributes(usage) {
|
|
|
891
970
|
}
|
|
892
971
|
async function runLlmSpan(tracer, info, fn) {
|
|
893
972
|
const span = tracer.startSpan(`${info.operation ?? "chat"} ${info.model}`, {
|
|
894
|
-
kind:
|
|
973
|
+
kind: import_api2.SpanKind.CLIENT,
|
|
895
974
|
attributes: llmBaseAttributes(info)
|
|
896
975
|
});
|
|
897
976
|
const setResponseModel = (model) => span.setAttribute("gen_ai.response.model", model);
|
|
@@ -903,11 +982,11 @@ async function runLlmSpan(tracer, info, fn) {
|
|
|
903
982
|
setAttributes: (attributes) => span.setAttributes(attributes)
|
|
904
983
|
};
|
|
905
984
|
try {
|
|
906
|
-
const result = await
|
|
907
|
-
|
|
985
|
+
const result = await import_api2.context.with(
|
|
986
|
+
import_api2.trace.setSpan(import_api2.context.active(), span),
|
|
908
987
|
() => fn(handle)
|
|
909
988
|
);
|
|
910
|
-
span.setStatus({ code:
|
|
989
|
+
span.setStatus({ code: import_api2.SpanStatusCode.OK });
|
|
911
990
|
return result;
|
|
912
991
|
} catch (err) {
|
|
913
992
|
if (err instanceof Error) {
|
|
@@ -915,7 +994,7 @@ async function runLlmSpan(tracer, info, fn) {
|
|
|
915
994
|
span.setAttribute("error.type", err.name);
|
|
916
995
|
}
|
|
917
996
|
span.setStatus({
|
|
918
|
-
code:
|
|
997
|
+
code: import_api2.SpanStatusCode.ERROR,
|
|
919
998
|
message: err instanceof Error ? err.message : String(err)
|
|
920
999
|
});
|
|
921
1000
|
throw err;
|
|
@@ -930,7 +1009,7 @@ function recordLlmCall(tracer, call) {
|
|
|
930
1009
|
const span = tracer.startSpan(
|
|
931
1010
|
`${call.operation ?? "chat"} ${call.model}`,
|
|
932
1011
|
{
|
|
933
|
-
kind:
|
|
1012
|
+
kind: import_api2.SpanKind.CLIENT,
|
|
934
1013
|
startTime,
|
|
935
1014
|
attributes: { ...llmBaseAttributes(call), ...usageAttributes(call) }
|
|
936
1015
|
}
|
|
@@ -941,27 +1020,27 @@ function recordLlmCall(tracer, call) {
|
|
|
941
1020
|
span.setAttribute("error.type", call.error.name);
|
|
942
1021
|
}
|
|
943
1022
|
span.setStatus({
|
|
944
|
-
code:
|
|
1023
|
+
code: import_api2.SpanStatusCode.ERROR,
|
|
945
1024
|
message: call.error instanceof Error ? call.error.message : String(call.error)
|
|
946
1025
|
});
|
|
947
1026
|
} else {
|
|
948
|
-
span.setStatus({ code:
|
|
1027
|
+
span.setStatus({ code: import_api2.SpanStatusCode.OK });
|
|
949
1028
|
}
|
|
950
1029
|
span.end(endTime);
|
|
951
1030
|
}
|
|
952
1031
|
async function runWithSpan(tracer, name, fn, attributes) {
|
|
953
1032
|
const span = tracer.startSpan(name, { attributes });
|
|
954
1033
|
try {
|
|
955
|
-
const result = await
|
|
956
|
-
|
|
1034
|
+
const result = await import_api2.context.with(
|
|
1035
|
+
import_api2.trace.setSpan(import_api2.context.active(), span),
|
|
957
1036
|
fn
|
|
958
1037
|
);
|
|
959
|
-
span.setStatus({ code:
|
|
1038
|
+
span.setStatus({ code: import_api2.SpanStatusCode.OK });
|
|
960
1039
|
return result;
|
|
961
1040
|
} catch (err) {
|
|
962
1041
|
if (err instanceof Error) span.recordException(err);
|
|
963
1042
|
span.setStatus({
|
|
964
|
-
code:
|
|
1043
|
+
code: import_api2.SpanStatusCode.ERROR,
|
|
965
1044
|
message: err instanceof Error ? err.message : String(err)
|
|
966
1045
|
});
|
|
967
1046
|
throw err;
|
|
@@ -979,7 +1058,7 @@ function expressRouteOf(req) {
|
|
|
979
1058
|
}
|
|
980
1059
|
function captureExpressRoute(_span, response) {
|
|
981
1060
|
if (!(response instanceof import_node_http.ServerResponse)) return;
|
|
982
|
-
const rpcMetadata = (0, import_core2.getRPCMetadata)(
|
|
1061
|
+
const rpcMetadata = (0, import_core2.getRPCMetadata)(import_api2.context.active());
|
|
983
1062
|
if (rpcMetadata?.type !== import_core2.RPCType.HTTP) return;
|
|
984
1063
|
const setRoute = () => {
|
|
985
1064
|
if (rpcMetadata.route) return;
|
|
@@ -1005,7 +1084,7 @@ function initAutterServer(options) {
|
|
|
1005
1084
|
activeRedactor = makeRedactor(options.redactAttributes ?? true);
|
|
1006
1085
|
const resource = new import_resources.Resource({
|
|
1007
1086
|
[import_semantic_conventions.ATTR_SERVICE_NAME]: options.service,
|
|
1008
|
-
"service.instance.id": (0, import_node_crypto.randomUUID)(),
|
|
1087
|
+
"service.instance.id": (options.instanceId || process.env.AUTTER_RUNTIME_INSTANCE_ID || (0, import_node_crypto.randomUUID)()).slice(0, 128),
|
|
1009
1088
|
...options.release ? { [import_semantic_conventions.ATTR_SERVICE_VERSION]: options.release } : {},
|
|
1010
1089
|
"deployment.environment": environment,
|
|
1011
1090
|
// Tells the ingester request metrics arrive on the metrics pipe, so
|
|
@@ -1070,6 +1149,7 @@ function initAutterServer(options) {
|
|
|
1070
1149
|
]
|
|
1071
1150
|
});
|
|
1072
1151
|
sdk.start();
|
|
1152
|
+
const stopMemoryMetrics = options.memoryMetrics === false ? null : startMemoryMetrics();
|
|
1073
1153
|
const alwaysOnProvider = new import_sdk_trace_base.BasicTracerProvider({
|
|
1074
1154
|
resource,
|
|
1075
1155
|
sampler: new import_sdk_trace_base.AlwaysOnSampler(),
|
|
@@ -1091,21 +1171,32 @@ function initAutterServer(options) {
|
|
|
1091
1171
|
const isError = error instanceof Error;
|
|
1092
1172
|
const message = isError ? error.message : String(error);
|
|
1093
1173
|
const redacted = activeRedactor(attributes);
|
|
1094
|
-
const activeSpan =
|
|
1174
|
+
const activeSpan = import_api2.trace.getActiveSpan();
|
|
1095
1175
|
const reuseActive = activeSpan?.isRecording() === true;
|
|
1096
1176
|
const span = reuseActive ? activeSpan : errorTracer.startSpan(isError ? error.name : "Error");
|
|
1097
1177
|
span.setAttributes({ "autter.severity": "error", ...redacted });
|
|
1098
1178
|
if (isError && error.stack) {
|
|
1099
|
-
|
|
1179
|
+
if (redacted["autter.handled"] === true) {
|
|
1180
|
+
span.addEvent("exception", {
|
|
1181
|
+
"exception.type": error.name,
|
|
1182
|
+
"exception.message": message,
|
|
1183
|
+
"exception.stacktrace": error.stack,
|
|
1184
|
+
"autter.handled": true,
|
|
1185
|
+
"autter.sampled": redacted["autter.sampled"] === true
|
|
1186
|
+
});
|
|
1187
|
+
} else {
|
|
1188
|
+
span.recordException(error);
|
|
1189
|
+
}
|
|
1100
1190
|
} else {
|
|
1101
1191
|
const stack = new Error().stack?.split("\n").filter((line, i) => i === 0 || !line.includes("captureException")).join("\n");
|
|
1102
1192
|
span.addEvent("exception", {
|
|
1103
1193
|
"exception.type": isError ? error.name : "Error",
|
|
1104
1194
|
"exception.message": message,
|
|
1105
|
-
...stack ? { "exception.stacktrace": stack } : {}
|
|
1195
|
+
...stack ? { "exception.stacktrace": stack } : {},
|
|
1196
|
+
...redacted["autter.handled"] === true ? { "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true } : {}
|
|
1106
1197
|
});
|
|
1107
1198
|
}
|
|
1108
|
-
span.setStatus({ code:
|
|
1199
|
+
span.setStatus({ code: import_api2.SpanStatusCode.ERROR, message });
|
|
1109
1200
|
if (!reuseActive) span.end();
|
|
1110
1201
|
}
|
|
1111
1202
|
function captureMessage2(message, severity = "warning", attributes) {
|
|
@@ -1123,18 +1214,31 @@ function initAutterServer(options) {
|
|
|
1123
1214
|
...stack ? { "exception.stacktrace": stack } : {},
|
|
1124
1215
|
"autter.severity": severity
|
|
1125
1216
|
});
|
|
1126
|
-
span.setStatus({ code:
|
|
1217
|
+
span.setStatus({ code: import_api2.SpanStatusCode.ERROR, message });
|
|
1127
1218
|
span.end();
|
|
1128
1219
|
}
|
|
1220
|
+
function reportOutcome2(name, message, attributes) {
|
|
1221
|
+
errorTraceBuffer?.retainActiveTrace();
|
|
1222
|
+
telemetryStats.markCaptured();
|
|
1223
|
+
const activeSpan = import_api2.trace.getActiveSpan();
|
|
1224
|
+
const reuseActive = activeSpan?.isRecording() === true;
|
|
1225
|
+
const span = reuseActive ? activeSpan : errorTracer.startSpan("OutcomeFailure");
|
|
1226
|
+
const safeOutcome = activeRedactor({ "autter.outcome.name": name.slice(0, 200), "autter.outcome.message": message.slice(0, 1e3) });
|
|
1227
|
+
const safeName = String(safeOutcome["autter.outcome.name"] ?? "operation");
|
|
1228
|
+
const safeMessage = String(safeOutcome["autter.outcome.message"] ?? "failed");
|
|
1229
|
+
span.addEvent("autter.outcome", {
|
|
1230
|
+
...activeRedactor(attributes),
|
|
1231
|
+
"autter.outcome.status": "error",
|
|
1232
|
+
"autter.outcome.name": safeName,
|
|
1233
|
+
"autter.outcome.message": safeMessage
|
|
1234
|
+
});
|
|
1235
|
+
span.setStatus({ code: import_api2.SpanStatusCode.ERROR, message: safeMessage });
|
|
1236
|
+
if (!reuseActive) span.end();
|
|
1237
|
+
}
|
|
1129
1238
|
if (options.captureGlobalErrors !== false) {
|
|
1130
1239
|
process.on("uncaughtExceptionMonitor", (err) => {
|
|
1131
1240
|
captureException2(err, { "autter.unhandled": true });
|
|
1132
|
-
void flushTarget.forceFlush()
|
|
1133
|
-
});
|
|
1134
|
-
process.on("unhandledRejection", (reason) => {
|
|
1135
|
-
captureException2(reason, {
|
|
1136
|
-
"autter.unhandled": true,
|
|
1137
|
-
...reason instanceof Error ? {} : { "autter.severity": "warning" }
|
|
1241
|
+
if (options.autoFlush === false) void Promise.resolve(flushTarget.forceFlush()).catch(() => {
|
|
1138
1242
|
});
|
|
1139
1243
|
});
|
|
1140
1244
|
}
|
|
@@ -1162,10 +1266,12 @@ function initAutterServer(options) {
|
|
|
1162
1266
|
const server = {
|
|
1163
1267
|
captureException: captureException2,
|
|
1164
1268
|
captureMessage: captureMessage2,
|
|
1269
|
+
reportOutcome: reportOutcome2,
|
|
1165
1270
|
withProcessSpan: (name, fn, attributes) => runWithSpan(processTracer, name, fn, activeRedactor(attributes)),
|
|
1166
1271
|
withLlmCall: (info, fn) => runLlmSpan(llmTracer, info, fn),
|
|
1167
1272
|
trackLlmCall: (call) => recordLlmCall(llmTracer, call),
|
|
1168
1273
|
shutdown: async () => {
|
|
1274
|
+
stopMemoryMetrics?.();
|
|
1169
1275
|
active = null;
|
|
1170
1276
|
activeAlwaysOnProvider = null;
|
|
1171
1277
|
autoFlushHandle?.dispose();
|
|
@@ -1184,35 +1290,35 @@ function captureException(error, attributes) {
|
|
|
1184
1290
|
active.captureException(error, attributes);
|
|
1185
1291
|
return;
|
|
1186
1292
|
}
|
|
1187
|
-
const span =
|
|
1293
|
+
const span = import_api2.trace.getTracer("autter-errors").startSpan(error instanceof Error ? error.name : "Error", { attributes });
|
|
1188
1294
|
if (error instanceof Error) span.recordException(error);
|
|
1189
1295
|
span.setStatus({
|
|
1190
|
-
code:
|
|
1296
|
+
code: import_api2.SpanStatusCode.ERROR,
|
|
1191
1297
|
message: error instanceof Error ? error.message : String(error)
|
|
1192
1298
|
});
|
|
1193
1299
|
span.end();
|
|
1194
1300
|
}
|
|
1195
1301
|
function withProcessSpan(name, fn, attributes) {
|
|
1196
1302
|
if (active) return active.withProcessSpan(name, fn, attributes);
|
|
1197
|
-
return runWithSpan(
|
|
1303
|
+
return runWithSpan(import_api2.trace.getTracer("autter-processes"), name, fn, attributes);
|
|
1198
1304
|
}
|
|
1199
1305
|
function withLlmCall(info, fn) {
|
|
1200
1306
|
if (active) return active.withLlmCall(info, fn);
|
|
1201
|
-
return runLlmSpan(
|
|
1307
|
+
return runLlmSpan(import_api2.trace.getTracer("autter-llm"), info, fn);
|
|
1202
1308
|
}
|
|
1203
1309
|
function trackLlmCall(call) {
|
|
1204
1310
|
if (active) {
|
|
1205
1311
|
active.trackLlmCall(call);
|
|
1206
1312
|
return;
|
|
1207
1313
|
}
|
|
1208
|
-
recordLlmCall(
|
|
1314
|
+
recordLlmCall(import_api2.trace.getTracer("autter-llm"), call);
|
|
1209
1315
|
}
|
|
1210
1316
|
function captureMessage(message, severity = "warning", attributes) {
|
|
1211
1317
|
if (active) {
|
|
1212
1318
|
active.captureMessage(message, severity, attributes);
|
|
1213
1319
|
return;
|
|
1214
1320
|
}
|
|
1215
|
-
const span =
|
|
1321
|
+
const span = import_api2.trace.getTracer("autter-errors").startSpan("Message", {
|
|
1216
1322
|
attributes: { "autter.severity": severity, ...attributes }
|
|
1217
1323
|
});
|
|
1218
1324
|
span.addEvent("exception", {
|
|
@@ -1220,7 +1326,22 @@ function captureMessage(message, severity = "warning", attributes) {
|
|
|
1220
1326
|
"exception.message": message,
|
|
1221
1327
|
"autter.severity": severity
|
|
1222
1328
|
});
|
|
1223
|
-
span.setStatus({ code:
|
|
1329
|
+
span.setStatus({ code: import_api2.SpanStatusCode.ERROR, message });
|
|
1330
|
+
span.end();
|
|
1331
|
+
}
|
|
1332
|
+
function reportOutcome(name, message, attributes) {
|
|
1333
|
+
if (active) return active.reportOutcome(name, message, attributes);
|
|
1334
|
+
const span = import_api2.trace.getTracer("autter-outcomes").startSpan("OutcomeFailure");
|
|
1335
|
+
const safeOutcome = redactAttributes({ "autter.outcome.name": name.slice(0, 200), "autter.outcome.message": message.slice(0, 1e3) });
|
|
1336
|
+
const safeName = String(safeOutcome["autter.outcome.name"] ?? "operation");
|
|
1337
|
+
const safeMessage = String(safeOutcome["autter.outcome.message"] ?? "failed");
|
|
1338
|
+
span.addEvent("autter.outcome", {
|
|
1339
|
+
...redactAttributes(attributes),
|
|
1340
|
+
"autter.outcome.status": "error",
|
|
1341
|
+
"autter.outcome.name": safeName,
|
|
1342
|
+
"autter.outcome.message": safeMessage
|
|
1343
|
+
});
|
|
1344
|
+
span.setStatus({ code: import_api2.SpanStatusCode.ERROR, message: safeMessage });
|
|
1224
1345
|
span.end();
|
|
1225
1346
|
}
|
|
1226
1347
|
function makeSafeCapture(options) {
|
|
@@ -1231,7 +1352,7 @@ function makeSafeCapture(options) {
|
|
|
1231
1352
|
};
|
|
1232
1353
|
}
|
|
1233
1354
|
function alwaysOnTracer(name) {
|
|
1234
|
-
return activeAlwaysOnProvider ? activeAlwaysOnProvider.getTracer(name) :
|
|
1355
|
+
return activeAlwaysOnProvider ? activeAlwaysOnProvider.getTracer(name) : import_api2.trace.getTracer(name);
|
|
1235
1356
|
}
|
|
1236
1357
|
function autterLlmTracer() {
|
|
1237
1358
|
return alwaysOnTracer("autter-llm");
|
|
@@ -1251,7 +1372,7 @@ async function emitLlmSelftestTrace() {
|
|
|
1251
1372
|
attributes: { "autter.selftest": true }
|
|
1252
1373
|
},
|
|
1253
1374
|
(llm) => {
|
|
1254
|
-
traceId =
|
|
1375
|
+
traceId = import_api2.trace.getActiveSpan()?.spanContext().traceId ?? "";
|
|
1255
1376
|
llm.setUsage({ inputTokens: 1, outputTokens: 1 });
|
|
1256
1377
|
llm.setCost(0);
|
|
1257
1378
|
}
|
|
@@ -1261,7 +1382,7 @@ async function emitLlmSelftestTrace() {
|
|
|
1261
1382
|
}
|
|
1262
1383
|
|
|
1263
1384
|
// src/llm-instrument.ts
|
|
1264
|
-
var
|
|
1385
|
+
var import_api3 = require("@opentelemetry/api");
|
|
1265
1386
|
var CALL_METHODS = /* @__PURE__ */ new Set([
|
|
1266
1387
|
"create",
|
|
1267
1388
|
// openai chat/completions/embeddings/responses, anthropic messages
|
|
@@ -1319,7 +1440,7 @@ function applyUsage(span, usage) {
|
|
|
1319
1440
|
function endWithError(span, err) {
|
|
1320
1441
|
if (err instanceof Error) span.recordException(err);
|
|
1321
1442
|
span.setStatus({
|
|
1322
|
-
code:
|
|
1443
|
+
code: import_api3.SpanStatusCode.ERROR,
|
|
1323
1444
|
message: err instanceof Error ? err.message : String(err)
|
|
1324
1445
|
});
|
|
1325
1446
|
span.end();
|
|
@@ -1339,7 +1460,7 @@ function wrapStream(stream, span) {
|
|
|
1339
1460
|
if (err !== void 0) {
|
|
1340
1461
|
endWithError(span, err);
|
|
1341
1462
|
} else {
|
|
1342
|
-
span.setStatus({ code:
|
|
1463
|
+
span.setStatus({ code: import_api3.SpanStatusCode.OK });
|
|
1343
1464
|
span.end();
|
|
1344
1465
|
}
|
|
1345
1466
|
};
|
|
@@ -1394,7 +1515,7 @@ function instrumentLlmClient(client, options) {
|
|
|
1394
1515
|
const operation = /embed/i.test(callPath) ? "embeddings" : "chat";
|
|
1395
1516
|
const userId = resolveUserId();
|
|
1396
1517
|
const span = autterLlmTracer().startSpan(`${operation} ${request.model}`, {
|
|
1397
|
-
kind:
|
|
1518
|
+
kind: import_api3.SpanKind.CLIENT,
|
|
1398
1519
|
attributes: {
|
|
1399
1520
|
"gen_ai.operation.name": operation,
|
|
1400
1521
|
"gen_ai.provider.name": provider,
|
|
@@ -1416,7 +1537,7 @@ function instrumentLlmClient(client, options) {
|
|
|
1416
1537
|
return wrapStream(result, span);
|
|
1417
1538
|
}
|
|
1418
1539
|
applyUsage(span, extractUsage(result));
|
|
1419
|
-
span.setStatus({ code:
|
|
1540
|
+
span.setStatus({ code: import_api3.SpanStatusCode.OK });
|
|
1420
1541
|
span.end();
|
|
1421
1542
|
return result;
|
|
1422
1543
|
}
|
|
@@ -1426,7 +1547,7 @@ function instrumentLlmClient(client, options) {
|
|
|
1426
1547
|
return wrapStream(resolved, span);
|
|
1427
1548
|
}
|
|
1428
1549
|
applyUsage(span, extractUsage(resolved));
|
|
1429
|
-
span.setStatus({ code:
|
|
1550
|
+
span.setStatus({ code: import_api3.SpanStatusCode.OK });
|
|
1430
1551
|
span.end();
|
|
1431
1552
|
return resolved;
|
|
1432
1553
|
},
|
|
@@ -1472,6 +1593,50 @@ function instrumentLlmClient(client, options) {
|
|
|
1472
1593
|
}
|
|
1473
1594
|
return deepProxy(client);
|
|
1474
1595
|
}
|
|
1596
|
+
|
|
1597
|
+
// src/caught-exceptions.ts
|
|
1598
|
+
var import_node_inspector = __toESM(require("inspector"), 1);
|
|
1599
|
+
function startCaughtExceptionSampler(options = {}) {
|
|
1600
|
+
const rate = Math.max(0, Math.min(1, options.sampleRate ?? 0.01));
|
|
1601
|
+
const cap = Math.max(1, Math.min(100, options.maxPerMinute ?? 10));
|
|
1602
|
+
const session = new import_node_inspector.default.Session();
|
|
1603
|
+
let active2 = true;
|
|
1604
|
+
let minute = 0;
|
|
1605
|
+
let sent = 0;
|
|
1606
|
+
session.connect();
|
|
1607
|
+
session.on("Debugger.paused", (message) => {
|
|
1608
|
+
session.post("Debugger.resume", () => {
|
|
1609
|
+
});
|
|
1610
|
+
if (!active2 || message.params.reason !== "exception") return;
|
|
1611
|
+
const nowMinute = Math.floor(Date.now() / 6e4);
|
|
1612
|
+
if (nowMinute !== minute) {
|
|
1613
|
+
minute = nowMinute;
|
|
1614
|
+
sent = 0;
|
|
1615
|
+
}
|
|
1616
|
+
if (sent >= cap || Math.random() >= rate) return;
|
|
1617
|
+
sent++;
|
|
1618
|
+
const frames = message.params.callFrames.slice(0, 8).filter((frame) => frame.url && !frame.url.startsWith("node:")).map((frame) => ` at ${frame.functionName || "anonymous"} (${frame.url.split("?")[0]}:${frame.location.lineNumber + 1}:${(frame.location.columnNumber ?? 0) + 1})`);
|
|
1619
|
+
if (!frames.length) return;
|
|
1620
|
+
const error = new Error("Sampled caught exception");
|
|
1621
|
+
error.name = "CaughtException";
|
|
1622
|
+
error.stack = `${error.name}: ${error.message}
|
|
1623
|
+
${frames.join("\n")}`;
|
|
1624
|
+
queueMicrotask(() => {
|
|
1625
|
+
if (active2) captureException(error, { "autter.handled": true, "autter.sampled": true });
|
|
1626
|
+
});
|
|
1627
|
+
});
|
|
1628
|
+
session.post("Debugger.enable", (error) => {
|
|
1629
|
+
if (!error && active2) session.post("Debugger.setPauseOnExceptions", { state: "all" }, () => {
|
|
1630
|
+
});
|
|
1631
|
+
});
|
|
1632
|
+
return { stop() {
|
|
1633
|
+
if (!active2) return;
|
|
1634
|
+
active2 = false;
|
|
1635
|
+
session.post("Debugger.setPauseOnExceptions", { state: "none" }, () => {
|
|
1636
|
+
session.post("Debugger.disable", () => session.disconnect());
|
|
1637
|
+
});
|
|
1638
|
+
} };
|
|
1639
|
+
}
|
|
1475
1640
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1476
1641
|
0 && (module.exports = {
|
|
1477
1642
|
captureException,
|
|
@@ -1484,7 +1649,9 @@ function instrumentLlmClient(client, options) {
|
|
|
1484
1649
|
instrumentLlmClient,
|
|
1485
1650
|
makeSafeCapture,
|
|
1486
1651
|
redactAttributes,
|
|
1652
|
+
reportOutcome,
|
|
1487
1653
|
sanitizeBrowserPayload,
|
|
1654
|
+
startCaughtExceptionSampler,
|
|
1488
1655
|
trackLlmCall,
|
|
1489
1656
|
withLlmCall,
|
|
1490
1657
|
withProcessSpan
|
package/dist/index.d.cts
CHANGED
|
@@ -100,6 +100,8 @@ interface AutterServerOptions {
|
|
|
100
100
|
/** Ingester base URL. Default: https://otlp.autter.dev */
|
|
101
101
|
endpoint?: string;
|
|
102
102
|
service: string;
|
|
103
|
+
/** Stable for this process lifetime; use an ECS task/container ID or pod UID plus restart count to correlate platform events. */
|
|
104
|
+
instanceId?: string;
|
|
103
105
|
environment?: string;
|
|
104
106
|
/** e.g. a git SHA. Maps to service.version / release. */
|
|
105
107
|
release?: string;
|
|
@@ -119,6 +121,8 @@ interface AutterServerOptions {
|
|
|
119
121
|
retainTracesAboveMs?: number;
|
|
120
122
|
/** Metric export interval. Default 60_000 ms. */
|
|
121
123
|
metricIntervalMs?: number;
|
|
124
|
+
/** Collect RSS, heap, limit, and GC signals per process instance. Default true. */
|
|
125
|
+
memoryMetrics?: boolean;
|
|
122
126
|
/** Capture crashing exceptions via process.uncaughtExceptionMonitor (default true). */
|
|
123
127
|
captureGlobalErrors?: boolean;
|
|
124
128
|
/**
|
|
@@ -206,6 +210,8 @@ interface AutterServer {
|
|
|
206
210
|
* errors with a lower severity so aggregations can slice one dataset.
|
|
207
211
|
*/
|
|
208
212
|
captureMessage(message: string, severity?: AutterSeverity, attributes?: Attributes): void;
|
|
213
|
+
/** Report a failed result from code that returned normally. Name must be stable. */
|
|
214
|
+
reportOutcome(name: string, message: string, attributes?: Attributes): void;
|
|
209
215
|
/**
|
|
210
216
|
* Wrap a named unit of work — a background job, queue consumer, cron
|
|
211
217
|
* tick, or DB-heavy call — in an always-recorded span. Unlike regular
|
|
@@ -264,6 +270,8 @@ declare function withLlmCall<T>(info: LlmCallInfo, fn: (llm: LlmCallHandle) => T
|
|
|
264
270
|
declare function trackLlmCall(call: TrackedLlmCall): void;
|
|
265
271
|
/** Module-level convenience for warnings/info — see AutterServer.captureMessage. */
|
|
266
272
|
declare function captureMessage(message: string, severity?: AutterSeverity, attributes?: Attributes): void;
|
|
273
|
+
/** Portable OTLP `autter.outcome` event for failed results without exceptions. */
|
|
274
|
+
declare function reportOutcome(name: string, message: string, attributes?: Attributes): void;
|
|
267
275
|
/** Capture surface with guaranteed redaction, regardless of how the host app
|
|
268
276
|
* configured initAutterServer — for library authors whose calls must never
|
|
269
277
|
* forward PII even when the host disabled it. Routes to the active server
|
|
@@ -322,6 +330,8 @@ interface AutoFlushOptions {
|
|
|
322
330
|
* signals, proceeding to exit anyway). Default 3000 ms.
|
|
323
331
|
*/
|
|
324
332
|
timeoutMs?: number;
|
|
333
|
+
/** Flush after a fatal uncaught exception, then exit with Node's crash code. Default true. */
|
|
334
|
+
flushOnUncaughtException?: boolean;
|
|
325
335
|
/** Signals handled besides the defaults. Default ["SIGINT", "SIGTERM"]. */
|
|
326
336
|
signals?: string[];
|
|
327
337
|
}
|
|
@@ -342,6 +352,8 @@ interface AutoFlushHandle {
|
|
|
342
352
|
* concurrently and let your handler decide the exit — installing this
|
|
343
353
|
* never changes an existing graceful-shutdown path. A second signal
|
|
344
354
|
* always exits immediately.
|
|
355
|
+
* - uncaughtException — delays Node's fatal exit only for the same bounded
|
|
356
|
+
* flush, then exits with its default crash status (1).
|
|
345
357
|
*
|
|
346
358
|
* Idempotent per options object; `initAutterServer` installs it by default
|
|
347
359
|
* (`autoFlush: false` opts out).
|
|
@@ -375,4 +387,12 @@ interface InstrumentLlmOptions {
|
|
|
375
387
|
}
|
|
376
388
|
declare function instrumentLlmClient<T extends object>(client: T, options?: InstrumentLlmOptions): T;
|
|
377
389
|
|
|
378
|
-
|
|
390
|
+
/** Opt-in diagnostic hook. V8 pauses briefly for every throw, including handled throws. */
|
|
391
|
+
declare function startCaughtExceptionSampler(options?: {
|
|
392
|
+
sampleRate?: number;
|
|
393
|
+
maxPerMinute?: number;
|
|
394
|
+
}): {
|
|
395
|
+
stop(): void;
|
|
396
|
+
};
|
|
397
|
+
|
|
398
|
+
export { type AutoFlushHandle, type AutoFlushOptions, type AutterServer, type AutterServerOptions, type AutterSeverity, type FlushTarget, type InstrumentLlmOptions, type LlmCallHandle, type LlmCallInfo, type LlmUsage, type RedactOptions, type RelayOptions, type SafeCapture, type TrackedLlmCall, captureException, captureMessage, createBrowserRelayFetchHandler, createBrowserRelayHandler, emitLlmSelftestTrace, initAutterServer, installAutterAutoFlush, instrumentLlmClient, makeSafeCapture, redactAttributes, reportOutcome, sanitizeBrowserPayload, startCaughtExceptionSampler, trackLlmCall, withLlmCall, withProcessSpan };
|
package/dist/index.d.ts
CHANGED
|
@@ -100,6 +100,8 @@ interface AutterServerOptions {
|
|
|
100
100
|
/** Ingester base URL. Default: https://otlp.autter.dev */
|
|
101
101
|
endpoint?: string;
|
|
102
102
|
service: string;
|
|
103
|
+
/** Stable for this process lifetime; use an ECS task/container ID or pod UID plus restart count to correlate platform events. */
|
|
104
|
+
instanceId?: string;
|
|
103
105
|
environment?: string;
|
|
104
106
|
/** e.g. a git SHA. Maps to service.version / release. */
|
|
105
107
|
release?: string;
|
|
@@ -119,6 +121,8 @@ interface AutterServerOptions {
|
|
|
119
121
|
retainTracesAboveMs?: number;
|
|
120
122
|
/** Metric export interval. Default 60_000 ms. */
|
|
121
123
|
metricIntervalMs?: number;
|
|
124
|
+
/** Collect RSS, heap, limit, and GC signals per process instance. Default true. */
|
|
125
|
+
memoryMetrics?: boolean;
|
|
122
126
|
/** Capture crashing exceptions via process.uncaughtExceptionMonitor (default true). */
|
|
123
127
|
captureGlobalErrors?: boolean;
|
|
124
128
|
/**
|
|
@@ -206,6 +210,8 @@ interface AutterServer {
|
|
|
206
210
|
* errors with a lower severity so aggregations can slice one dataset.
|
|
207
211
|
*/
|
|
208
212
|
captureMessage(message: string, severity?: AutterSeverity, attributes?: Attributes): void;
|
|
213
|
+
/** Report a failed result from code that returned normally. Name must be stable. */
|
|
214
|
+
reportOutcome(name: string, message: string, attributes?: Attributes): void;
|
|
209
215
|
/**
|
|
210
216
|
* Wrap a named unit of work — a background job, queue consumer, cron
|
|
211
217
|
* tick, or DB-heavy call — in an always-recorded span. Unlike regular
|
|
@@ -264,6 +270,8 @@ declare function withLlmCall<T>(info: LlmCallInfo, fn: (llm: LlmCallHandle) => T
|
|
|
264
270
|
declare function trackLlmCall(call: TrackedLlmCall): void;
|
|
265
271
|
/** Module-level convenience for warnings/info — see AutterServer.captureMessage. */
|
|
266
272
|
declare function captureMessage(message: string, severity?: AutterSeverity, attributes?: Attributes): void;
|
|
273
|
+
/** Portable OTLP `autter.outcome` event for failed results without exceptions. */
|
|
274
|
+
declare function reportOutcome(name: string, message: string, attributes?: Attributes): void;
|
|
267
275
|
/** Capture surface with guaranteed redaction, regardless of how the host app
|
|
268
276
|
* configured initAutterServer — for library authors whose calls must never
|
|
269
277
|
* forward PII even when the host disabled it. Routes to the active server
|
|
@@ -322,6 +330,8 @@ interface AutoFlushOptions {
|
|
|
322
330
|
* signals, proceeding to exit anyway). Default 3000 ms.
|
|
323
331
|
*/
|
|
324
332
|
timeoutMs?: number;
|
|
333
|
+
/** Flush after a fatal uncaught exception, then exit with Node's crash code. Default true. */
|
|
334
|
+
flushOnUncaughtException?: boolean;
|
|
325
335
|
/** Signals handled besides the defaults. Default ["SIGINT", "SIGTERM"]. */
|
|
326
336
|
signals?: string[];
|
|
327
337
|
}
|
|
@@ -342,6 +352,8 @@ interface AutoFlushHandle {
|
|
|
342
352
|
* concurrently and let your handler decide the exit — installing this
|
|
343
353
|
* never changes an existing graceful-shutdown path. A second signal
|
|
344
354
|
* always exits immediately.
|
|
355
|
+
* - uncaughtException — delays Node's fatal exit only for the same bounded
|
|
356
|
+
* flush, then exits with its default crash status (1).
|
|
345
357
|
*
|
|
346
358
|
* Idempotent per options object; `initAutterServer` installs it by default
|
|
347
359
|
* (`autoFlush: false` opts out).
|
|
@@ -375,4 +387,12 @@ interface InstrumentLlmOptions {
|
|
|
375
387
|
}
|
|
376
388
|
declare function instrumentLlmClient<T extends object>(client: T, options?: InstrumentLlmOptions): T;
|
|
377
389
|
|
|
378
|
-
|
|
390
|
+
/** Opt-in diagnostic hook. V8 pauses briefly for every throw, including handled throws. */
|
|
391
|
+
declare function startCaughtExceptionSampler(options?: {
|
|
392
|
+
sampleRate?: number;
|
|
393
|
+
maxPerMinute?: number;
|
|
394
|
+
}): {
|
|
395
|
+
stop(): void;
|
|
396
|
+
};
|
|
397
|
+
|
|
398
|
+
export { type AutoFlushHandle, type AutoFlushOptions, type AutterServer, type AutterServerOptions, type AutterSeverity, type FlushTarget, type InstrumentLlmOptions, type LlmCallHandle, type LlmCallInfo, type LlmUsage, type RedactOptions, type RelayOptions, type SafeCapture, type TrackedLlmCall, captureException, captureMessage, createBrowserRelayFetchHandler, createBrowserRelayHandler, emitLlmSelftestTrace, initAutterServer, installAutterAutoFlush, instrumentLlmClient, makeSafeCapture, redactAttributes, reportOutcome, sanitizeBrowserPayload, startCaughtExceptionSampler, trackLlmCall, withLlmCall, withProcessSpan };
|
package/dist/index.js
CHANGED
|
@@ -29,7 +29,10 @@ var EVENT_TYPES = /* @__PURE__ */ new Set([
|
|
|
29
29
|
"unhandled_rejection",
|
|
30
30
|
"message",
|
|
31
31
|
"session_start",
|
|
32
|
-
"track_event"
|
|
32
|
+
"track_event",
|
|
33
|
+
"outcome",
|
|
34
|
+
"request_failure",
|
|
35
|
+
"timing"
|
|
33
36
|
]);
|
|
34
37
|
var SEVERITIES = /* @__PURE__ */ new Set(["fatal", "error", "warning", "info"]);
|
|
35
38
|
function sanitizeBrowserPayload(raw) {
|
|
@@ -51,6 +54,7 @@ function sanitizeBrowserPayload(raw) {
|
|
|
51
54
|
...typeof e.severity === "string" && SEVERITIES.has(e.severity) ? { severity: e.severity } : {},
|
|
52
55
|
message: typeof e.message === "string" ? e.message.slice(0, 4e3) : "",
|
|
53
56
|
...typeof e.name === "string" ? { name: e.name.slice(0, 200) } : {},
|
|
57
|
+
...typeof e.durationMs === "number" && Number.isFinite(e.durationMs) ? { durationMs: Math.max(0, Math.min(12e4, e.durationMs)) } : {},
|
|
54
58
|
...typeof e.stack === "string" ? { stack: e.stack.slice(0, 32e3) } : {},
|
|
55
59
|
...typeof e.errorType === "string" ? { errorType: e.errorType.slice(0, 200) } : {},
|
|
56
60
|
...typeof e.filename === "string" ? { filename: e.filename.split("?")[0].slice(0, 1e3) } : {},
|
|
@@ -303,6 +307,7 @@ function installAutterAutoFlush(options = {}) {
|
|
|
303
307
|
const warnOnUnflushedExit = options.warnOnUnflushedExit !== false;
|
|
304
308
|
const timeoutMs = Math.max(100, options.timeoutMs ?? 3e3);
|
|
305
309
|
const signals = options.signals ?? DEFAULT_SIGNALS;
|
|
310
|
+
const flushOnUncaughtException = options.flushOnUncaughtException !== false;
|
|
306
311
|
let disposed = false;
|
|
307
312
|
let inFlight = null;
|
|
308
313
|
const doFlush = async () => {
|
|
@@ -348,6 +353,19 @@ function installAutterAutoFlush(options = {}) {
|
|
|
348
353
|
return inFlight;
|
|
349
354
|
};
|
|
350
355
|
const signalCounts = /* @__PURE__ */ new Map();
|
|
356
|
+
let handlingUncaughtException = false;
|
|
357
|
+
const onUncaughtException = (error) => {
|
|
358
|
+
if (process.listenerCount("uncaughtException") > 1) return;
|
|
359
|
+
if (handlingUncaughtException) return;
|
|
360
|
+
handlingUncaughtException = true;
|
|
361
|
+
logger.error(error instanceof Error ? error.stack ?? error.message : String(error));
|
|
362
|
+
const keepAlive = setTimeout(() => {
|
|
363
|
+
}, timeoutMs + 50);
|
|
364
|
+
void flush("uncaughtException").finally(() => {
|
|
365
|
+
clearTimeout(keepAlive);
|
|
366
|
+
process.exit(1);
|
|
367
|
+
});
|
|
368
|
+
};
|
|
351
369
|
const onSignal = (signal) => {
|
|
352
370
|
const count = (signalCounts.get(signal) ?? 0) + 1;
|
|
353
371
|
signalCounts.set(signal, count);
|
|
@@ -387,6 +405,9 @@ function installAutterAutoFlush(options = {}) {
|
|
|
387
405
|
};
|
|
388
406
|
const signalHandlers = /* @__PURE__ */ new Map();
|
|
389
407
|
process.on("beforeExit", onBeforeExit);
|
|
408
|
+
if (flushOnUncaughtException) {
|
|
409
|
+
process.on("uncaughtException", onUncaughtException);
|
|
410
|
+
}
|
|
390
411
|
for (const signal of signals) {
|
|
391
412
|
const handler = () => onSignal(signal);
|
|
392
413
|
signalHandlers.set(signal, handler);
|
|
@@ -398,6 +419,7 @@ function installAutterAutoFlush(options = {}) {
|
|
|
398
419
|
dispose() {
|
|
399
420
|
disposed = true;
|
|
400
421
|
process.removeListener("beforeExit", onBeforeExit);
|
|
422
|
+
process.removeListener("uncaughtException", onUncaughtException);
|
|
401
423
|
process.removeListener("exit", onExit);
|
|
402
424
|
for (const [signal, handler] of signalHandlers) {
|
|
403
425
|
process.removeListener(signal, handler);
|
|
@@ -653,6 +675,51 @@ function makeRedactor(options) {
|
|
|
653
675
|
return (attributes) => redactWith(attributes, r);
|
|
654
676
|
}
|
|
655
677
|
|
|
678
|
+
// src/memory.ts
|
|
679
|
+
import { PerformanceObserver } from "perf_hooks";
|
|
680
|
+
import { getHeapStatistics } from "v8";
|
|
681
|
+
import { metrics } from "@opentelemetry/api";
|
|
682
|
+
function startMemoryMetrics() {
|
|
683
|
+
const meter = metrics.getMeter("autter-process-memory");
|
|
684
|
+
const rss = meter.createObservableGauge("autter.process.memory.rss", { unit: "By" });
|
|
685
|
+
const heapUsed = meter.createObservableGauge("autter.process.memory.heap.used", { unit: "By" });
|
|
686
|
+
const heapLimit = meter.createObservableGauge("autter.process.memory.heap.limit", { unit: "By" });
|
|
687
|
+
const containerLimit = meter.createObservableGauge("autter.process.memory.limit", { unit: "By" });
|
|
688
|
+
const afterGc = meter.createObservableGauge("autter.process.memory.heap.after_gc", { unit: "By" });
|
|
689
|
+
const gcCount = meter.createObservableCounter("autter.process.gc.count", { unit: "{collection}" });
|
|
690
|
+
const gcDuration = meter.createObservableCounter("autter.process.gc.duration", { unit: "ms" });
|
|
691
|
+
let collections = 0;
|
|
692
|
+
let durationMs = 0;
|
|
693
|
+
let lastAfterGc = null;
|
|
694
|
+
const callbacks = [
|
|
695
|
+
[rss, (result) => result.observe(process.memoryUsage().rss)],
|
|
696
|
+
[heapUsed, (result) => result.observe(process.memoryUsage().heapUsed)],
|
|
697
|
+
[heapLimit, (result) => result.observe(getHeapStatistics().heap_size_limit)],
|
|
698
|
+
[containerLimit, (result) => {
|
|
699
|
+
const constrained = process.constrainedMemory?.() ?? 0;
|
|
700
|
+
if (constrained > 0) result.observe(constrained);
|
|
701
|
+
}],
|
|
702
|
+
[afterGc, (result) => {
|
|
703
|
+
if (lastAfterGc !== null) result.observe(lastAfterGc);
|
|
704
|
+
}],
|
|
705
|
+
[gcCount, (result) => result.observe(collections)],
|
|
706
|
+
[gcDuration, (result) => result.observe(durationMs)]
|
|
707
|
+
];
|
|
708
|
+
for (const [instrument, callback] of callbacks) instrument.addCallback(callback);
|
|
709
|
+
const observer = new PerformanceObserver((list) => {
|
|
710
|
+
for (const entry of list.getEntries()) {
|
|
711
|
+
collections++;
|
|
712
|
+
durationMs += entry.duration;
|
|
713
|
+
lastAfterGc = process.memoryUsage().heapUsed;
|
|
714
|
+
}
|
|
715
|
+
});
|
|
716
|
+
observer.observe({ entryTypes: ["gc"] });
|
|
717
|
+
return () => {
|
|
718
|
+
observer.disconnect();
|
|
719
|
+
for (const [instrument, callback] of callbacks) instrument.removeCallback(callback);
|
|
720
|
+
};
|
|
721
|
+
}
|
|
722
|
+
|
|
656
723
|
// src/server.ts
|
|
657
724
|
var LLM_MARKER_ATTRIBUTES = [
|
|
658
725
|
"gen_ai.operation.name",
|
|
@@ -985,7 +1052,7 @@ function initAutterServer(options) {
|
|
|
985
1052
|
activeRedactor = makeRedactor(options.redactAttributes ?? true);
|
|
986
1053
|
const resource = new Resource({
|
|
987
1054
|
[ATTR_SERVICE_NAME]: options.service,
|
|
988
|
-
"service.instance.id": randomUUID(),
|
|
1055
|
+
"service.instance.id": (options.instanceId || process.env.AUTTER_RUNTIME_INSTANCE_ID || randomUUID()).slice(0, 128),
|
|
989
1056
|
...options.release ? { [ATTR_SERVICE_VERSION]: options.release } : {},
|
|
990
1057
|
"deployment.environment": environment,
|
|
991
1058
|
// Tells the ingester request metrics arrive on the metrics pipe, so
|
|
@@ -1050,6 +1117,7 @@ function initAutterServer(options) {
|
|
|
1050
1117
|
]
|
|
1051
1118
|
});
|
|
1052
1119
|
sdk.start();
|
|
1120
|
+
const stopMemoryMetrics = options.memoryMetrics === false ? null : startMemoryMetrics();
|
|
1053
1121
|
const alwaysOnProvider = new BasicTracerProvider({
|
|
1054
1122
|
resource,
|
|
1055
1123
|
sampler: new AlwaysOnSampler(),
|
|
@@ -1076,13 +1144,24 @@ function initAutterServer(options) {
|
|
|
1076
1144
|
const span = reuseActive ? activeSpan : errorTracer.startSpan(isError ? error.name : "Error");
|
|
1077
1145
|
span.setAttributes({ "autter.severity": "error", ...redacted });
|
|
1078
1146
|
if (isError && error.stack) {
|
|
1079
|
-
|
|
1147
|
+
if (redacted["autter.handled"] === true) {
|
|
1148
|
+
span.addEvent("exception", {
|
|
1149
|
+
"exception.type": error.name,
|
|
1150
|
+
"exception.message": message,
|
|
1151
|
+
"exception.stacktrace": error.stack,
|
|
1152
|
+
"autter.handled": true,
|
|
1153
|
+
"autter.sampled": redacted["autter.sampled"] === true
|
|
1154
|
+
});
|
|
1155
|
+
} else {
|
|
1156
|
+
span.recordException(error);
|
|
1157
|
+
}
|
|
1080
1158
|
} else {
|
|
1081
1159
|
const stack = new Error().stack?.split("\n").filter((line, i) => i === 0 || !line.includes("captureException")).join("\n");
|
|
1082
1160
|
span.addEvent("exception", {
|
|
1083
1161
|
"exception.type": isError ? error.name : "Error",
|
|
1084
1162
|
"exception.message": message,
|
|
1085
|
-
...stack ? { "exception.stacktrace": stack } : {}
|
|
1163
|
+
...stack ? { "exception.stacktrace": stack } : {},
|
|
1164
|
+
...redacted["autter.handled"] === true ? { "autter.handled": true, "autter.sampled": redacted["autter.sampled"] === true } : {}
|
|
1086
1165
|
});
|
|
1087
1166
|
}
|
|
1088
1167
|
span.setStatus({ code: SpanStatusCode.ERROR, message });
|
|
@@ -1106,15 +1185,28 @@ function initAutterServer(options) {
|
|
|
1106
1185
|
span.setStatus({ code: SpanStatusCode.ERROR, message });
|
|
1107
1186
|
span.end();
|
|
1108
1187
|
}
|
|
1188
|
+
function reportOutcome2(name, message, attributes) {
|
|
1189
|
+
errorTraceBuffer?.retainActiveTrace();
|
|
1190
|
+
telemetryStats.markCaptured();
|
|
1191
|
+
const activeSpan = trace.getActiveSpan();
|
|
1192
|
+
const reuseActive = activeSpan?.isRecording() === true;
|
|
1193
|
+
const span = reuseActive ? activeSpan : errorTracer.startSpan("OutcomeFailure");
|
|
1194
|
+
const safeOutcome = activeRedactor({ "autter.outcome.name": name.slice(0, 200), "autter.outcome.message": message.slice(0, 1e3) });
|
|
1195
|
+
const safeName = String(safeOutcome["autter.outcome.name"] ?? "operation");
|
|
1196
|
+
const safeMessage = String(safeOutcome["autter.outcome.message"] ?? "failed");
|
|
1197
|
+
span.addEvent("autter.outcome", {
|
|
1198
|
+
...activeRedactor(attributes),
|
|
1199
|
+
"autter.outcome.status": "error",
|
|
1200
|
+
"autter.outcome.name": safeName,
|
|
1201
|
+
"autter.outcome.message": safeMessage
|
|
1202
|
+
});
|
|
1203
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: safeMessage });
|
|
1204
|
+
if (!reuseActive) span.end();
|
|
1205
|
+
}
|
|
1109
1206
|
if (options.captureGlobalErrors !== false) {
|
|
1110
1207
|
process.on("uncaughtExceptionMonitor", (err) => {
|
|
1111
1208
|
captureException2(err, { "autter.unhandled": true });
|
|
1112
|
-
void flushTarget.forceFlush()
|
|
1113
|
-
});
|
|
1114
|
-
process.on("unhandledRejection", (reason) => {
|
|
1115
|
-
captureException2(reason, {
|
|
1116
|
-
"autter.unhandled": true,
|
|
1117
|
-
...reason instanceof Error ? {} : { "autter.severity": "warning" }
|
|
1209
|
+
if (options.autoFlush === false) void Promise.resolve(flushTarget.forceFlush()).catch(() => {
|
|
1118
1210
|
});
|
|
1119
1211
|
});
|
|
1120
1212
|
}
|
|
@@ -1142,10 +1234,12 @@ function initAutterServer(options) {
|
|
|
1142
1234
|
const server = {
|
|
1143
1235
|
captureException: captureException2,
|
|
1144
1236
|
captureMessage: captureMessage2,
|
|
1237
|
+
reportOutcome: reportOutcome2,
|
|
1145
1238
|
withProcessSpan: (name, fn, attributes) => runWithSpan(processTracer, name, fn, activeRedactor(attributes)),
|
|
1146
1239
|
withLlmCall: (info, fn) => runLlmSpan(llmTracer, info, fn),
|
|
1147
1240
|
trackLlmCall: (call) => recordLlmCall(llmTracer, call),
|
|
1148
1241
|
shutdown: async () => {
|
|
1242
|
+
stopMemoryMetrics?.();
|
|
1149
1243
|
active = null;
|
|
1150
1244
|
activeAlwaysOnProvider = null;
|
|
1151
1245
|
autoFlushHandle?.dispose();
|
|
@@ -1203,6 +1297,21 @@ function captureMessage(message, severity = "warning", attributes) {
|
|
|
1203
1297
|
span.setStatus({ code: SpanStatusCode.ERROR, message });
|
|
1204
1298
|
span.end();
|
|
1205
1299
|
}
|
|
1300
|
+
function reportOutcome(name, message, attributes) {
|
|
1301
|
+
if (active) return active.reportOutcome(name, message, attributes);
|
|
1302
|
+
const span = trace.getTracer("autter-outcomes").startSpan("OutcomeFailure");
|
|
1303
|
+
const safeOutcome = redactAttributes({ "autter.outcome.name": name.slice(0, 200), "autter.outcome.message": message.slice(0, 1e3) });
|
|
1304
|
+
const safeName = String(safeOutcome["autter.outcome.name"] ?? "operation");
|
|
1305
|
+
const safeMessage = String(safeOutcome["autter.outcome.message"] ?? "failed");
|
|
1306
|
+
span.addEvent("autter.outcome", {
|
|
1307
|
+
...redactAttributes(attributes),
|
|
1308
|
+
"autter.outcome.status": "error",
|
|
1309
|
+
"autter.outcome.name": safeName,
|
|
1310
|
+
"autter.outcome.message": safeMessage
|
|
1311
|
+
});
|
|
1312
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: safeMessage });
|
|
1313
|
+
span.end();
|
|
1314
|
+
}
|
|
1206
1315
|
function makeSafeCapture(options) {
|
|
1207
1316
|
const clean = (attributes) => attributes ? { ...redactAttributes(attributes, options) } : void 0;
|
|
1208
1317
|
return {
|
|
@@ -1455,6 +1564,50 @@ function instrumentLlmClient(client, options) {
|
|
|
1455
1564
|
}
|
|
1456
1565
|
return deepProxy(client);
|
|
1457
1566
|
}
|
|
1567
|
+
|
|
1568
|
+
// src/caught-exceptions.ts
|
|
1569
|
+
import inspector from "inspector";
|
|
1570
|
+
function startCaughtExceptionSampler(options = {}) {
|
|
1571
|
+
const rate = Math.max(0, Math.min(1, options.sampleRate ?? 0.01));
|
|
1572
|
+
const cap = Math.max(1, Math.min(100, options.maxPerMinute ?? 10));
|
|
1573
|
+
const session = new inspector.Session();
|
|
1574
|
+
let active2 = true;
|
|
1575
|
+
let minute = 0;
|
|
1576
|
+
let sent = 0;
|
|
1577
|
+
session.connect();
|
|
1578
|
+
session.on("Debugger.paused", (message) => {
|
|
1579
|
+
session.post("Debugger.resume", () => {
|
|
1580
|
+
});
|
|
1581
|
+
if (!active2 || message.params.reason !== "exception") return;
|
|
1582
|
+
const nowMinute = Math.floor(Date.now() / 6e4);
|
|
1583
|
+
if (nowMinute !== minute) {
|
|
1584
|
+
minute = nowMinute;
|
|
1585
|
+
sent = 0;
|
|
1586
|
+
}
|
|
1587
|
+
if (sent >= cap || Math.random() >= rate) return;
|
|
1588
|
+
sent++;
|
|
1589
|
+
const frames = message.params.callFrames.slice(0, 8).filter((frame) => frame.url && !frame.url.startsWith("node:")).map((frame) => ` at ${frame.functionName || "anonymous"} (${frame.url.split("?")[0]}:${frame.location.lineNumber + 1}:${(frame.location.columnNumber ?? 0) + 1})`);
|
|
1590
|
+
if (!frames.length) return;
|
|
1591
|
+
const error = new Error("Sampled caught exception");
|
|
1592
|
+
error.name = "CaughtException";
|
|
1593
|
+
error.stack = `${error.name}: ${error.message}
|
|
1594
|
+
${frames.join("\n")}`;
|
|
1595
|
+
queueMicrotask(() => {
|
|
1596
|
+
if (active2) captureException(error, { "autter.handled": true, "autter.sampled": true });
|
|
1597
|
+
});
|
|
1598
|
+
});
|
|
1599
|
+
session.post("Debugger.enable", (error) => {
|
|
1600
|
+
if (!error && active2) session.post("Debugger.setPauseOnExceptions", { state: "all" }, () => {
|
|
1601
|
+
});
|
|
1602
|
+
});
|
|
1603
|
+
return { stop() {
|
|
1604
|
+
if (!active2) return;
|
|
1605
|
+
active2 = false;
|
|
1606
|
+
session.post("Debugger.setPauseOnExceptions", { state: "none" }, () => {
|
|
1607
|
+
session.post("Debugger.disable", () => session.disconnect());
|
|
1608
|
+
});
|
|
1609
|
+
} };
|
|
1610
|
+
}
|
|
1458
1611
|
export {
|
|
1459
1612
|
captureException,
|
|
1460
1613
|
captureMessage,
|
|
@@ -1466,7 +1619,9 @@ export {
|
|
|
1466
1619
|
instrumentLlmClient,
|
|
1467
1620
|
makeSafeCapture,
|
|
1468
1621
|
redactAttributes,
|
|
1622
|
+
reportOutcome,
|
|
1469
1623
|
sanitizeBrowserPayload,
|
|
1624
|
+
startCaughtExceptionSampler,
|
|
1470
1625
|
trackLlmCall,
|
|
1471
1626
|
withLlmCall,
|
|
1472
1627
|
withProcessSpan
|
package/package.json
CHANGED