@autter/otlp-ingester 1.3.4 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clickhouse.d.ts +2 -0
- package/dist/clickhouse.js +15 -0
- package/dist/context.d.ts +19 -0
- package/dist/context.js +92 -0
- package/dist/logs.d.ts +42 -0
- package/dist/logs.js +126 -0
- package/dist/migrations.js +2 -0
- package/dist/normalize-otlp.d.ts +2 -6
- package/dist/normalize-otlp.js +8 -7
- package/dist/otlp-proto.d.ts +2 -0
- package/dist/otlp-proto.js +21 -0
- package/dist/server.js +25 -1
- package/package.json +1 -1
package/dist/clickhouse.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { IngesterConfig } from "./config.js";
|
|
|
2
2
|
import { type LatencyHistogram } from "./latency.js";
|
|
3
3
|
import { type ProfileSample } from "./profiles.js";
|
|
4
4
|
import { type MemorySample, type PlatformEvent } from "./memory.js";
|
|
5
|
+
import { type RuntimeLogRecord } from "./logs.js";
|
|
5
6
|
import { type Migration } from "./migrations.js";
|
|
6
7
|
import type { IngestContext, RuntimeLlmCall, RuntimeMetricPoint, RuntimeOccurrence, RuntimeSpanRow } from "./types.js";
|
|
7
8
|
export declare class ClickHouseStore {
|
|
@@ -25,6 +26,7 @@ export declare class ClickHouseStore {
|
|
|
25
26
|
ping(): Promise<boolean>;
|
|
26
27
|
insertOccurrences(ctx: IngestContext, occurrences: RuntimeOccurrence[]): Promise<void>;
|
|
27
28
|
insertSpans(ctx: IngestContext, spans: RuntimeSpanRow[]): Promise<void>;
|
|
29
|
+
insertLogs(ctx: IngestContext, logs: RuntimeLogRecord[]): Promise<void>;
|
|
28
30
|
insertProfileSamples(ctx: IngestContext, samples: ProfileSample[]): Promise<void>;
|
|
29
31
|
insertMemorySamples(ctx: IngestContext, samples: MemorySample[]): Promise<void>;
|
|
30
32
|
insertPlatformEvent(ctx: IngestContext, event: PlatformEvent): Promise<void>;
|
package/dist/clickhouse.js
CHANGED
|
@@ -3,6 +3,7 @@ import { latencyTableDDL } from "./latency.js";
|
|
|
3
3
|
import { profileTableDDL } from "./profiles.js";
|
|
4
4
|
import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
|
|
5
5
|
import { sourceMapTableDDL } from "./source-maps.js";
|
|
6
|
+
import { logTableDDL } from "./logs.js";
|
|
6
7
|
import { MIGRATIONS, migrationsTableDDL, } from "./migrations.js";
|
|
7
8
|
/**
|
|
8
9
|
* ClickHouse persistence. The schema is idempotent and applied lazily once
|
|
@@ -53,6 +54,7 @@ export class ClickHouseStore {
|
|
|
53
54
|
memoryTableDDL(db),
|
|
54
55
|
platformEventTableDDL(db),
|
|
55
56
|
sourceMapTableDDL(db),
|
|
57
|
+
logTableDDL(db),
|
|
56
58
|
`CREATE TABLE IF NOT EXISTS ${db}.runtime_error_occurrences (
|
|
57
59
|
org_id String,
|
|
58
60
|
repository_id String,
|
|
@@ -265,6 +267,19 @@ export class ClickHouseStore {
|
|
|
265
267
|
})),
|
|
266
268
|
});
|
|
267
269
|
}
|
|
270
|
+
async insertLogs(ctx, logs) {
|
|
271
|
+
if (!logs.length)
|
|
272
|
+
return;
|
|
273
|
+
if (!this.configured)
|
|
274
|
+
throw new Error("CLICKHOUSE_URL is not configured");
|
|
275
|
+
await this.ensureSchema();
|
|
276
|
+
await this.getClient().insert({ table: this.table("runtime_logs"), format: "JSONEachRow", clickhouse_settings: INSERT_SETTINGS,
|
|
277
|
+
values: logs.map((row) => ({ org_id: ctx.orgId, repository_id: ctx.repositoryId, event_id: row.id,
|
|
278
|
+
service: row.service, environment: row.environment, release: row.release, trace_id: row.traceId,
|
|
279
|
+
span_id: row.spanId, operation_id: row.operationId, operation: row.operation, event_type: row.type,
|
|
280
|
+
severity: row.severity, message: row.message, outcome: row.outcome, duration_ms: row.durationMs,
|
|
281
|
+
attributes: JSON.stringify(row.attributes), occurred_at: row.occurredAt.toISOString() })) });
|
|
282
|
+
}
|
|
268
283
|
async insertProfileSamples(ctx, samples) {
|
|
269
284
|
if (!samples.length)
|
|
270
285
|
return;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */
|
|
2
|
+
export declare function sanitizeRuntimeContext(input: unknown): Record<string, unknown>;
|
|
3
|
+
export interface OtlpValue {
|
|
4
|
+
stringValue?: string;
|
|
5
|
+
intValue?: string | number;
|
|
6
|
+
doubleValue?: number;
|
|
7
|
+
boolValue?: boolean;
|
|
8
|
+
arrayValue?: {
|
|
9
|
+
values?: OtlpValue[];
|
|
10
|
+
};
|
|
11
|
+
kvlistValue?: {
|
|
12
|
+
values?: OtlpAttribute[];
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export interface OtlpAttribute {
|
|
16
|
+
key?: string;
|
|
17
|
+
value?: OtlpValue;
|
|
18
|
+
}
|
|
19
|
+
export declare function decodeOtlpAttributes(attributes: OtlpAttribute[] | undefined): Record<string, unknown>;
|
package/dist/context.js
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */
|
|
2
|
+
export function sanitizeRuntimeContext(input) {
|
|
3
|
+
let budget = 512;
|
|
4
|
+
const seen = new WeakSet();
|
|
5
|
+
const scrub = (value) => value
|
|
6
|
+
.replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, "[redacted]")
|
|
7
|
+
.replace(/\b(?:bearer\s+[A-Za-z0-9._~+\/=~-]{10,}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+|gh[pousr]_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{20,}|xox[baprs]-[A-Za-z0-9-]{10,}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|(?:AKIA|ASIA)[A-Z0-9]{16})\b/gi, "[redacted]")
|
|
8
|
+
.replace(/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g, "[redacted]")
|
|
9
|
+
.replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/:@]+:[^\s/@]+@/gi, "$1[redacted]@")
|
|
10
|
+
.replace(/(https?:\/\/[^\s?#]+)[?#][^\s]*/gi, "$1");
|
|
11
|
+
const visit = (value, key, depth) => {
|
|
12
|
+
if (--budget < 0 || depth > 6)
|
|
13
|
+
return "[truncated]";
|
|
14
|
+
if (/__proto__|constructor|prototype/i.test(key))
|
|
15
|
+
return undefined;
|
|
16
|
+
const usage = /(?:^|\.)(?:input|output|total|prompt|completion)_?tokens$/i.test(key) &&
|
|
17
|
+
typeof value === "number" &&
|
|
18
|
+
Number.isFinite(value) &&
|
|
19
|
+
value >= 0;
|
|
20
|
+
if (!usage &&
|
|
21
|
+
/password|passwd|secret|token|credential|authorization|cookie|email|phone|ssn|card[._-]?number|connection[._-]?string|api[._-]?key|private[._-]?key|request[._-]?body|response[._-]?body|headers|url[._-]?query/i.test(key))
|
|
22
|
+
return "[redacted]";
|
|
23
|
+
if (typeof value === "string") {
|
|
24
|
+
// Serialized custom context crosses the same privacy boundary as nested values.
|
|
25
|
+
if (/^\s*[\[{]/.test(value)) {
|
|
26
|
+
try {
|
|
27
|
+
return visit(JSON.parse(value), key, depth + 1);
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
/* plain text */
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const text = /(?:url|path|route|target)$/i.test(key)
|
|
34
|
+
? value.split(/[?#]/)[0]
|
|
35
|
+
: value;
|
|
36
|
+
return scrub(text).slice(0, /stack/i.test(key) ? 32000 : 2048);
|
|
37
|
+
}
|
|
38
|
+
if (typeof value === "number")
|
|
39
|
+
return Number.isFinite(value) ? value : undefined;
|
|
40
|
+
if (typeof value === "boolean" || value === null)
|
|
41
|
+
return value;
|
|
42
|
+
if (!value || typeof value !== "object")
|
|
43
|
+
return undefined;
|
|
44
|
+
if (seen.has(value))
|
|
45
|
+
return "[circular]";
|
|
46
|
+
seen.add(value);
|
|
47
|
+
if (Array.isArray(value))
|
|
48
|
+
return value.slice(0, 64).map((v) => visit(v, key, depth + 1));
|
|
49
|
+
const result = {};
|
|
50
|
+
for (const [k, v] of Object.entries(value).slice(0, 128)) {
|
|
51
|
+
const safe = visit(v, k, depth + 1);
|
|
52
|
+
if (safe !== undefined)
|
|
53
|
+
result[k.slice(0, 200)] = safe;
|
|
54
|
+
}
|
|
55
|
+
return result;
|
|
56
|
+
};
|
|
57
|
+
const result = visit(input, "", 0);
|
|
58
|
+
return result && typeof result === "object" && !Array.isArray(result)
|
|
59
|
+
? result
|
|
60
|
+
: {};
|
|
61
|
+
}
|
|
62
|
+
export function decodeOtlpAttributes(attributes) {
|
|
63
|
+
const decode = (v, depth = 0) => {
|
|
64
|
+
if (!v || depth > 6)
|
|
65
|
+
return undefined;
|
|
66
|
+
if (v.stringValue !== undefined)
|
|
67
|
+
return v.stringValue;
|
|
68
|
+
if (v.intValue !== undefined)
|
|
69
|
+
return Number(v.intValue);
|
|
70
|
+
if (v.doubleValue !== undefined)
|
|
71
|
+
return v.doubleValue;
|
|
72
|
+
if (v.boolValue !== undefined)
|
|
73
|
+
return v.boolValue;
|
|
74
|
+
if (v.arrayValue)
|
|
75
|
+
return (v.arrayValue.values ?? [])
|
|
76
|
+
.slice(0, 64)
|
|
77
|
+
.map((value) => decode(value, depth + 1));
|
|
78
|
+
if (v.kvlistValue)
|
|
79
|
+
return Object.fromEntries((v.kvlistValue.values ?? [])
|
|
80
|
+
.slice(0, 128)
|
|
81
|
+
.filter((a) => a.key && !/^(?:__proto__|constructor|prototype)$/.test(a.key))
|
|
82
|
+
.map((a) => [a.key, decode(a.value, depth + 1)]));
|
|
83
|
+
return undefined;
|
|
84
|
+
};
|
|
85
|
+
return sanitizeRuntimeContext(Object.fromEntries((attributes ?? [])
|
|
86
|
+
.slice()
|
|
87
|
+
.sort((a, b) => Number(/^autter\.(?:operation|event)\./.test(b.key ?? "")) -
|
|
88
|
+
Number(/^autter\.(?:operation|event)\./.test(a.key ?? "")))
|
|
89
|
+
.slice(0, 128)
|
|
90
|
+
.filter((a) => a.key && !/^(?:__proto__|constructor|prototype)$/.test(a.key))
|
|
91
|
+
.map((a) => [a.key, decode(a.value)])));
|
|
92
|
+
}
|
package/dist/logs.d.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { type OtlpAttribute, type OtlpValue } from "./context.js";
|
|
2
|
+
export interface RuntimeLogRecord {
|
|
3
|
+
id: string;
|
|
4
|
+
service: string;
|
|
5
|
+
environment: string;
|
|
6
|
+
release: string;
|
|
7
|
+
traceId: string;
|
|
8
|
+
spanId: string;
|
|
9
|
+
operationId: string;
|
|
10
|
+
operation: string;
|
|
11
|
+
type: "log" | "operation";
|
|
12
|
+
severity: string;
|
|
13
|
+
message: string;
|
|
14
|
+
outcome: string;
|
|
15
|
+
durationMs: number;
|
|
16
|
+
attributes: Record<string, unknown>;
|
|
17
|
+
occurredAt: Date;
|
|
18
|
+
}
|
|
19
|
+
export interface OtlpLogsRequest {
|
|
20
|
+
resourceLogs?: Array<{
|
|
21
|
+
resource?: {
|
|
22
|
+
attributes?: OtlpAttribute[];
|
|
23
|
+
};
|
|
24
|
+
scopeLogs?: Array<{
|
|
25
|
+
scope?: {
|
|
26
|
+
name?: string;
|
|
27
|
+
};
|
|
28
|
+
logRecords?: Array<{
|
|
29
|
+
timeUnixNano?: string | number;
|
|
30
|
+
observedTimeUnixNano?: string | number;
|
|
31
|
+
severityNumber?: number;
|
|
32
|
+
severityText?: string;
|
|
33
|
+
traceId?: string;
|
|
34
|
+
spanId?: string;
|
|
35
|
+
body?: OtlpValue;
|
|
36
|
+
attributes?: OtlpAttribute[];
|
|
37
|
+
}>;
|
|
38
|
+
}>;
|
|
39
|
+
}>;
|
|
40
|
+
}
|
|
41
|
+
export declare function normalizeLogs(request: OtlpLogsRequest): RuntimeLogRecord[];
|
|
42
|
+
export declare function logTableDDL(db: string): string;
|
package/dist/logs.js
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { decodeOtlpAttributes, sanitizeRuntimeContext, } from "./context.js";
|
|
3
|
+
import { normalizeRoute } from "./fingerprint.js";
|
|
4
|
+
export function normalizeLogs(request) {
|
|
5
|
+
if (!request || !Array.isArray(request.resourceLogs))
|
|
6
|
+
throw new Error("resourceLogs must be an array");
|
|
7
|
+
const rows = [];
|
|
8
|
+
if (request.resourceLogs.length > 128)
|
|
9
|
+
throw new Error("too many log resources");
|
|
10
|
+
for (const resource of request.resourceLogs) {
|
|
11
|
+
if (!resource ||
|
|
12
|
+
(resource.scopeLogs && !Array.isArray(resource.scopeLogs)) ||
|
|
13
|
+
(resource.scopeLogs?.length ?? 0) > 128)
|
|
14
|
+
throw new Error("invalid log scopes");
|
|
15
|
+
const info = decodeOtlpAttributes(resource.resource?.attributes);
|
|
16
|
+
for (const scope of resource.scopeLogs ?? []) {
|
|
17
|
+
if (!scope || (scope.logRecords && !Array.isArray(scope.logRecords)))
|
|
18
|
+
throw new Error("invalid log records");
|
|
19
|
+
for (const record of scope.logRecords ?? []) {
|
|
20
|
+
if (!record || typeof record !== "object")
|
|
21
|
+
throw new Error("invalid log record");
|
|
22
|
+
if (rows.length >= 2000)
|
|
23
|
+
throw new Error("too many log records");
|
|
24
|
+
const attrs = decodeOtlpAttributes(record.attributes);
|
|
25
|
+
let event = {};
|
|
26
|
+
const decodedBody = decodeOtlpAttributes([
|
|
27
|
+
{ key: "event", value: record.body },
|
|
28
|
+
]).event;
|
|
29
|
+
const body = typeof decodedBody === "string"
|
|
30
|
+
? decodedBody
|
|
31
|
+
: decodedBody == null
|
|
32
|
+
? ""
|
|
33
|
+
: JSON.stringify(decodedBody);
|
|
34
|
+
if (decodedBody &&
|
|
35
|
+
typeof decodedBody === "object" &&
|
|
36
|
+
!Array.isArray(decodedBody))
|
|
37
|
+
event = sanitizeRuntimeContext(decodedBody);
|
|
38
|
+
const metadata = Object.fromEntries(Object.entries(attrs).filter(([key]) => /^autter\.(?:operation|event)\./.test(key)));
|
|
39
|
+
const attributes = sanitizeRuntimeContext({
|
|
40
|
+
...metadata,
|
|
41
|
+
...event,
|
|
42
|
+
...attrs,
|
|
43
|
+
});
|
|
44
|
+
const nanos = record.timeUnixNano ?? record.observedTimeUnixNano;
|
|
45
|
+
const occurredAt = nanos === undefined
|
|
46
|
+
? new Date()
|
|
47
|
+
: new Date(Number(BigInt(String(nanos)) / 1000000n));
|
|
48
|
+
if (!Number.isFinite(occurredAt.getTime()))
|
|
49
|
+
throw new Error("invalid log timestamp");
|
|
50
|
+
const number = Number(record.severityNumber ?? 9);
|
|
51
|
+
const text = String(record.severityText ?? event.level ?? "").toLowerCase();
|
|
52
|
+
const severity = number >= 21 || text === "fatal"
|
|
53
|
+
? "fatal"
|
|
54
|
+
: number >= 17 || text === "error"
|
|
55
|
+
? "error"
|
|
56
|
+
: number >= 13 || /warn/.test(text)
|
|
57
|
+
? "warning"
|
|
58
|
+
: number < 9 || text === "debug"
|
|
59
|
+
? "debug"
|
|
60
|
+
: "info";
|
|
61
|
+
const errorMessage = event.error && typeof event.error === "object"
|
|
62
|
+
? event.error.message
|
|
63
|
+
: event.error;
|
|
64
|
+
const message = String(sanitizeRuntimeContext({
|
|
65
|
+
message: event.message ?? errorMessage ?? body,
|
|
66
|
+
}).message ?? "").slice(0, 4000);
|
|
67
|
+
const operationId = String(attributes["autter.operation.id"] ?? "").slice(0, 128);
|
|
68
|
+
const traceId = /^[a-f0-9]{32}$/i.test(record.traceId ?? "")
|
|
69
|
+
? record.traceId.toLowerCase().replace(/^0+$/, "")
|
|
70
|
+
: "";
|
|
71
|
+
const spanId = /^[a-f0-9]{16}$/i.test(record.spanId ?? "")
|
|
72
|
+
? record.spanId.toLowerCase().replace(/^0+$/, "")
|
|
73
|
+
: "";
|
|
74
|
+
if (typeof attributes.path === "string")
|
|
75
|
+
attributes.path = normalizeRoute(attributes.path.split(/[?#]/)[0]);
|
|
76
|
+
rows.push({
|
|
77
|
+
id: createHash("sha256")
|
|
78
|
+
.update(JSON.stringify([
|
|
79
|
+
info,
|
|
80
|
+
nanos,
|
|
81
|
+
traceId,
|
|
82
|
+
spanId,
|
|
83
|
+
severity,
|
|
84
|
+
attributes,
|
|
85
|
+
message,
|
|
86
|
+
]))
|
|
87
|
+
.digest("hex")
|
|
88
|
+
.slice(0, 32),
|
|
89
|
+
service: String(info["service.name"] ?? "unknown").slice(0, 200),
|
|
90
|
+
environment: String(info["deployment.environment.name"] ??
|
|
91
|
+
info["deployment.environment"] ??
|
|
92
|
+
"production").slice(0, 100),
|
|
93
|
+
release: String(info["service.version"] ?? "").slice(0, 200),
|
|
94
|
+
traceId,
|
|
95
|
+
spanId,
|
|
96
|
+
operationId,
|
|
97
|
+
operation: String(attributes["autter.operation.name"] ?? "").slice(0, 200),
|
|
98
|
+
type: attributes["autter.event.type"] === "operation"
|
|
99
|
+
? "operation"
|
|
100
|
+
: "log",
|
|
101
|
+
severity,
|
|
102
|
+
message,
|
|
103
|
+
outcome: String(attributes["autter.operation.outcome"] ?? "").slice(0, 30),
|
|
104
|
+
durationMs: Math.max(0, Number(attributes["autter.operation.duration_ms"]) || 0),
|
|
105
|
+
attributes,
|
|
106
|
+
occurredAt,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return rows;
|
|
112
|
+
}
|
|
113
|
+
export function logTableDDL(db) {
|
|
114
|
+
return `CREATE TABLE IF NOT EXISTS ${db}.runtime_logs (
|
|
115
|
+
org_id String, repository_id String, event_id String,
|
|
116
|
+
service LowCardinality(String), environment LowCardinality(String), release String DEFAULT '',
|
|
117
|
+
trace_id String DEFAULT '', span_id String DEFAULT '', operation_id String DEFAULT '',
|
|
118
|
+
operation String DEFAULT '', event_type LowCardinality(String) DEFAULT 'log',
|
|
119
|
+
severity LowCardinality(String) DEFAULT 'info', message String CODEC(ZSTD(1)),
|
|
120
|
+
outcome LowCardinality(String) DEFAULT '', duration_ms Float64 DEFAULT 0,
|
|
121
|
+
attributes String DEFAULT '{}' CODEC(ZSTD(1)), occurred_at DateTime64(3, 'UTC'),
|
|
122
|
+
ingested_at DateTime64(3, 'UTC') DEFAULT now64(3)
|
|
123
|
+
) ENGINE = ReplacingMergeTree PARTITION BY toDate(occurred_at)
|
|
124
|
+
ORDER BY (org_id, repository_id, occurred_at, event_id)
|
|
125
|
+
TTL toDateTime(occurred_at) + INTERVAL 14 DAY`;
|
|
126
|
+
}
|
package/dist/migrations.js
CHANGED
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
import { latencyTableDDL } from "./latency.js";
|
|
36
36
|
import { profileTableDDL } from "./profiles.js";
|
|
37
37
|
import { sourceMapTableDDL } from "./source-maps.js";
|
|
38
|
+
import { logTableDDL } from "./logs.js";
|
|
38
39
|
import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
|
|
39
40
|
export const MIGRATIONS = [
|
|
40
41
|
// 0001 intentionally reserved as the baseline marker: databases created
|
|
@@ -127,6 +128,7 @@ export const MIGRATIONS = [
|
|
|
127
128
|
{ id: "0010-memory-temporality", statements: [
|
|
128
129
|
`ALTER TABLE {db}.runtime_memory_samples ADD COLUMN IF NOT EXISTS temporality LowCardinality(String) DEFAULT 'gauge'`,
|
|
129
130
|
] },
|
|
131
|
+
{ id: "0011-runtime-logs", statements: [logTableDDL("{db}")] },
|
|
130
132
|
];
|
|
131
133
|
/** The tracking table itself — created by the runner before anything else. */
|
|
132
134
|
export function migrationsTableDDL(db) {
|
package/dist/normalize-otlp.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type OtlpValue } from "./context.js";
|
|
1
2
|
import { type RuntimeLlmCall, type RuntimeMetricPoint, type RuntimeOccurrenceInput, type RuntimeSpanRow } from "./types.js";
|
|
2
3
|
/**
|
|
3
4
|
* OTLP/HTTP JSON → runtime signal. Structural types cover only the fields
|
|
@@ -10,12 +11,7 @@ import { type RuntimeLlmCall, type RuntimeMetricPoint, type RuntimeOccurrenceInp
|
|
|
10
11
|
*/
|
|
11
12
|
interface OtlpKeyValue {
|
|
12
13
|
key?: string;
|
|
13
|
-
value?:
|
|
14
|
-
stringValue?: string;
|
|
15
|
-
intValue?: string | number;
|
|
16
|
-
doubleValue?: number;
|
|
17
|
-
boolValue?: boolean;
|
|
18
|
-
};
|
|
14
|
+
value?: OtlpValue;
|
|
19
15
|
}
|
|
20
16
|
interface OtlpEvent {
|
|
21
17
|
name?: string;
|
package/dist/normalize-otlp.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { decodeOtlpAttributes, sanitizeRuntimeContext } from "./context.js";
|
|
1
2
|
import { normalizeRoute } from "./fingerprint.js";
|
|
2
3
|
import { extractLlmCall } from "./llm.js";
|
|
3
4
|
import { asSeverity, } from "./types.js";
|
|
@@ -133,6 +134,7 @@ export function normalizeTraces(request) {
|
|
|
133
134
|
statusCode,
|
|
134
135
|
durationMs,
|
|
135
136
|
attributes: {
|
|
137
|
+
...decodeOtlpAttributes(span.attributes),
|
|
136
138
|
"http.request.method": (attrs.get("http.request.method") ?? attrs.get("http.method") ?? "").slice(0, 20),
|
|
137
139
|
},
|
|
138
140
|
startedAt,
|
|
@@ -162,10 +164,8 @@ export function normalizeTraces(request) {
|
|
|
162
164
|
statusCode,
|
|
163
165
|
traceId: span.traceId ?? null,
|
|
164
166
|
sessionId: null,
|
|
165
|
-
attributes:
|
|
166
|
-
"autter.handled": true,
|
|
167
|
-
"autter.sampled": eventAttrs.get("autter.sampled") === "true",
|
|
168
|
-
} : null,
|
|
167
|
+
attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes),
|
|
168
|
+
...(handled ? { "autter.handled": true, "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) }),
|
|
169
169
|
occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
|
|
170
170
|
});
|
|
171
171
|
}
|
|
@@ -183,8 +183,9 @@ export function normalizeTraces(request) {
|
|
|
183
183
|
environment: resource.environment, release: resource.release,
|
|
184
184
|
errorType: "OutcomeFailure",
|
|
185
185
|
message: `${name}: ${(outcome.get("autter.outcome.message") ?? "failed").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 1000)}`,
|
|
186
|
-
stack: null, route, method: methodOf(attrs), statusCode,
|
|
187
|
-
traceId: span.traceId ?? null, sessionId: null,
|
|
186
|
+
stack: String(decodeOtlpAttributes(event.attributes)["autter.outcome.stack"] ?? "") || null, route, method: methodOf(attrs), statusCode,
|
|
187
|
+
traceId: span.traceId ?? null, sessionId: null,
|
|
188
|
+
attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }),
|
|
188
189
|
occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
|
|
189
190
|
});
|
|
190
191
|
}
|
|
@@ -206,7 +207,7 @@ export function normalizeTraces(request) {
|
|
|
206
207
|
statusCode,
|
|
207
208
|
traceId: span.traceId ?? null,
|
|
208
209
|
sessionId: null,
|
|
209
|
-
attributes:
|
|
210
|
+
attributes: decodeOtlpAttributes(span.attributes),
|
|
210
211
|
occurredAt: startedAt,
|
|
211
212
|
});
|
|
212
213
|
}
|
package/dist/otlp-proto.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
import type { OtlpMetricsRequest, OtlpTraceRequest } from "./normalize-otlp.js";
|
|
2
|
+
import type { OtlpLogsRequest } from "./logs.js";
|
|
2
3
|
export declare function decodeTraceRequest(body: Buffer): OtlpTraceRequest;
|
|
3
4
|
export declare function decodeMetricsRequest(body: Buffer): OtlpMetricsRequest;
|
|
5
|
+
export declare function decodeLogsRequest(body: Buffer): OtlpLogsRequest;
|
package/dist/otlp-proto.js
CHANGED
|
@@ -21,10 +21,27 @@ message AnyValue {
|
|
|
21
21
|
bool bool_value = 2;
|
|
22
22
|
int64 int_value = 3;
|
|
23
23
|
double double_value = 4;
|
|
24
|
+
ArrayValue array_value = 5;
|
|
25
|
+
KeyValueList kvlist_value = 6;
|
|
24
26
|
}
|
|
25
27
|
}
|
|
28
|
+
message ArrayValue { repeated AnyValue values = 1; }
|
|
29
|
+
message KeyValueList { repeated KeyValue values = 1; }
|
|
26
30
|
message KeyValue { string key = 1; AnyValue value = 2; }
|
|
27
31
|
message Resource { repeated KeyValue attributes = 1; }
|
|
32
|
+
message ExportLogsServiceRequest { repeated ResourceLogs resource_logs = 1; }
|
|
33
|
+
message ResourceLogs { Resource resource = 1; repeated ScopeLogs scope_logs = 2; }
|
|
34
|
+
message ScopeLogs { repeated LogRecord log_records = 2; }
|
|
35
|
+
message LogRecord {
|
|
36
|
+
fixed64 time_unix_nano = 1;
|
|
37
|
+
int32 severity_number = 2;
|
|
38
|
+
string severity_text = 3;
|
|
39
|
+
AnyValue body = 5;
|
|
40
|
+
repeated KeyValue attributes = 6;
|
|
41
|
+
bytes trace_id = 9;
|
|
42
|
+
bytes span_id = 10;
|
|
43
|
+
fixed64 observed_time_unix_nano = 11;
|
|
44
|
+
}
|
|
28
45
|
|
|
29
46
|
message ExportTraceServiceRequest { repeated ResourceSpans resource_spans = 1; }
|
|
30
47
|
message ResourceSpans { Resource resource = 1; repeated ScopeSpans scope_spans = 2; }
|
|
@@ -80,6 +97,7 @@ message HistogramDataPoint {
|
|
|
80
97
|
const root = protobuf.parse(PROTO).root;
|
|
81
98
|
const TraceRequest = root.lookupType("otlp.ExportTraceServiceRequest");
|
|
82
99
|
const MetricsRequest = root.lookupType("otlp.ExportMetricsServiceRequest");
|
|
100
|
+
const LogsRequest = root.lookupType("otlp.ExportLogsServiceRequest");
|
|
83
101
|
const TO_OBJECT_OPTIONS = {
|
|
84
102
|
longs: String, // 64-bit ints → strings (matches OTLP/JSON)
|
|
85
103
|
defaults: false,
|
|
@@ -110,3 +128,6 @@ export function decodeMetricsRequest(body) {
|
|
|
110
128
|
const message = MetricsRequest.decode(body);
|
|
111
129
|
return hexifyIds(MetricsRequest.toObject(message, TO_OBJECT_OPTIONS));
|
|
112
130
|
}
|
|
131
|
+
export function decodeLogsRequest(body) {
|
|
132
|
+
return hexifyIds(LogsRequest.toObject(LogsRequest.decode(body), TO_OBJECT_OPTIONS));
|
|
133
|
+
}
|
package/dist/server.js
CHANGED
|
@@ -5,7 +5,8 @@ import { normalizeLatencyHistograms } from "./latency.js";
|
|
|
5
5
|
import { deriveFields, fingerprintOccurrence, occurrenceIdFor, } from "./fingerprint.js";
|
|
6
6
|
import { browserPayloadSchema, normalizeBrowserPayload, } from "./normalize-browser.js";
|
|
7
7
|
import { normalizeMetrics, normalizeTraces, } from "./normalize-otlp.js";
|
|
8
|
-
import { decodeMetricsRequest, decodeTraceRequest } from "./otlp-proto.js";
|
|
8
|
+
import { decodeMetricsRequest, decodeTraceRequest, decodeLogsRequest } from "./otlp-proto.js";
|
|
9
|
+
import { normalizeLogs } from "./logs.js";
|
|
9
10
|
import { decodeProfile } from "./profiles.js";
|
|
10
11
|
import { normalizeMemoryMetrics, normalizePlatformEvent, platformEventSchema } from "./memory.js";
|
|
11
12
|
import { validateSourceMap } from "./source-maps.js";
|
|
@@ -262,6 +263,29 @@ export function createIngesterApp(config) {
|
|
|
262
263
|
sink?.enqueue(ctx, fingerprinted, metricPoints, llmCalls);
|
|
263
264
|
otlpSuccess(req, res);
|
|
264
265
|
});
|
|
266
|
+
app.post("/v1/logs", async (req, res) => {
|
|
267
|
+
const ctx = await authenticate(req, res, "otlp");
|
|
268
|
+
if (!ctx)
|
|
269
|
+
return;
|
|
270
|
+
let logs;
|
|
271
|
+
try {
|
|
272
|
+
logs = normalizeLogs(req.is("application/x-protobuf") ? decodeLogsRequest(req.body) : req.body);
|
|
273
|
+
}
|
|
274
|
+
catch {
|
|
275
|
+
res.status(400).json({ error: "invalid OTLP logs payload" });
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
278
|
+
try {
|
|
279
|
+
await store.insertLogs(ctx, logs);
|
|
280
|
+
}
|
|
281
|
+
catch (err) {
|
|
282
|
+
storageError(res, err);
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
// Logs are diagnostic evidence. Exceptions and failed outcomes use the trace sink,
|
|
286
|
+
// so one operation does not create duplicate issues through two export paths.
|
|
287
|
+
otlpSuccess(req, res);
|
|
288
|
+
});
|
|
265
289
|
app.post("/v1/metrics", async (req, res) => {
|
|
266
290
|
const ctx = await authenticate(req, res, "otlp");
|
|
267
291
|
if (!ctx)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@autter/otlp-ingester",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.0",
|
|
4
4
|
"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|