@autter/otlp-ingester 1.3.4 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -0
- package/dist/clickhouse.d.ts +35 -0
- package/dist/clickhouse.js +156 -2
- package/dist/compat-manifest.json +94 -0
- package/dist/compat.d.ts +147 -0
- package/dist/compat.js +219 -0
- package/dist/config.d.ts +17 -0
- package/dist/config.js +18 -0
- package/dist/context.d.ts +19 -0
- package/dist/context.js +102 -0
- package/dist/error-fields.d.ts +41 -0
- package/dist/error-fields.js +103 -0
- package/dist/fingerprint.d.ts +29 -0
- package/dist/fingerprint.js +45 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +5 -3
- package/dist/llm.js +16 -3
- package/dist/log-promotion.d.ts +55 -0
- package/dist/log-promotion.js +119 -0
- package/dist/logs.d.ts +98 -0
- package/dist/logs.js +261 -0
- package/dist/migrations.d.ts +3 -0
- package/dist/migrations.js +44 -0
- package/dist/normalize-browser.d.ts +16 -4
- package/dist/normalize-browser.js +22 -18
- package/dist/normalize-otlp.d.ts +2 -6
- package/dist/normalize-otlp.js +22 -9
- package/dist/otlp-proto.d.ts +2 -0
- package/dist/otlp-proto.js +21 -0
- package/dist/redact.d.ts +33 -0
- package/dist/redact.js +96 -0
- package/dist/sdk-versions.d.ts +58 -0
- package/dist/sdk-versions.js +167 -0
- package/dist/server.d.ts +3 -0
- package/dist/server.js +171 -5
- package/dist/sink.d.ts +9 -0
- package/dist/sink.js +26 -4
- package/dist/types.d.ts +18 -0
- package/package.json +1 -1
package/dist/compat.js
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Version compatibility between Autter Runtime SDKs, this ingester and its
|
|
3
|
+
* ClickHouse schema — the single source of truth.
|
|
4
|
+
*
|
|
5
|
+
* `compat-manifest.json` lists every feature that needs a minimum ingester
|
|
6
|
+
* version, a ClickHouse migration, a dedicated ingest route, or a minimum SDK
|
|
7
|
+
* version. Everything else derives from it:
|
|
8
|
+
*
|
|
9
|
+
* - the ingester serves it (evaluated against its own version and applied
|
|
10
|
+
* migrations) at `GET /v1/compat`;
|
|
11
|
+
* - `@autter/runtime-node` / `@autter/runtime-next` bundle this file at build
|
|
12
|
+
* time to warn once when an enabled feature needs a newer ingester, and to
|
|
13
|
+
* power `npx @autter/runtime-node doctor`;
|
|
14
|
+
* - the Python adapter (`adapters/python/compat.py`) embeds a copy that a
|
|
15
|
+
* parity test keeps identical.
|
|
16
|
+
*
|
|
17
|
+
* This module is dependency-free and side-effect-free on purpose: it is
|
|
18
|
+
* bundled into the Node SDK. Keep it that way.
|
|
19
|
+
*
|
|
20
|
+
* Adding a feature: append an entry to compat-manifest.json with the first
|
|
21
|
+
* ingester release that handles it, the migrations it writes to, its route
|
|
22
|
+
* (if it has its own), and the first SDK release that emits it. The
|
|
23
|
+
* consistency test in compat.test.ts checks ids, migrations and routes.
|
|
24
|
+
*/
|
|
25
|
+
import manifestJson from "./compat-manifest.json" with { type: "json" };
|
|
26
|
+
export const COMPAT_MANIFEST = manifestJson;
|
|
27
|
+
export const INGESTER_PACKAGE = "@autter/otlp-ingester";
|
|
28
|
+
/** Response header carrying the ingester version on every ingest response. */
|
|
29
|
+
export const INGESTER_VERSION_HEADER = "x-autter-ingester-version";
|
|
30
|
+
// ── versions ────────────────────────────────────────────────────────────
|
|
31
|
+
/** [major, minor, patch, isPrerelease] or null when unparsable. */
|
|
32
|
+
function parseVersion(version) {
|
|
33
|
+
if (typeof version !== "string")
|
|
34
|
+
return null;
|
|
35
|
+
const match = /^v?(\d{1,6})\.(\d{1,6})\.(\d{1,6})(-[0-9A-Za-z.-]+)?/.exec(version.trim());
|
|
36
|
+
if (!match)
|
|
37
|
+
return null;
|
|
38
|
+
return [
|
|
39
|
+
Number(match[1]),
|
|
40
|
+
Number(match[2]),
|
|
41
|
+
Number(match[3]),
|
|
42
|
+
// A prerelease sorts before its release: 1.4.0-rc.1 < 1.4.0.
|
|
43
|
+
match[4] ? 0 : 1,
|
|
44
|
+
];
|
|
45
|
+
}
|
|
46
|
+
/** Semver-ish comparison; null when either side is unparsable. */
|
|
47
|
+
export function compareVersions(a, b) {
|
|
48
|
+
const left = parseVersion(a);
|
|
49
|
+
const right = parseVersion(b);
|
|
50
|
+
if (!left || !right)
|
|
51
|
+
return null;
|
|
52
|
+
for (let i = 0; i < 4; i++) {
|
|
53
|
+
if (left[i] !== right[i])
|
|
54
|
+
return left[i] < right[i] ? -1 : 1;
|
|
55
|
+
}
|
|
56
|
+
return 0;
|
|
57
|
+
}
|
|
58
|
+
/** True/false when comparable, null when the version is unknown/unparsable. */
|
|
59
|
+
export function versionAtLeast(version, minimum) {
|
|
60
|
+
if (!version)
|
|
61
|
+
return null;
|
|
62
|
+
const result = compareVersions(version, minimum);
|
|
63
|
+
return result === null ? null : result >= 0;
|
|
64
|
+
}
|
|
65
|
+
export function featureById(id, manifest = COMPAT_MANIFEST) {
|
|
66
|
+
return manifest.features.find((feature) => feature.id === id);
|
|
67
|
+
}
|
|
68
|
+
/** Features the given browser payload event types belong to. */
|
|
69
|
+
export function featuresForBrowserEvents(types, manifest = COMPAT_MANIFEST) {
|
|
70
|
+
const wanted = new Set(types);
|
|
71
|
+
return manifest.features
|
|
72
|
+
.filter((feature) => feature.browserEvents?.some((type) => wanted.has(type)))
|
|
73
|
+
.map((feature) => feature.id);
|
|
74
|
+
}
|
|
75
|
+
export function ingesterUpgradeHint(manifest = COMPAT_MANIFEST) {
|
|
76
|
+
return `Upgrade the ingester: ${manifest.upgrade.ingester}. See ${manifest.docs}`;
|
|
77
|
+
}
|
|
78
|
+
function sdkUpgradeHint(pkg, manifest) {
|
|
79
|
+
return `Upgrade the SDK: ${manifest.upgrade.sdk.replace("{package}", pkg)}. See ${manifest.docs}`;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Every incompatibility for the features in use. Unknown is never an issue:
|
|
83
|
+
* when the ingester version can't be determined (and a legacy route probe
|
|
84
|
+
* didn't prove a route missing) nothing is reported, so the check has no
|
|
85
|
+
* false positives.
|
|
86
|
+
*/
|
|
87
|
+
export function evaluateCompat(input) {
|
|
88
|
+
const manifest = input.manifest ?? COMPAT_MANIFEST;
|
|
89
|
+
const issues = [];
|
|
90
|
+
const seen = new Set();
|
|
91
|
+
for (const id of input.features) {
|
|
92
|
+
if (seen.has(id))
|
|
93
|
+
continue;
|
|
94
|
+
seen.add(id);
|
|
95
|
+
const feature = featureById(id, manifest);
|
|
96
|
+
if (!feature)
|
|
97
|
+
continue;
|
|
98
|
+
const sdkMin = input.sdk ? feature.sdks[input.sdk.name] : undefined;
|
|
99
|
+
if (input.sdk && sdkMin && versionAtLeast(input.sdk.version, sdkMin) === false) {
|
|
100
|
+
const fix = sdkUpgradeHint(input.sdk.name, manifest);
|
|
101
|
+
issues.push({
|
|
102
|
+
feature: id,
|
|
103
|
+
title: feature.title,
|
|
104
|
+
kind: "sdk_too_old",
|
|
105
|
+
component: input.sdk.name,
|
|
106
|
+
required: sdkMin,
|
|
107
|
+
actual: input.sdk.version,
|
|
108
|
+
message: `${feature.title} needs ${input.sdk.name} >= ${sdkMin}; yours is ${input.sdk.version}. ${fix}`,
|
|
109
|
+
fix,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
const ingester = input.ingester;
|
|
113
|
+
if (!ingester)
|
|
114
|
+
continue;
|
|
115
|
+
const fix = ingesterUpgradeHint(manifest);
|
|
116
|
+
const atLeast = versionAtLeast(ingester.version, feature.ingester);
|
|
117
|
+
if (atLeast === false) {
|
|
118
|
+
issues.push({
|
|
119
|
+
feature: id,
|
|
120
|
+
title: feature.title,
|
|
121
|
+
kind: "ingester_too_old",
|
|
122
|
+
component: INGESTER_PACKAGE,
|
|
123
|
+
required: feature.ingester,
|
|
124
|
+
actual: ingester.version,
|
|
125
|
+
message: `${feature.title} needs ingester >= ${feature.ingester}; yours is ${ingester.version}. ${fix}`,
|
|
126
|
+
fix,
|
|
127
|
+
});
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
if (ingester.version === null &&
|
|
131
|
+
feature.route &&
|
|
132
|
+
ingester.routes?.[feature.route] === false) {
|
|
133
|
+
const actual = "1.4.0 or older (no /v1/compat)";
|
|
134
|
+
issues.push({
|
|
135
|
+
feature: id,
|
|
136
|
+
title: feature.title,
|
|
137
|
+
kind: "ingester_too_old",
|
|
138
|
+
component: INGESTER_PACKAGE,
|
|
139
|
+
required: feature.ingester,
|
|
140
|
+
actual,
|
|
141
|
+
message: `${feature.title} needs ingester >= ${feature.ingester}; yours is ${actual} and has no ${feature.route} route, so this data is dropped. ${fix}`,
|
|
142
|
+
fix,
|
|
143
|
+
});
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const schema = ingester.schema;
|
|
147
|
+
if (atLeast && schema && feature.migrations.length) {
|
|
148
|
+
const applied = new Set(schema.applied);
|
|
149
|
+
const missing = feature.migrations.filter((m) => !applied.has(m));
|
|
150
|
+
if (schema.status === "failed" || (schema.status === "ready" && missing.length)) {
|
|
151
|
+
const schemaFix = "Check the ingester's CLICKHOUSE_URL/credentials and its logs, then restart it so the boot migrations run. " +
|
|
152
|
+
`See ${manifest.docs}`;
|
|
153
|
+
const what = missing.length ? missing.join(", ") : feature.migrations.join(", ");
|
|
154
|
+
issues.push({
|
|
155
|
+
feature: id,
|
|
156
|
+
title: feature.title,
|
|
157
|
+
kind: "schema_not_applied",
|
|
158
|
+
component: INGESTER_PACKAGE,
|
|
159
|
+
required: feature.migrations.join(", "),
|
|
160
|
+
actual: schema.status === "failed" ? "migrations failed" : `missing ${what}`,
|
|
161
|
+
message: `${feature.title} needs ClickHouse migration ${what}, which ingester ${ingester.version} has not applied (schema ${schema.status}). ${schemaFix}`,
|
|
162
|
+
fix: schemaFix,
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return issues;
|
|
168
|
+
}
|
|
169
|
+
export function buildCompatReport(input) {
|
|
170
|
+
const manifest = input.manifest ?? COMPAT_MANIFEST;
|
|
171
|
+
const applied = new Set(input.applied);
|
|
172
|
+
const appliedOrdered = input.allMigrations.filter((id) => applied.has(id));
|
|
173
|
+
return {
|
|
174
|
+
ingester: { name: INGESTER_PACKAGE, version: input.version },
|
|
175
|
+
schema: {
|
|
176
|
+
status: input.schemaStatus,
|
|
177
|
+
level: appliedOrdered[appliedOrdered.length - 1] ?? "",
|
|
178
|
+
latest: input.allMigrations[input.allMigrations.length - 1] ?? "",
|
|
179
|
+
applied: appliedOrdered,
|
|
180
|
+
},
|
|
181
|
+
features: manifest.features.map((feature) => {
|
|
182
|
+
const ingesterSupported = versionAtLeast(input.version, feature.ingester) !== false;
|
|
183
|
+
const missingMigrations = feature.migrations.filter((m) => !applied.has(m));
|
|
184
|
+
return {
|
|
185
|
+
...feature,
|
|
186
|
+
ingesterSupported,
|
|
187
|
+
missingMigrations,
|
|
188
|
+
available: ingesterSupported &&
|
|
189
|
+
(feature.migrations.length === 0 ||
|
|
190
|
+
(input.schemaStatus === "ready" && missingMigrations.length === 0)),
|
|
191
|
+
};
|
|
192
|
+
}),
|
|
193
|
+
manifestVersion: manifest.manifestVersion,
|
|
194
|
+
docs: manifest.docs,
|
|
195
|
+
upgrade: manifest.upgrade,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
/** IngesterCompatInfo from a /v1/compat response body (defensive: it's network input). */
|
|
199
|
+
export function ingesterInfoFromReport(body) {
|
|
200
|
+
if (!body || typeof body !== "object")
|
|
201
|
+
return null;
|
|
202
|
+
const report = body;
|
|
203
|
+
const version = report.ingester?.version;
|
|
204
|
+
if (typeof version !== "string" || !parseVersion(version))
|
|
205
|
+
return null;
|
|
206
|
+
const schema = report.schema;
|
|
207
|
+
const statuses = ["ready", "pending", "failed", "unconfigured"];
|
|
208
|
+
return {
|
|
209
|
+
version,
|
|
210
|
+
...(schema && statuses.includes(schema.status) && Array.isArray(schema.applied)
|
|
211
|
+
? {
|
|
212
|
+
schema: {
|
|
213
|
+
status: schema.status,
|
|
214
|
+
applied: schema.applied.filter((id) => typeof id === "string"),
|
|
215
|
+
},
|
|
216
|
+
}
|
|
217
|
+
: {}),
|
|
218
|
+
};
|
|
219
|
+
}
|
package/dist/config.d.ts
CHANGED
|
@@ -32,10 +32,27 @@ export interface IngesterConfig {
|
|
|
32
32
|
rateLimitPerMinute: number;
|
|
33
33
|
/** Per-key requests per minute for publishable client keys. */
|
|
34
34
|
clientRateLimitPerMinute: number;
|
|
35
|
+
/** ClickHouse dedupe lookups per tenant per minute for /v1/logs promotion. */
|
|
36
|
+
promotionLookupsPerMinute?: number;
|
|
35
37
|
/** Retention, overridable per deployment. */
|
|
36
38
|
occurrenceTtlDays: number;
|
|
37
39
|
spanTtlDays: number;
|
|
38
40
|
metricsTtlDays: number;
|
|
39
41
|
llmCallTtlDays: number;
|
|
42
|
+
/**
|
|
43
|
+
* runtime_logs retention (LOG_TTL_DAYS, default 14). Request summaries are
|
|
44
|
+
* always kept (never sampled), so this is the main volume knob. Applied to
|
|
45
|
+
* fresh tables by the baseline and to existing ones at boot.
|
|
46
|
+
*/
|
|
47
|
+
logTtlDays: number;
|
|
48
|
+
/**
|
|
49
|
+
* Extra regex sources scrubbed from stored/forwarded free text and
|
|
50
|
+
* attribute values, and extra regex sources for sensitive attribute
|
|
51
|
+
* keys — on top of the built-in secret/PII patterns (see redact.ts).
|
|
52
|
+
* Env: AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS (JSON
|
|
53
|
+
* arrays of strings).
|
|
54
|
+
*/
|
|
55
|
+
redactValuePatterns?: string[];
|
|
56
|
+
redactKeyPatterns?: string[];
|
|
40
57
|
}
|
|
41
58
|
export declare function loadConfig(): IngesterConfig;
|
package/dist/config.js
CHANGED
|
@@ -5,6 +5,20 @@ function intEnv(name, fallback) {
|
|
|
5
5
|
const value = Number.parseInt(raw, 10);
|
|
6
6
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
7
7
|
}
|
|
8
|
+
/** JSON array of regex sources; invalid input fails startup loudly rather
|
|
9
|
+
* than silently running without the operator's extra patterns. */
|
|
10
|
+
function patternListEnv(name) {
|
|
11
|
+
const raw = process.env[name];
|
|
12
|
+
if (!raw)
|
|
13
|
+
return [];
|
|
14
|
+
const parsed = JSON.parse(raw);
|
|
15
|
+
if (!Array.isArray(parsed) || !parsed.every((p) => typeof p === "string")) {
|
|
16
|
+
throw new Error(`${name} must be a JSON array of regex source strings`);
|
|
17
|
+
}
|
|
18
|
+
for (const source of parsed)
|
|
19
|
+
new RegExp(source);
|
|
20
|
+
return parsed;
|
|
21
|
+
}
|
|
8
22
|
function parseIngestKeys(raw) {
|
|
9
23
|
if (!raw)
|
|
10
24
|
return [];
|
|
@@ -42,12 +56,16 @@ export function loadConfig() {
|
|
|
42
56
|
maxBodyBytes: intEnv("MAX_BODY_BYTES", 1024 * 1024),
|
|
43
57
|
rateLimitPerMinute: intEnv("RATE_LIMIT_PER_MINUTE", 300),
|
|
44
58
|
clientRateLimitPerMinute: intEnv("CLIENT_RATE_LIMIT_PER_MINUTE", 120),
|
|
59
|
+
promotionLookupsPerMinute: intEnv("PROMOTION_LOOKUPS_PER_MINUTE", 30),
|
|
45
60
|
occurrenceTtlDays: intEnv("OCCURRENCE_TTL_DAYS", 14),
|
|
46
61
|
spanTtlDays: intEnv("SPAN_TTL_DAYS", 7),
|
|
47
62
|
metricsTtlDays: intEnv("METRICS_TTL_DAYS", 90),
|
|
48
63
|
// LLM calls keep the metrics horizon, not the span one — cost trends
|
|
49
64
|
// need months, and per-call volume is small next to HTTP spans.
|
|
50
65
|
llmCallTtlDays: intEnv("LLM_CALL_TTL_DAYS", 90),
|
|
66
|
+
logTtlDays: intEnv("LOG_TTL_DAYS", 14),
|
|
67
|
+
redactValuePatterns: patternListEnv("AUTTER_REDACT_VALUE_PATTERNS"),
|
|
68
|
+
redactKeyPatterns: patternListEnv("AUTTER_REDACT_KEY_PATTERNS"),
|
|
51
69
|
};
|
|
52
70
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(config.clickhouseDatabase)) {
|
|
53
71
|
throw new Error(`Invalid CLICKHOUSE_DATABASE name: ${config.clickhouseDatabase}`);
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */
|
|
2
|
+
export declare function sanitizeRuntimeContext(input: unknown): Record<string, unknown>;
|
|
3
|
+
export interface OtlpValue {
|
|
4
|
+
stringValue?: string;
|
|
5
|
+
intValue?: string | number;
|
|
6
|
+
doubleValue?: number;
|
|
7
|
+
boolValue?: boolean;
|
|
8
|
+
arrayValue?: {
|
|
9
|
+
values?: OtlpValue[];
|
|
10
|
+
};
|
|
11
|
+
kvlistValue?: {
|
|
12
|
+
values?: OtlpAttribute[];
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export interface OtlpAttribute {
|
|
16
|
+
key?: string;
|
|
17
|
+
value?: OtlpValue;
|
|
18
|
+
}
|
|
19
|
+
export declare function decodeOtlpAttributes(attributes: OtlpAttribute[] | undefined): Record<string, unknown>;
|
package/dist/context.js
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { isSensitiveKey, scrubText } from "./redact.js";
|
|
2
|
+
/** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */
|
|
3
|
+
export function sanitizeRuntimeContext(input) {
|
|
4
|
+
let budget = 512;
|
|
5
|
+
const seen = new WeakSet();
|
|
6
|
+
// Shared secret/PII patterns (redact.ts), then drop URL query strings
|
|
7
|
+
// and fragments outright — whitelist-style privacy for free text.
|
|
8
|
+
const scrub = (value) => scrubText(value)
|
|
9
|
+
// Query strings: procedural (a `[^\s?#]+[?#]` regex is quadratic on
|
|
10
|
+
// long runs of URLs without a query).
|
|
11
|
+
.replace(/https?:\/\/[^\s]*/gi, (url) => {
|
|
12
|
+
const rest = url.slice(url.indexOf("://") + 3);
|
|
13
|
+
const cut = rest.search(/[?#]/);
|
|
14
|
+
return cut <= 0 ? url : url.slice(0, url.length - rest.length + cut);
|
|
15
|
+
});
|
|
16
|
+
const visit = (value, key, depth) => {
|
|
17
|
+
if (--budget < 0 || depth > 6)
|
|
18
|
+
return "[truncated]";
|
|
19
|
+
if (/__proto__|constructor|prototype/i.test(key))
|
|
20
|
+
return undefined;
|
|
21
|
+
const usage =
|
|
22
|
+
// Numeric token COUNTS (incl. the AI rollup's cache_read_tokens) are
|
|
23
|
+
// usage, not secrets.
|
|
24
|
+
/(?:^|\.)(?:input|output|total|prompt|completion|cache_read|cache_creation)_?tokens$/i.test(key) &&
|
|
25
|
+
typeof value === "number" &&
|
|
26
|
+
Number.isFinite(value) &&
|
|
27
|
+
value >= 0;
|
|
28
|
+
if (!usage && key && isSensitiveKey(key))
|
|
29
|
+
return "[redacted]";
|
|
30
|
+
if (typeof value === "string") {
|
|
31
|
+
// Serialized custom context crosses the same privacy boundary as nested values.
|
|
32
|
+
if (/^\s*[\[{]/.test(value)) {
|
|
33
|
+
try {
|
|
34
|
+
return visit(JSON.parse(value), key, depth + 1);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
/* plain text */
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
const text = /(?:url|path|route|target)$/i.test(key)
|
|
41
|
+
? value.split(/[?#]/)[0]
|
|
42
|
+
: value;
|
|
43
|
+
// Cut before scrubbing so scrub work is bounded by the stored size;
|
|
44
|
+
// the slack keeps a secret straddling the cut whole for the scrubber.
|
|
45
|
+
const limit = /stack/i.test(key) ? 32000 : 2048;
|
|
46
|
+
return scrub(text.slice(0, limit + 512)).slice(0, limit);
|
|
47
|
+
}
|
|
48
|
+
if (typeof value === "number")
|
|
49
|
+
return Number.isFinite(value) ? value : undefined;
|
|
50
|
+
if (typeof value === "boolean" || value === null)
|
|
51
|
+
return value;
|
|
52
|
+
if (!value || typeof value !== "object")
|
|
53
|
+
return undefined;
|
|
54
|
+
if (seen.has(value))
|
|
55
|
+
return "[circular]";
|
|
56
|
+
seen.add(value);
|
|
57
|
+
if (Array.isArray(value))
|
|
58
|
+
return value.slice(0, 64).map((v) => visit(v, key, depth + 1));
|
|
59
|
+
const result = {};
|
|
60
|
+
for (const [k, v] of Object.entries(value).slice(0, 128)) {
|
|
61
|
+
const safe = visit(v, k, depth + 1);
|
|
62
|
+
if (safe !== undefined)
|
|
63
|
+
result[k.slice(0, 200)] = safe;
|
|
64
|
+
}
|
|
65
|
+
return result;
|
|
66
|
+
};
|
|
67
|
+
const result = visit(input, "", 0);
|
|
68
|
+
return result && typeof result === "object" && !Array.isArray(result)
|
|
69
|
+
? result
|
|
70
|
+
: {};
|
|
71
|
+
}
|
|
72
|
+
export function decodeOtlpAttributes(attributes) {
|
|
73
|
+
const decode = (v, depth = 0) => {
|
|
74
|
+
if (!v || depth > 6)
|
|
75
|
+
return undefined;
|
|
76
|
+
if (v.stringValue !== undefined)
|
|
77
|
+
return v.stringValue;
|
|
78
|
+
if (v.intValue !== undefined)
|
|
79
|
+
return Number(v.intValue);
|
|
80
|
+
if (v.doubleValue !== undefined)
|
|
81
|
+
return v.doubleValue;
|
|
82
|
+
if (v.boolValue !== undefined)
|
|
83
|
+
return v.boolValue;
|
|
84
|
+
if (v.arrayValue)
|
|
85
|
+
return (v.arrayValue.values ?? [])
|
|
86
|
+
.slice(0, 64)
|
|
87
|
+
.map((value) => decode(value, depth + 1));
|
|
88
|
+
if (v.kvlistValue)
|
|
89
|
+
return Object.fromEntries((v.kvlistValue.values ?? [])
|
|
90
|
+
.slice(0, 128)
|
|
91
|
+
.filter((a) => a.key && !/^(?:__proto__|constructor|prototype)$/.test(a.key))
|
|
92
|
+
.map((a) => [a.key, decode(a.value, depth + 1)]));
|
|
93
|
+
return undefined;
|
|
94
|
+
};
|
|
95
|
+
return sanitizeRuntimeContext(Object.fromEntries((attributes ?? [])
|
|
96
|
+
.slice()
|
|
97
|
+
.sort((a, b) => Number(/^autter\.(?:operation|event)\./.test(b.key ?? "")) -
|
|
98
|
+
Number(/^autter\.(?:operation|event)\./.test(a.key ?? "")))
|
|
99
|
+
.slice(0, 128)
|
|
100
|
+
.filter((a) => a.key && !/^(?:__proto__|constructor|prototype)$/.test(a.key))
|
|
101
|
+
.map((a) => [a.key, decode(a.value)])));
|
|
102
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Declared error metadata on the wire (span/event attributes, log
|
|
3
|
+
* attributes, browser context keys):
|
|
4
|
+
*
|
|
5
|
+
* autter.error.code string matching CODE_PATTERN (≤80) — grouping key
|
|
6
|
+
* autter.error.why declared cause (≤1000 chars)
|
|
7
|
+
* autter.error.fix declared remedy (≤1000 chars)
|
|
8
|
+
* autter.error.link docs link (http(s) only, ≤500 chars)
|
|
9
|
+
* autter.error.expected boolean — expected business failure
|
|
10
|
+
* autter.request.id request id joining the error to its summary
|
|
11
|
+
*
|
|
12
|
+
* Every field is optional and validated independently: a bad value is
|
|
13
|
+
* dropped on its own and never rejects the signal. Values are lifted onto
|
|
14
|
+
* dedicated occurrence fields (and columns) instead of living only in the
|
|
15
|
+
* attributes JSON, because the generic context sanitiser rewrites URLs
|
|
16
|
+
* (query AND fragment stripped) — the declared link keeps its #anchor here.
|
|
17
|
+
*/
|
|
18
|
+
export declare const REQUEST_ID_PATTERN: RegExp;
|
|
19
|
+
export interface DeclaredErrorFields {
|
|
20
|
+
errorCode?: string;
|
|
21
|
+
why?: string;
|
|
22
|
+
fix?: string;
|
|
23
|
+
link?: string;
|
|
24
|
+
expected?: boolean;
|
|
25
|
+
requestId?: string;
|
|
26
|
+
}
|
|
27
|
+
type AttributeSource = Map<string, string> | Record<string, unknown> | null | undefined;
|
|
28
|
+
/**
|
|
29
|
+
* http(s) only; credentials and query string removed (a docs link never
|
|
30
|
+
* needs them), fragment kept. Over-long links are dropped rather than
|
|
31
|
+
* truncated — a cut URL is a broken link.
|
|
32
|
+
*/
|
|
33
|
+
export declare function declaredLink(value: unknown): string | undefined;
|
|
34
|
+
export declare function validRequestId(value: unknown): string | undefined;
|
|
35
|
+
/**
|
|
36
|
+
* Lift the declared fields from one or more attribute sources; the first
|
|
37
|
+
* source carrying a key wins (e.g. exception event before its span).
|
|
38
|
+
* Returns only the fields that are present and valid.
|
|
39
|
+
*/
|
|
40
|
+
export declare function liftErrorFields(...sources: AttributeSource[]): DeclaredErrorFields;
|
|
41
|
+
export {};
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { sanitizeRuntimeContext } from "./context.js";
|
|
2
|
+
import { validErrorCode } from "./fingerprint.js";
|
|
3
|
+
/**
|
|
4
|
+
* Declared error metadata on the wire (span/event attributes, log
|
|
5
|
+
* attributes, browser context keys):
|
|
6
|
+
*
|
|
7
|
+
* autter.error.code string matching CODE_PATTERN (≤80) — grouping key
|
|
8
|
+
* autter.error.why declared cause (≤1000 chars)
|
|
9
|
+
* autter.error.fix declared remedy (≤1000 chars)
|
|
10
|
+
* autter.error.link docs link (http(s) only, ≤500 chars)
|
|
11
|
+
* autter.error.expected boolean — expected business failure
|
|
12
|
+
* autter.request.id request id joining the error to its summary
|
|
13
|
+
*
|
|
14
|
+
* Every field is optional and validated independently: a bad value is
|
|
15
|
+
* dropped on its own and never rejects the signal. Values are lifted onto
|
|
16
|
+
* dedicated occurrence fields (and columns) instead of living only in the
|
|
17
|
+
* attributes JSON, because the generic context sanitiser rewrites URLs
|
|
18
|
+
* (query AND fragment stripped) — the declared link keeps its #anchor here.
|
|
19
|
+
*/
|
|
20
|
+
export const REQUEST_ID_PATTERN = /^[\w.-]{8,128}$/;
|
|
21
|
+
const EMAIL_RE = /(?<![A-Z0-9._%+-])[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
|
|
22
|
+
function lookup(sources, key) {
|
|
23
|
+
for (const source of sources) {
|
|
24
|
+
if (!source)
|
|
25
|
+
continue;
|
|
26
|
+
const value = source instanceof Map ? source.get(key) : source[key];
|
|
27
|
+
if (value !== undefined && value !== null && value !== "")
|
|
28
|
+
return value;
|
|
29
|
+
}
|
|
30
|
+
return undefined;
|
|
31
|
+
}
|
|
32
|
+
/** First source whose value for `key` passes `valid` (an invalid value on
|
|
33
|
+
* the exception event must not hide a valid one on its span). */
|
|
34
|
+
function lookupValid(sources, key, valid) {
|
|
35
|
+
for (const source of sources) {
|
|
36
|
+
const value = valid(lookup([source], key));
|
|
37
|
+
if (value !== undefined)
|
|
38
|
+
return value;
|
|
39
|
+
}
|
|
40
|
+
return undefined;
|
|
41
|
+
}
|
|
42
|
+
/** Free text through the same scrubber as custom context, then capped. */
|
|
43
|
+
function declaredText(value, max) {
|
|
44
|
+
if (typeof value !== "string")
|
|
45
|
+
return undefined;
|
|
46
|
+
const text = sanitizeRuntimeContext({ text: value.trim() }).text;
|
|
47
|
+
return typeof text === "string" && text ? text.slice(0, max) : undefined;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* http(s) only; credentials and query string removed (a docs link never
|
|
51
|
+
* needs them), fragment kept. Over-long links are dropped rather than
|
|
52
|
+
* truncated — a cut URL is a broken link.
|
|
53
|
+
*/
|
|
54
|
+
export function declaredLink(value) {
|
|
55
|
+
if (typeof value !== "string")
|
|
56
|
+
return undefined;
|
|
57
|
+
let url;
|
|
58
|
+
try {
|
|
59
|
+
url = new URL(value.trim());
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
if (url.protocol !== "https:" && url.protocol !== "http:")
|
|
65
|
+
return undefined;
|
|
66
|
+
url.username = "";
|
|
67
|
+
url.password = "";
|
|
68
|
+
url.search = "";
|
|
69
|
+
const link = url.toString().replace(EMAIL_RE, "[redacted]");
|
|
70
|
+
return link.length <= 500 ? link : undefined;
|
|
71
|
+
}
|
|
72
|
+
export function validRequestId(value) {
|
|
73
|
+
return typeof value === "string" && REQUEST_ID_PATTERN.test(value)
|
|
74
|
+
? value
|
|
75
|
+
: undefined;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Lift the declared fields from one or more attribute sources; the first
|
|
79
|
+
* source carrying a key wins (e.g. exception event before its span).
|
|
80
|
+
* Returns only the fields that are present and valid.
|
|
81
|
+
*/
|
|
82
|
+
export function liftErrorFields(...sources) {
|
|
83
|
+
const out = {};
|
|
84
|
+
const code = lookupValid(sources, "autter.error.code", validErrorCode);
|
|
85
|
+
if (code)
|
|
86
|
+
out.errorCode = code;
|
|
87
|
+
const why = declaredText(lookup(sources, "autter.error.why"), 1000);
|
|
88
|
+
if (why)
|
|
89
|
+
out.why = why;
|
|
90
|
+
const fix = declaredText(lookup(sources, "autter.error.fix"), 1000);
|
|
91
|
+
if (fix)
|
|
92
|
+
out.fix = fix;
|
|
93
|
+
const link = declaredLink(lookup(sources, "autter.error.link"));
|
|
94
|
+
if (link)
|
|
95
|
+
out.link = link;
|
|
96
|
+
const expected = lookup(sources, "autter.error.expected");
|
|
97
|
+
if (expected === true || expected === "true")
|
|
98
|
+
out.expected = true;
|
|
99
|
+
const requestId = lookupValid(sources, "autter.request.id", validRequestId);
|
|
100
|
+
if (requestId)
|
|
101
|
+
out.requestId = requestId;
|
|
102
|
+
return out;
|
|
103
|
+
}
|
package/dist/fingerprint.d.ts
CHANGED
|
@@ -3,6 +3,35 @@ export declare function normalizeMessage(message: string): string;
|
|
|
3
3
|
/** Replace id-like path segments so /orders/812 and /orders/44 group. */
|
|
4
4
|
export declare function normalizeRoute(route: string | null): string;
|
|
5
5
|
export declare function normalizeStackFrames(stack: string | null, topN?: number): string[];
|
|
6
|
+
/**
|
|
7
|
+
* Error codes (`autter.error.code`) — namespaced, stable, low-cardinality
|
|
8
|
+
* identifiers such as "billing.declined". Anything else (Node's "ECONNRESET",
|
|
9
|
+
* ids, free text) is not a code and the error groups by message as before.
|
|
10
|
+
* Shared spec: runtime-core and the Autter backend validate identically.
|
|
11
|
+
*/
|
|
12
|
+
export declare const CODE_PATTERN: RegExp;
|
|
13
|
+
export declare const CODE_MAX_LENGTH = 80;
|
|
14
|
+
export declare function validErrorCode(value: unknown): string | undefined;
|
|
15
|
+
export type FingerprintScheme = "message-v1" | "code-v1";
|
|
16
|
+
/**
|
|
17
|
+
* Scheme "code-v1": a SOURCE-INDEPENDENT issue identity for coded errors.
|
|
18
|
+
* Source, error type, message, stack and route are deliberately left out so
|
|
19
|
+
* the same code from the trace path, log promotion, the browser or an
|
|
20
|
+
* external connector (Sentry, PostHog, …) lands in ONE issue. Route and
|
|
21
|
+
* message stay as facets on the occurrence rows. Mirrored byte-for-byte by
|
|
22
|
+
* the backend (`runtime-code-fingerprint.ts`) — shared test vectors pin it.
|
|
23
|
+
*/
|
|
24
|
+
export declare function codeFingerprint(service: string, code: string): string;
|
|
25
|
+
/**
|
|
26
|
+
* Issue identity plus the scheme that produced it. A valid
|
|
27
|
+
* `autter.error.code` selects "code-v1"; everything else keeps the
|
|
28
|
+
* historical message/stack/route fingerprint ("message-v1") byte-for-byte,
|
|
29
|
+
* so existing issues never regroup.
|
|
30
|
+
*/
|
|
31
|
+
export declare function occurrenceFingerprint(input: RuntimeOccurrenceInput): {
|
|
32
|
+
fingerprint: string;
|
|
33
|
+
scheme: FingerprintScheme;
|
|
34
|
+
};
|
|
6
35
|
export declare function fingerprintOccurrence(input: RuntimeOccurrenceInput): string;
|
|
7
36
|
/**
|
|
8
37
|
* Deterministic per-occurrence identity (as opposed to the fingerprint,
|
package/dist/fingerprint.js
CHANGED
|
@@ -233,7 +233,52 @@ export function normalizeStackFrames(stack, topN = 5) {
|
|
|
233
233
|
}
|
|
234
234
|
return frames.length > 0 ? frames : fallbackFrames(lines, topN);
|
|
235
235
|
}
|
|
236
|
+
/**
|
|
237
|
+
* Error codes (`autter.error.code`) — namespaced, stable, low-cardinality
|
|
238
|
+
* identifiers such as "billing.declined". Anything else (Node's "ECONNRESET",
|
|
239
|
+
* ids, free text) is not a code and the error groups by message as before.
|
|
240
|
+
* Shared spec: runtime-core and the Autter backend validate identically.
|
|
241
|
+
*/
|
|
242
|
+
export const CODE_PATTERN = /^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$/;
|
|
243
|
+
export const CODE_MAX_LENGTH = 80;
|
|
244
|
+
export function validErrorCode(value) {
|
|
245
|
+
return typeof value === "string" &&
|
|
246
|
+
value.length <= CODE_MAX_LENGTH &&
|
|
247
|
+
CODE_PATTERN.test(value)
|
|
248
|
+
? value
|
|
249
|
+
: undefined;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Scheme "code-v1": a SOURCE-INDEPENDENT issue identity for coded errors.
|
|
253
|
+
* Source, error type, message, stack and route are deliberately left out so
|
|
254
|
+
* the same code from the trace path, log promotion, the browser or an
|
|
255
|
+
* external connector (Sentry, PostHog, …) lands in ONE issue. Route and
|
|
256
|
+
* message stay as facets on the occurrence rows. Mirrored byte-for-byte by
|
|
257
|
+
* the backend (`runtime-code-fingerprint.ts`) — shared test vectors pin it.
|
|
258
|
+
*/
|
|
259
|
+
export function codeFingerprint(service, code) {
|
|
260
|
+
return createHash("sha256")
|
|
261
|
+
.update(`code-v1\u0000${service}\u0000${code}`)
|
|
262
|
+
.digest("hex")
|
|
263
|
+
.slice(0, 32);
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Issue identity plus the scheme that produced it. A valid
|
|
267
|
+
* `autter.error.code` selects "code-v1"; everything else keeps the
|
|
268
|
+
* historical message/stack/route fingerprint ("message-v1") byte-for-byte,
|
|
269
|
+
* so existing issues never regroup.
|
|
270
|
+
*/
|
|
271
|
+
export function occurrenceFingerprint(input) {
|
|
272
|
+
const code = validErrorCode(input.errorCode);
|
|
273
|
+
return code
|
|
274
|
+
? { fingerprint: codeFingerprint(input.service, code), scheme: "code-v1" }
|
|
275
|
+
: { fingerprint: messageFingerprint(input), scheme: "message-v1" };
|
|
276
|
+
}
|
|
236
277
|
export function fingerprintOccurrence(input) {
|
|
278
|
+
return occurrenceFingerprint(input).fingerprint;
|
|
279
|
+
}
|
|
280
|
+
/** Scheme "message-v1" — the pre-1.5.0 fingerprint, unchanged. */
|
|
281
|
+
function messageFingerprint(input) {
|
|
237
282
|
const parts = [
|
|
238
283
|
input.source,
|
|
239
284
|
input.service,
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { loadConfig } from "./config.js";
|
|
2
2
|
import { createIngesterApp } from "./server.js";
|
|
3
|
+
import { ingesterVersion } from "./sdk-versions.js";
|
|
3
4
|
const config = loadConfig();
|
|
4
|
-
const { app, store, sink } = createIngesterApp(config);
|
|
5
|
+
const { app, store, sink, reportIngesterInfo } = createIngesterApp(config);
|
|
5
6
|
const server = app.listen(config.port, () => {
|
|
6
|
-
console.log(`autter otlp-ingester listening on :${config.port} ` +
|
|
7
|
+
console.log(`autter otlp-ingester ${ingesterVersion()} listening on :${config.port} ` +
|
|
7
8
|
`(clickhouse: ${config.clickhouseUrl ? "configured" : "NOT configured"})`);
|
|
8
9
|
});
|
|
9
10
|
// Warm the schema at boot so the first ingest request doesn't pay for DDL.
|
|
10
11
|
if (store.configured) {
|
|
11
|
-
store.ensureSchema().catch((err) => {
|
|
12
|
+
store.ensureSchema().then(reportIngesterInfo).catch((err) => {
|
|
12
13
|
console.error("clickhouse schema bootstrap failed (will retry on first ingest):", err?.message ?? err);
|
|
13
14
|
});
|
|
14
15
|
}
|
|
@@ -32,5 +33,6 @@ process.on("SIGTERM", () => void shutdown("SIGTERM"));
|
|
|
32
33
|
process.on("SIGINT", () => void shutdown("SIGINT"));
|
|
33
34
|
export { createIngesterApp } from "./server.js";
|
|
34
35
|
export { loadConfig } from "./config.js";
|
|
36
|
+
export * from "./compat.js";
|
|
35
37
|
export { SinkForwarder } from "./sink.js";
|
|
36
38
|
export * from "./types.js";
|