@autter/otlp-ingester 1.3.4 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -0
- package/dist/clickhouse.d.ts +35 -0
- package/dist/clickhouse.js +156 -2
- package/dist/compat-manifest.json +94 -0
- package/dist/compat.d.ts +147 -0
- package/dist/compat.js +219 -0
- package/dist/config.d.ts +17 -0
- package/dist/config.js +18 -0
- package/dist/context.d.ts +19 -0
- package/dist/context.js +102 -0
- package/dist/error-fields.d.ts +41 -0
- package/dist/error-fields.js +103 -0
- package/dist/fingerprint.d.ts +29 -0
- package/dist/fingerprint.js +45 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +5 -3
- package/dist/llm.js +16 -3
- package/dist/log-promotion.d.ts +55 -0
- package/dist/log-promotion.js +119 -0
- package/dist/logs.d.ts +98 -0
- package/dist/logs.js +261 -0
- package/dist/migrations.d.ts +3 -0
- package/dist/migrations.js +44 -0
- package/dist/normalize-browser.d.ts +16 -4
- package/dist/normalize-browser.js +22 -18
- package/dist/normalize-otlp.d.ts +2 -6
- package/dist/normalize-otlp.js +22 -9
- package/dist/otlp-proto.d.ts +2 -0
- package/dist/otlp-proto.js +21 -0
- package/dist/redact.d.ts +33 -0
- package/dist/redact.js +96 -0
- package/dist/sdk-versions.d.ts +58 -0
- package/dist/sdk-versions.js +167 -0
- package/dist/server.d.ts +3 -0
- package/dist/server.js +171 -5
- package/dist/sink.d.ts +9 -0
- package/dist/sink.js +26 -4
- package/dist/types.d.ts +18 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -9,10 +9,13 @@ one per-repo signal model, fingerprints errors, and writes ClickHouse.
|
|
|
9
9
|
| Route | Payload | Purpose |
|
|
10
10
|
| --- | --- | --- |
|
|
11
11
|
| `POST /v1/traces` | OTLP/JSON `ExportTraceServiceRequest` | Error spans → occurrences; all spans → `runtime_spans`; server spans → usage rollups; GenAI spans → `runtime_llm_calls` |
|
|
12
|
+
| `POST /v1/logs` | OTLP `ExportLogsServiceRequest`, server key | Wide events (request/operation summaries, logs) → `runtime_logs` (+ `runtime_request_1m` rollup); logger-only error records (`autter.capture.mode=log`) → occurrences |
|
|
12
13
|
| `POST /v1/metrics` | OTLP `ExportMetricsServiceRequest` | HTTP-server duration histograms → usage rollups; portable process memory/GC metrics → per-instance memory samples |
|
|
13
14
|
| `POST /v1/platform-events` | JSON, server key | ECS/Kubernetes OOM kills and restarts → memory incident correlation |
|
|
14
15
|
| `POST /v1/profiles` | Symbolized pprof, server key | CPU/in-use heap profile samples |
|
|
15
16
|
| `POST /v1/browser` | Browser payload `version: 1` | Errors/rejections → occurrences; session pings → rollups |
|
|
17
|
+
| `POST /v1/logs` | OTLP logs, server key | Structured logs and operation summaries → `runtime_logs` |
|
|
18
|
+
| `GET /v1/compat` | — | Version, schema level and supported features (public) |
|
|
16
19
|
| `GET /healthz` | — | Liveness + ClickHouse reachability |
|
|
17
20
|
|
|
18
21
|
Auth on every ingest route: `Authorization: Bearer <ingest key>`,
|
|
@@ -45,6 +48,17 @@ AUTTER_INGEST_KEYS='[
|
|
|
45
48
|
The validator webhook may return the same extra fields:
|
|
46
49
|
`{ orgId, repositoryId, scope?, allowedOrigins? }`.
|
|
47
50
|
|
|
51
|
+
### Version compatibility
|
|
52
|
+
|
|
53
|
+
`GET /v1/compat` (public, no tenant data) returns this ingester's version,
|
|
54
|
+
its ClickHouse schema state, and which features from
|
|
55
|
+
`src/compat-manifest.json` it supports. Add
|
|
56
|
+
`?features=operation_logging&sdk=@autter/runtime-node@1.3.0` to get the evaluated
|
|
57
|
+
incompatibilities. Every response carries `x-autter-ingester-version`. The
|
|
58
|
+
ingester records the SDK name and version each service sends with
|
|
59
|
+
(`runtime_sdk_versions`) and its own version (`runtime_ingester_info`), and
|
|
60
|
+
the Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIBILITY.md).
|
|
61
|
+
|
|
48
62
|
## Configuration
|
|
49
63
|
|
|
50
64
|
| Env | Default | Description |
|
|
@@ -64,8 +78,44 @@ The validator webhook may return the same extra fields:
|
|
|
64
78
|
| `MAX_BODY_BYTES` | `1048576` | Request body cap |
|
|
65
79
|
| `RATE_LIMIT_PER_MINUTE` | `300` | Per-key fixed window (server keys) |
|
|
66
80
|
| `CLIENT_RATE_LIMIT_PER_MINUTE` | `120` | Per-key fixed window (client keys) |
|
|
81
|
+
| `PROMOTION_LOOKUPS_PER_MINUTE` | `30` | Per-tenant ClickHouse dedupe lookups for `/v1/logs` error promotion; over budget, records are promoted with in-batch dedupe only |
|
|
67
82
|
| `OCCURRENCE_TTL_DAYS` / `SPAN_TTL_DAYS` / `METRICS_TTL_DAYS` | `14` / `7` / `90` | ClickHouse TTLs (applied at table creation) |
|
|
68
83
|
| `LLM_CALL_TTL_DAYS` | `90` | Retention for `runtime_llm_calls` rows |
|
|
84
|
+
| `LOG_TTL_DAYS` | `14` | Retention for `runtime_logs` (request summaries are always kept, never sampled, so this is the main volume knob). Unlike the other TTLs it is also applied to **existing** tables: at boot the ingester compares it with the table's TTL and runs `ALTER TABLE … MODIFY TTL` only when they differ; a failure is logged and retried next boot |
|
|
85
|
+
| `AUTTER_REDACT_VALUE_PATTERNS` | — | JSON array of extra regex sources masked in stored/forwarded text and attribute values (built-in secret/PII patterns always apply) |
|
|
86
|
+
| `AUTTER_REDACT_KEY_PATTERNS` | — | JSON array of extra regex sources for attribute keys whose values are masked wholesale |
|
|
87
|
+
|
|
88
|
+
## Coded errors and request summaries (1.5.0)
|
|
89
|
+
|
|
90
|
+
- **Code-based grouping.** An occurrence carrying a valid `autter.error.code`
|
|
91
|
+
(`^[a-z][a-z0-9_]*(\.[a-z0-9_]+){0,3}$`, ≤80 chars) is fingerprinted by
|
|
92
|
+
scheme `code-v1`: `sha256("code-v1\0" + service + "\0" + code)[:32]` —
|
|
93
|
+
independent of source, message, stack and route, so one code is one issue
|
|
94
|
+
across traces, log promotion, the browser and external connectors.
|
|
95
|
+
Uncoded errors keep the historical fingerprint (`message-v1`) byte-for-byte.
|
|
96
|
+
`fingerprint_scheme` is stored per occurrence and sent to the sink.
|
|
97
|
+
- **Declared fields.** `autter.error.why|fix|link|expected` and
|
|
98
|
+
`autter.request.id` are lifted from span/exception-event attributes, log
|
|
99
|
+
attributes and browser context into `runtime_error_occurrences`
|
|
100
|
+
(`error_why`, `error_fix`, `error_link`, `expected`, `request_id`) and the
|
|
101
|
+
sink payload. why/fix are scrubbed and capped at 1000 chars; links must be
|
|
102
|
+
http(s) and ≤500 chars. Exception spans inherit the request id of their
|
|
103
|
+
server span.
|
|
104
|
+
- **Request summaries.** `/v1/logs` lifts `autter.operation.kind`,
|
|
105
|
+
`autter.request.id`, `http.route` (id-normalised), `http.response.status_code`,
|
|
106
|
+
`autter.error.code` and the `autter.operation.ai` rollup (`cost_usd`, `calls`)
|
|
107
|
+
into `runtime_logs` columns. `kind = 'request'` rows feed the
|
|
108
|
+
`runtime_request_1m` materialized view (per route/method/minute: count,
|
|
109
|
+
failed, duration sum, p50/p95). The view counts every insert, so a retried
|
|
110
|
+
batch can be counted twice — route stats are approximate under exporter
|
|
111
|
+
retries; `runtime_logs` itself collapses duplicates.
|
|
112
|
+
- **Log promotion.** SDKs without a trace pipeline (logger-only mode,
|
|
113
|
+
`@autter/runtime-edge`) send errors as log records with `exception.*` and
|
|
114
|
+
`autter.capture.mode = "log"`. `/v1/logs` promotes those (severity ≥ error)
|
|
115
|
+
to server occurrences — stored, counted and forwarded to the sink —
|
|
116
|
+
skipping a record when its trace id already produced an occurrence in the
|
|
117
|
+
same batch or (ClickHouse lookup) within ±60 s. If the lookup fails the
|
|
118
|
+
record is promoted anyway.
|
|
69
119
|
|
|
70
120
|
## LLM / GenAI calls
|
|
71
121
|
|
package/dist/clickhouse.d.ts
CHANGED
|
@@ -2,12 +2,17 @@ import type { IngesterConfig } from "./config.js";
|
|
|
2
2
|
import { type LatencyHistogram } from "./latency.js";
|
|
3
3
|
import { type ProfileSample } from "./profiles.js";
|
|
4
4
|
import { type MemorySample, type PlatformEvent } from "./memory.js";
|
|
5
|
+
import { type RuntimeLogRecord } from "./logs.js";
|
|
6
|
+
import { type SdkVersionRow } from "./sdk-versions.js";
|
|
7
|
+
import type { SchemaStatus } from "./compat.js";
|
|
5
8
|
import { type Migration } from "./migrations.js";
|
|
6
9
|
import type { IngestContext, RuntimeLlmCall, RuntimeMetricPoint, RuntimeOccurrence, RuntimeSpanRow } from "./types.js";
|
|
7
10
|
export declare class ClickHouseStore {
|
|
8
11
|
private readonly config;
|
|
9
12
|
private client;
|
|
10
13
|
private ensurePromise;
|
|
14
|
+
private schemaState;
|
|
15
|
+
private appliedIds;
|
|
11
16
|
constructor(config: IngesterConfig);
|
|
12
17
|
get configured(): boolean;
|
|
13
18
|
private getClient;
|
|
@@ -22,9 +27,39 @@ export declare class ClickHouseStore {
|
|
|
22
27
|
* next request retries.
|
|
23
28
|
*/
|
|
24
29
|
ensureSchema(migrations?: Migration[]): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Bring runtime_logs' TTL in line with LOG_TTL_DAYS. Idempotent: reads the
|
|
32
|
+
* current TTL from system.tables and only issues `MODIFY TTL` when it
|
|
33
|
+
* differs (MODIFY TTL rewrites TTL info for existing parts, so it must not
|
|
34
|
+
* run on every boot). Best-effort — a failure is logged and never blocks
|
|
35
|
+
* ingest; the next boot retries.
|
|
36
|
+
*/
|
|
37
|
+
applyLogTtl(): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Trace ids (from `traceIds`) that already produced an occurrence for
|
|
40
|
+
* this tenant in [from, to] — the cross-batch half of the log-promotion
|
|
41
|
+
* dedupe. `excludeIds` are the candidate occurrences' own deterministic
|
|
42
|
+
* ids, so a retried batch does not dedupe against its own earlier,
|
|
43
|
+
* partially-committed write. Throws on failure; the caller decides.
|
|
44
|
+
*/
|
|
45
|
+
recentOccurrenceTraceIds(ctx: IngestContext, traceIds: string[], from: Date, to: Date, excludeIds: string[]): Promise<Set<string>>;
|
|
46
|
+
/** Schema state as of the last ensureSchema attempt — never triggers DDL
|
|
47
|
+
* (it backs the unauthenticated /v1/compat endpoint). */
|
|
48
|
+
schemaStatus(): {
|
|
49
|
+
status: SchemaStatus;
|
|
50
|
+
applied: string[];
|
|
51
|
+
};
|
|
52
|
+
insertSdkVersions(ctx: IngestContext, rows: SdkVersionRow[]): Promise<void>;
|
|
53
|
+
/** Record this ingester's version and schema level (global, not tenant data). */
|
|
54
|
+
insertIngesterInfo(info: {
|
|
55
|
+
version: string;
|
|
56
|
+
schemaLevel: string;
|
|
57
|
+
report: unknown;
|
|
58
|
+
}): Promise<void>;
|
|
25
59
|
ping(): Promise<boolean>;
|
|
26
60
|
insertOccurrences(ctx: IngestContext, occurrences: RuntimeOccurrence[]): Promise<void>;
|
|
27
61
|
insertSpans(ctx: IngestContext, spans: RuntimeSpanRow[]): Promise<void>;
|
|
62
|
+
insertLogs(ctx: IngestContext, logs: RuntimeLogRecord[]): Promise<void>;
|
|
28
63
|
insertProfileSamples(ctx: IngestContext, samples: ProfileSample[]): Promise<void>;
|
|
29
64
|
insertMemorySamples(ctx: IngestContext, samples: MemorySample[]): Promise<void>;
|
|
30
65
|
insertPlatformEvent(ctx: IngestContext, event: PlatformEvent): Promise<void>;
|
package/dist/clickhouse.js
CHANGED
|
@@ -3,7 +3,9 @@ import { latencyTableDDL } from "./latency.js";
|
|
|
3
3
|
import { profileTableDDL } from "./profiles.js";
|
|
4
4
|
import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
|
|
5
5
|
import { sourceMapTableDDL } from "./source-maps.js";
|
|
6
|
-
import {
|
|
6
|
+
import { logTableDDL, requestRollupTableDDL, } from "./logs.js";
|
|
7
|
+
import { ingesterInfoTableDDL, sdkVersionTableDDL, } from "./sdk-versions.js";
|
|
8
|
+
import { MIGRATIONS, OCCURRENCE_CODE_COLUMNS, OCCURRENCE_REQUEST_ID_INDEX, migrationsTableDDL, } from "./migrations.js";
|
|
7
9
|
/**
|
|
8
10
|
* ClickHouse persistence. The schema is idempotent and applied lazily once
|
|
9
11
|
* per process. Every row is keyed by (org_id, repository_id) — tenant
|
|
@@ -15,10 +17,16 @@ const INSERT_SETTINGS = {
|
|
|
15
17
|
async_insert: 1,
|
|
16
18
|
wait_for_async_insert: 1,
|
|
17
19
|
};
|
|
20
|
+
/** "YYYY-MM-DD hh:mm:ss.sss" (UTC) — the DateTime64 query-parameter form. */
|
|
21
|
+
function chDateTime(date) {
|
|
22
|
+
return date.toISOString().replace("T", " ").replace("Z", "");
|
|
23
|
+
}
|
|
18
24
|
export class ClickHouseStore {
|
|
19
25
|
config;
|
|
20
26
|
client = null;
|
|
21
27
|
ensurePromise = null;
|
|
28
|
+
schemaState = "pending";
|
|
29
|
+
appliedIds = [];
|
|
22
30
|
constructor(config) {
|
|
23
31
|
this.config = config;
|
|
24
32
|
}
|
|
@@ -46,6 +54,7 @@ export class ClickHouseStore {
|
|
|
46
54
|
schemaStatements() {
|
|
47
55
|
const db = this.config.clickhouseDatabase;
|
|
48
56
|
const { occurrenceTtlDays, spanTtlDays, metricsTtlDays, llmCallTtlDays } = this.config;
|
|
57
|
+
const logTtlDays = this.config.logTtlDays ?? 14;
|
|
49
58
|
return [
|
|
50
59
|
`CREATE DATABASE IF NOT EXISTS ${db}`,
|
|
51
60
|
latencyTableDDL(db, metricsTtlDays),
|
|
@@ -53,6 +62,13 @@ export class ClickHouseStore {
|
|
|
53
62
|
memoryTableDDL(db),
|
|
54
63
|
platformEventTableDDL(db),
|
|
55
64
|
sourceMapTableDDL(db),
|
|
65
|
+
logTableDDL(db, logTtlDays),
|
|
66
|
+
// The table only: its materialized view reads 0012 columns that an
|
|
67
|
+
// existing runtime_logs lacks until migrations run, so the view is
|
|
68
|
+
// created by migration 0014 alone (migrations also run on fresh DBs).
|
|
69
|
+
requestRollupTableDDL(db),
|
|
70
|
+
sdkVersionTableDDL(db, metricsTtlDays),
|
|
71
|
+
ingesterInfoTableDDL(db),
|
|
56
72
|
`CREATE TABLE IF NOT EXISTS ${db}.runtime_error_occurrences (
|
|
57
73
|
org_id String,
|
|
58
74
|
repository_id String,
|
|
@@ -77,7 +93,9 @@ export class ClickHouseStore {
|
|
|
77
93
|
session_id String DEFAULT '',
|
|
78
94
|
attributes String DEFAULT '{}' CODEC(ZSTD(1)),
|
|
79
95
|
occurred_at DateTime64(3, 'UTC'),
|
|
80
|
-
ingested_at DateTime64(3, 'UTC') DEFAULT now64(3)
|
|
96
|
+
ingested_at DateTime64(3, 'UTC') DEFAULT now64(3),
|
|
97
|
+
${OCCURRENCE_CODE_COLUMNS.join(",\n\t\t\t\t")},
|
|
98
|
+
${OCCURRENCE_REQUEST_ID_INDEX}
|
|
81
99
|
)
|
|
82
100
|
ENGINE = MergeTree
|
|
83
101
|
PARTITION BY toDate(occurred_at)
|
|
@@ -190,12 +208,126 @@ export class ClickHouseStore {
|
|
|
190
208
|
});
|
|
191
209
|
console.log(`clickhouse migration applied: ${migration.id}`);
|
|
192
210
|
}
|
|
211
|
+
await this.applyLogTtl();
|
|
212
|
+
this.appliedIds = migrations.map((migration) => migration.id);
|
|
213
|
+
this.schemaState = "ready";
|
|
193
214
|
})().catch((err) => {
|
|
194
215
|
this.ensurePromise = null;
|
|
216
|
+
this.schemaState = "failed";
|
|
195
217
|
throw err;
|
|
196
218
|
});
|
|
197
219
|
return this.ensurePromise;
|
|
198
220
|
}
|
|
221
|
+
/**
|
|
222
|
+
* Bring runtime_logs' TTL in line with LOG_TTL_DAYS. Idempotent: reads the
|
|
223
|
+
* current TTL from system.tables and only issues `MODIFY TTL` when it
|
|
224
|
+
* differs (MODIFY TTL rewrites TTL info for existing parts, so it must not
|
|
225
|
+
* run on every boot). Best-effort — a failure is logged and never blocks
|
|
226
|
+
* ingest; the next boot retries.
|
|
227
|
+
*/
|
|
228
|
+
async applyLogTtl() {
|
|
229
|
+
const days = this.config.logTtlDays ?? 14;
|
|
230
|
+
const db = this.config.clickhouseDatabase;
|
|
231
|
+
try {
|
|
232
|
+
const client = this.getClient();
|
|
233
|
+
const rows = await client.query({
|
|
234
|
+
query: `SELECT engine_full FROM system.tables WHERE database = {db:String} AND name = 'runtime_logs'`,
|
|
235
|
+
query_params: { db },
|
|
236
|
+
format: "JSONEachRow",
|
|
237
|
+
});
|
|
238
|
+
const [table] = await rows.json();
|
|
239
|
+
if (!table)
|
|
240
|
+
return;
|
|
241
|
+
const current = /\bTTL\b[^]*?toIntervalDay\((\d+)\)/.exec(table.engine_full);
|
|
242
|
+
if (!current) {
|
|
243
|
+
console.warn("runtime_logs TTL not recognised — LOG_TTL_DAYS not applied");
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
if (Number(current[1]) === days)
|
|
247
|
+
return;
|
|
248
|
+
await client.command({
|
|
249
|
+
query: `ALTER TABLE ${db}.runtime_logs MODIFY TTL toDateTime(occurred_at) + INTERVAL ${days} DAY`,
|
|
250
|
+
});
|
|
251
|
+
console.log(`runtime_logs TTL changed from ${current[1]} to ${days} days (LOG_TTL_DAYS)`);
|
|
252
|
+
}
|
|
253
|
+
catch (err) {
|
|
254
|
+
console.warn("applying LOG_TTL_DAYS to runtime_logs failed (will retry next boot):", err instanceof Error ? err.message : err);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Trace ids (from `traceIds`) that already produced an occurrence for
|
|
259
|
+
* this tenant in [from, to] — the cross-batch half of the log-promotion
|
|
260
|
+
* dedupe. `excludeIds` are the candidate occurrences' own deterministic
|
|
261
|
+
* ids, so a retried batch does not dedupe against its own earlier,
|
|
262
|
+
* partially-committed write. Throws on failure; the caller decides.
|
|
263
|
+
*/
|
|
264
|
+
async recentOccurrenceTraceIds(ctx, traceIds, from, to, excludeIds) {
|
|
265
|
+
if (traceIds.length === 0 || !this.configured)
|
|
266
|
+
return new Set();
|
|
267
|
+
await this.ensureSchema();
|
|
268
|
+
const rows = await this.getClient().query({
|
|
269
|
+
query: `SELECT DISTINCT trace_id FROM ${this.table("runtime_error_occurrences")}
|
|
270
|
+
WHERE org_id = {org:String} AND repository_id = {repo:String}
|
|
271
|
+
AND occurred_at >= {from:DateTime64(3, 'UTC')} AND occurred_at <= {to:DateTime64(3, 'UTC')}
|
|
272
|
+
AND has({traces:Array(String)}, trace_id)
|
|
273
|
+
AND NOT has({exclude:Array(String)}, occurrence_id)`,
|
|
274
|
+
query_params: {
|
|
275
|
+
org: ctx.orgId,
|
|
276
|
+
repo: ctx.repositoryId,
|
|
277
|
+
from: chDateTime(from),
|
|
278
|
+
to: chDateTime(to),
|
|
279
|
+
traces: traceIds,
|
|
280
|
+
exclude: excludeIds,
|
|
281
|
+
},
|
|
282
|
+
format: "JSONEachRow",
|
|
283
|
+
});
|
|
284
|
+
return new Set((await rows.json()).map((row) => row.trace_id));
|
|
285
|
+
}
|
|
286
|
+
/** Schema state as of the last ensureSchema attempt — never triggers DDL
|
|
287
|
+
* (it backs the unauthenticated /v1/compat endpoint). */
|
|
288
|
+
schemaStatus() {
|
|
289
|
+
if (!this.configured)
|
|
290
|
+
return { status: "unconfigured", applied: [] };
|
|
291
|
+
return { status: this.schemaState, applied: [...this.appliedIds] };
|
|
292
|
+
}
|
|
293
|
+
async insertSdkVersions(ctx, rows) {
|
|
294
|
+
if (!rows.length || !this.configured)
|
|
295
|
+
return;
|
|
296
|
+
await this.ensureSchema();
|
|
297
|
+
await this.getClient().insert({
|
|
298
|
+
table: this.table("runtime_sdk_versions"),
|
|
299
|
+
format: "JSONEachRow",
|
|
300
|
+
clickhouse_settings: INSERT_SETTINGS,
|
|
301
|
+
values: rows.map((row) => ({
|
|
302
|
+
org_id: ctx.orgId,
|
|
303
|
+
repository_id: ctx.repositoryId,
|
|
304
|
+
service: row.service,
|
|
305
|
+
environment: row.environment,
|
|
306
|
+
sdk_name: row.sdkName,
|
|
307
|
+
sdk_version: row.sdkVersion,
|
|
308
|
+
sdk_language: row.sdkLanguage,
|
|
309
|
+
ingester_version: row.ingesterVersion,
|
|
310
|
+
last_seen: row.seenAt.toISOString(),
|
|
311
|
+
})),
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
/** Record this ingester's version and schema level (global, not tenant data). */
|
|
315
|
+
async insertIngesterInfo(info) {
|
|
316
|
+
if (!this.configured)
|
|
317
|
+
return;
|
|
318
|
+
await this.ensureSchema();
|
|
319
|
+
await this.getClient().insert({
|
|
320
|
+
table: this.table("runtime_ingester_info"),
|
|
321
|
+
format: "JSONEachRow",
|
|
322
|
+
clickhouse_settings: INSERT_SETTINGS,
|
|
323
|
+
values: [{
|
|
324
|
+
ingester_version: info.version,
|
|
325
|
+
schema_level: info.schemaLevel,
|
|
326
|
+
report: JSON.stringify(info.report),
|
|
327
|
+
reported_at: new Date().toISOString(),
|
|
328
|
+
}],
|
|
329
|
+
});
|
|
330
|
+
}
|
|
199
331
|
async ping() {
|
|
200
332
|
if (!this.configured)
|
|
201
333
|
return false;
|
|
@@ -234,6 +366,13 @@ export class ClickHouseStore {
|
|
|
234
366
|
session_id: o.sessionId ?? "",
|
|
235
367
|
attributes: JSON.stringify(o.attributes ?? {}),
|
|
236
368
|
occurred_at: o.occurredAt.toISOString(),
|
|
369
|
+
error_code: o.errorCode ?? "",
|
|
370
|
+
error_why: o.why ?? "",
|
|
371
|
+
error_fix: o.fix ?? "",
|
|
372
|
+
error_link: o.link ?? "",
|
|
373
|
+
expected: o.expected ? 1 : 0,
|
|
374
|
+
request_id: o.requestId ?? "",
|
|
375
|
+
fingerprint_scheme: o.fingerprintScheme,
|
|
237
376
|
})),
|
|
238
377
|
});
|
|
239
378
|
}
|
|
@@ -265,6 +404,21 @@ export class ClickHouseStore {
|
|
|
265
404
|
})),
|
|
266
405
|
});
|
|
267
406
|
}
|
|
407
|
+
async insertLogs(ctx, logs) {
|
|
408
|
+
if (!logs.length)
|
|
409
|
+
return;
|
|
410
|
+
if (!this.configured)
|
|
411
|
+
throw new Error("CLICKHOUSE_URL is not configured");
|
|
412
|
+
await this.ensureSchema();
|
|
413
|
+
await this.getClient().insert({ table: this.table("runtime_logs"), format: "JSONEachRow", clickhouse_settings: INSERT_SETTINGS,
|
|
414
|
+
values: logs.map((row) => ({ org_id: ctx.orgId, repository_id: ctx.repositoryId, event_id: row.id,
|
|
415
|
+
service: row.service, environment: row.environment, release: row.release, trace_id: row.traceId,
|
|
416
|
+
span_id: row.spanId, operation_id: row.operationId, operation: row.operation, event_type: row.type,
|
|
417
|
+
severity: row.severity, message: row.message, outcome: row.outcome, duration_ms: row.durationMs,
|
|
418
|
+
attributes: JSON.stringify(row.attributes), occurred_at: row.occurredAt.toISOString(),
|
|
419
|
+
kind: row.kind, request_id: row.requestId, route: row.route, status_code: row.statusCode,
|
|
420
|
+
error_code: row.errorCode, ai_cost_usd: row.aiCostUsd, ai_calls: row.aiCalls })) });
|
|
421
|
+
}
|
|
268
422
|
async insertProfileSamples(ctx, samples) {
|
|
269
423
|
if (!samples.length)
|
|
270
424
|
return;
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
{
|
|
2
|
+
"manifestVersion": 1,
|
|
3
|
+
"docs": "https://github.com/Autter-dev/autter-runtime/blob/main/docs/COMPATIBILITY.md",
|
|
4
|
+
"upgrade": {
|
|
5
|
+
"ingester": "docker pull ghcr.io/autter-dev/otlp-ingester:latest and restart it (ClickHouse migrations run at boot)",
|
|
6
|
+
"sdk": "npm install {package}@latest"
|
|
7
|
+
},
|
|
8
|
+
"features": [
|
|
9
|
+
{
|
|
10
|
+
"id": "llm_calls",
|
|
11
|
+
"title": "LLM call tracking",
|
|
12
|
+
"ingester": "1.1.0",
|
|
13
|
+
"migrations": ["0004-llm-calls"],
|
|
14
|
+
"sdks": { "@autter/runtime-node": "1.1.0", "@autter/runtime-next": "1.1.0" }
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"id": "endpoint_latency",
|
|
18
|
+
"title": "Endpoint latency regression detection",
|
|
19
|
+
"ingester": "1.3.1",
|
|
20
|
+
"migrations": ["0005-latency-histograms"],
|
|
21
|
+
"sdks": { "@autter/runtime-node": "1.3.0", "@autter/runtime-next": "1.3.0" }
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"id": "browser_network_events",
|
|
25
|
+
"title": "Browser network, timing and outcome capture",
|
|
26
|
+
"ingester": "1.3.2",
|
|
27
|
+
"migrations": [],
|
|
28
|
+
"sdks": { "@autter/runtime-browser": "1.3.2", "@autter/runtime-next": "1.3.2" },
|
|
29
|
+
"browserEvents": ["request_failure", "timing", "outcome"]
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"id": "profiles",
|
|
33
|
+
"title": "Profile ingestion",
|
|
34
|
+
"ingester": "1.3.2",
|
|
35
|
+
"migrations": ["0006-profile-samples"],
|
|
36
|
+
"route": "/v1/profiles",
|
|
37
|
+
"sdks": {}
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"id": "source_maps",
|
|
41
|
+
"title": "Source map upload",
|
|
42
|
+
"ingester": "1.3.2",
|
|
43
|
+
"migrations": ["0007-source-maps"],
|
|
44
|
+
"route": "/v1/sourcemaps",
|
|
45
|
+
"sdks": {}
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"id": "memory_metrics",
|
|
49
|
+
"title": "Memory pressure detection",
|
|
50
|
+
"ingester": "1.3.3",
|
|
51
|
+
"migrations": ["0008-memory-signals", "0010-memory-temporality"],
|
|
52
|
+
"sdks": { "@autter/runtime-node": "1.3.3", "@autter/runtime-next": "1.3.3" }
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "platform_events",
|
|
56
|
+
"title": "Platform OOM/restart events",
|
|
57
|
+
"ingester": "1.3.3",
|
|
58
|
+
"migrations": ["0008-memory-signals"],
|
|
59
|
+
"route": "/v1/platform-events",
|
|
60
|
+
"sdks": {}
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
"id": "csp_violations",
|
|
64
|
+
"title": "Browser CSP violation capture",
|
|
65
|
+
"ingester": "1.3.4",
|
|
66
|
+
"migrations": [],
|
|
67
|
+
"sdks": { "@autter/runtime-browser": "1.3.3", "@autter/runtime-next": "1.3.4" },
|
|
68
|
+
"browserEvents": ["csp_violation"]
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"id": "operation_logging",
|
|
72
|
+
"title": "Operation logging",
|
|
73
|
+
"ingester": "1.4.0",
|
|
74
|
+
"migrations": ["0011-runtime-logs"],
|
|
75
|
+
"route": "/v1/logs",
|
|
76
|
+
"sdks": { "@autter/runtime-node": "1.4.0", "@autter/runtime-next": "1.4.0" }
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"id": "request_summaries",
|
|
80
|
+
"title": "Request summaries and the per-route request rollup",
|
|
81
|
+
"ingester": "1.5.0",
|
|
82
|
+
"migrations": ["0012-runtime-logs-requests", "0014-runtime-request-1m"],
|
|
83
|
+
"route": "/v1/logs",
|
|
84
|
+
"sdks": { "@autter/runtime-node": "1.5.0", "@autter/runtime-next": "1.5.0", "@autter/runtime-edge": "1.0.0" }
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"id": "coded_errors",
|
|
88
|
+
"title": "Coded errors and code-based grouping",
|
|
89
|
+
"ingester": "1.5.0",
|
|
90
|
+
"migrations": ["0013-occurrence-codes"],
|
|
91
|
+
"sdks": { "@autter/runtime-node": "1.5.0", "@autter/runtime-next": "1.5.0", "@autter/runtime-browser": "1.4.0", "@autter/runtime-edge": "1.0.0" }
|
|
92
|
+
}
|
|
93
|
+
]
|
|
94
|
+
}
|
package/dist/compat.d.ts
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Version compatibility between Autter Runtime SDKs, this ingester and its
|
|
3
|
+
* ClickHouse schema — the single source of truth.
|
|
4
|
+
*
|
|
5
|
+
* `compat-manifest.json` lists every feature that needs a minimum ingester
|
|
6
|
+
* version, a ClickHouse migration, a dedicated ingest route, or a minimum SDK
|
|
7
|
+
* version. Everything else derives from it:
|
|
8
|
+
*
|
|
9
|
+
* - the ingester serves it (evaluated against its own version and applied
|
|
10
|
+
* migrations) at `GET /v1/compat`;
|
|
11
|
+
* - `@autter/runtime-node` / `@autter/runtime-next` bundle this file at build
|
|
12
|
+
* time to warn once when an enabled feature needs a newer ingester, and to
|
|
13
|
+
* power `npx @autter/runtime-node doctor`;
|
|
14
|
+
* - the Python adapter (`adapters/python/compat.py`) embeds a copy that a
|
|
15
|
+
* parity test keeps identical.
|
|
16
|
+
*
|
|
17
|
+
* This module is dependency-free and side-effect-free on purpose: it is
|
|
18
|
+
* bundled into the Node SDK. Keep it that way.
|
|
19
|
+
*
|
|
20
|
+
* Adding a feature: append an entry to compat-manifest.json with the first
|
|
21
|
+
* ingester release that handles it, the migrations it writes to, its route
|
|
22
|
+
* (if it has its own), and the first SDK release that emits it. The
|
|
23
|
+
* consistency test in compat.test.ts checks ids, migrations and routes.
|
|
24
|
+
*/
|
|
25
|
+
export interface CompatFeature {
|
|
26
|
+
id: string;
|
|
27
|
+
title: string;
|
|
28
|
+
/** First ingester release that stores this feature correctly. */
|
|
29
|
+
ingester: string;
|
|
30
|
+
/** ClickHouse migrations (migrations.ts ids) the feature writes to. */
|
|
31
|
+
migrations: string[];
|
|
32
|
+
/** Dedicated ingest route, when the feature has one. */
|
|
33
|
+
route?: string;
|
|
34
|
+
/** First release of each SDK package that emits the feature. */
|
|
35
|
+
sdks: Record<string, string>;
|
|
36
|
+
/** Browser payload event types that belong to this feature. */
|
|
37
|
+
browserEvents?: string[];
|
|
38
|
+
}
|
|
39
|
+
export interface CompatManifest {
|
|
40
|
+
manifestVersion: number;
|
|
41
|
+
docs: string;
|
|
42
|
+
upgrade: {
|
|
43
|
+
ingester: string;
|
|
44
|
+
sdk: string;
|
|
45
|
+
};
|
|
46
|
+
features: CompatFeature[];
|
|
47
|
+
}
|
|
48
|
+
export declare const COMPAT_MANIFEST: CompatManifest;
|
|
49
|
+
export declare const INGESTER_PACKAGE = "@autter/otlp-ingester";
|
|
50
|
+
/** Response header carrying the ingester version on every ingest response. */
|
|
51
|
+
export declare const INGESTER_VERSION_HEADER = "x-autter-ingester-version";
|
|
52
|
+
/** Semver-ish comparison; null when either side is unparsable. */
|
|
53
|
+
export declare function compareVersions(a: string, b: string): number | null;
|
|
54
|
+
/** True/false when comparable, null when the version is unknown/unparsable. */
|
|
55
|
+
export declare function versionAtLeast(version: string | null | undefined, minimum: string): boolean | null;
|
|
56
|
+
export type SchemaStatus = "ready" | "pending" | "failed" | "unconfigured";
|
|
57
|
+
/** What is known about the ingester an SDK talks to. */
|
|
58
|
+
export interface IngesterCompatInfo {
|
|
59
|
+
/** Exact version, or null when the ingester predates /v1/compat. */
|
|
60
|
+
version: string | null;
|
|
61
|
+
/** True when /v1/compat returned 404 (ingester 1.4.0 or older). */
|
|
62
|
+
legacy?: boolean;
|
|
63
|
+
schema?: {
|
|
64
|
+
status: SchemaStatus;
|
|
65
|
+
applied: string[];
|
|
66
|
+
};
|
|
67
|
+
/** Legacy ingesters: route → exists, learned by probing. */
|
|
68
|
+
routes?: Record<string, boolean>;
|
|
69
|
+
}
|
|
70
|
+
export type CompatIssueKind = "ingester_too_old" | "schema_not_applied" | "sdk_too_old";
|
|
71
|
+
export interface CompatIssue {
|
|
72
|
+
feature: string;
|
|
73
|
+
title: string;
|
|
74
|
+
kind: CompatIssueKind;
|
|
75
|
+
/** Package that must change: the ingester or an SDK package name. */
|
|
76
|
+
component: string;
|
|
77
|
+
required: string;
|
|
78
|
+
/** What was found ("1.3.4", "1.4.0 or older", …). */
|
|
79
|
+
actual: string;
|
|
80
|
+
/** One line: what is wrong, versions, and the fix. */
|
|
81
|
+
message: string;
|
|
82
|
+
fix: string;
|
|
83
|
+
}
|
|
84
|
+
export interface SdkIdentity {
|
|
85
|
+
name: string;
|
|
86
|
+
version: string;
|
|
87
|
+
}
|
|
88
|
+
export declare function featureById(id: string, manifest?: CompatManifest): CompatFeature | undefined;
|
|
89
|
+
/** Features the given browser payload event types belong to. */
|
|
90
|
+
export declare function featuresForBrowserEvents(types: Iterable<string>, manifest?: CompatManifest): string[];
|
|
91
|
+
export declare function ingesterUpgradeHint(manifest?: CompatManifest): string;
|
|
92
|
+
/**
|
|
93
|
+
* Every incompatibility for the features in use. Unknown is never an issue:
|
|
94
|
+
* when the ingester version can't be determined (and a legacy route probe
|
|
95
|
+
* didn't prove a route missing) nothing is reported, so the check has no
|
|
96
|
+
* false positives.
|
|
97
|
+
*/
|
|
98
|
+
export declare function evaluateCompat(input: {
|
|
99
|
+
features: Iterable<string>;
|
|
100
|
+
ingester: IngesterCompatInfo | null;
|
|
101
|
+
sdk?: SdkIdentity | null;
|
|
102
|
+
manifest?: CompatManifest;
|
|
103
|
+
}): CompatIssue[];
|
|
104
|
+
export interface CompatFeatureStatus {
|
|
105
|
+
id: string;
|
|
106
|
+
title: string;
|
|
107
|
+
ingester: string;
|
|
108
|
+
migrations: string[];
|
|
109
|
+
route?: string;
|
|
110
|
+
sdks: Record<string, string>;
|
|
111
|
+
browserEvents?: string[];
|
|
112
|
+
/** This ingester's version is new enough. */
|
|
113
|
+
ingesterSupported: boolean;
|
|
114
|
+
/** Migrations the feature needs that are not applied yet. */
|
|
115
|
+
missingMigrations: string[];
|
|
116
|
+
/** Ingester new enough AND schema ready with every migration applied. */
|
|
117
|
+
available: boolean;
|
|
118
|
+
}
|
|
119
|
+
export interface CompatReport {
|
|
120
|
+
ingester: {
|
|
121
|
+
name: string;
|
|
122
|
+
version: string;
|
|
123
|
+
};
|
|
124
|
+
schema: {
|
|
125
|
+
status: SchemaStatus;
|
|
126
|
+
/** Highest applied migration id ("" when none known). */
|
|
127
|
+
level: string;
|
|
128
|
+
/** Highest migration this ingester ships. */
|
|
129
|
+
latest: string;
|
|
130
|
+
applied: string[];
|
|
131
|
+
};
|
|
132
|
+
features: CompatFeatureStatus[];
|
|
133
|
+
manifestVersion: number;
|
|
134
|
+
docs: string;
|
|
135
|
+
upgrade: CompatManifest["upgrade"];
|
|
136
|
+
/** Present when the request named features (and optionally an SDK). */
|
|
137
|
+
issues?: CompatIssue[];
|
|
138
|
+
}
|
|
139
|
+
export declare function buildCompatReport(input: {
|
|
140
|
+
version: string;
|
|
141
|
+
schemaStatus: SchemaStatus;
|
|
142
|
+
applied: string[];
|
|
143
|
+
allMigrations: string[];
|
|
144
|
+
manifest?: CompatManifest;
|
|
145
|
+
}): CompatReport;
|
|
146
|
+
/** IngesterCompatInfo from a /v1/compat response body (defensive: it's network input). */
|
|
147
|
+
export declare function ingesterInfoFromReport(body: unknown): IngesterCompatInfo | null;
|