@thotischner/observability-mcp 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis/history.d.ts +36 -2
- package/dist/analysis/history.js +60 -2
- package/dist/analysis/history.test.js +46 -0
- package/dist/auth/csrf.d.ts +6 -0
- package/dist/auth/csrf.js +4 -0
- package/dist/auth/csrf.test.js +22 -0
- package/dist/auth/lockout.d.ts +72 -0
- package/dist/auth/lockout.js +134 -0
- package/dist/auth/lockout.test.d.ts +1 -0
- package/dist/auth/lockout.test.js +133 -0
- package/dist/auth/middleware.d.ts +5 -0
- package/dist/auth/middleware.js +6 -1
- package/dist/auth/middleware.test.js +31 -0
- package/dist/auth/password-policy.d.ts +52 -0
- package/dist/auth/password-policy.js +125 -0
- package/dist/auth/password-policy.test.d.ts +1 -0
- package/dist/auth/password-policy.test.js +111 -0
- package/dist/auth/revocation.d.ts +93 -0
- package/dist/auth/revocation.js +193 -0
- package/dist/auth/revocation.test.d.ts +1 -0
- package/dist/auth/revocation.test.js +136 -0
- package/dist/auth/session.d.ts +7 -0
- package/dist/auth/session.js +6 -0
- package/dist/auth/session.test.js +21 -0
- package/dist/connectors/interface.d.ts +5 -1
- package/dist/connectors/loki.d.ts +45 -1
- package/dist/connectors/loki.js +141 -8
- package/dist/connectors/loki.test.js +171 -1
- package/dist/index.js +217 -3
- package/dist/openapi.js +39 -0
- package/dist/openapi.test.js +1 -0
- package/dist/security/csp.d.ts +64 -0
- package/dist/security/csp.js +135 -0
- package/dist/security/csp.test.d.ts +1 -0
- package/dist/security/csp.test.js +97 -0
- package/dist/tools/query-logs.d.ts +40 -0
- package/dist/tools/query-logs.js +69 -3
- package/dist/tools/validation.d.ts +13 -0
- package/dist/tools/validation.js +74 -0
- package/dist/tools/validation.test.js +54 -1
- package/dist/types.d.ts +48 -0
- package/dist/ui/index.html +42 -15
- package/package.json +1 -1
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { RevocationStore } from "./revocation.js";
|
|
7
|
+
function tmpFile(name = "revocations.jsonl") {
|
|
8
|
+
const dir = mkdtempSync(join(tmpdir(), "omcp-revoke-"));
|
|
9
|
+
return join(dir, name);
|
|
10
|
+
}
|
|
11
|
+
test("memory store revokes a single session by sid", async () => {
|
|
12
|
+
const store = await RevocationStore.create();
|
|
13
|
+
assert.equal(store.persistent, false);
|
|
14
|
+
assert.equal(store.isRevoked({ sub: "alice", iat: 1000, sid: "s1" }), false);
|
|
15
|
+
await store.revokeSession("s1");
|
|
16
|
+
assert.equal(store.isRevoked({ sub: "alice", iat: 1000, sid: "s1" }), true);
|
|
17
|
+
// A different session of the same subject is untouched.
|
|
18
|
+
assert.equal(store.isRevoked({ sub: "alice", iat: 1000, sid: "s2" }), false);
|
|
19
|
+
});
|
|
20
|
+
test("session without a sid is never caught by a session-kind revocation", async () => {
|
|
21
|
+
const store = await RevocationStore.create();
|
|
22
|
+
await store.revokeSession("s1");
|
|
23
|
+
assert.equal(store.isRevoked({ sub: "alice", iat: 1000 }), false);
|
|
24
|
+
});
|
|
25
|
+
test("subject revocation catches sessions issued at or before the cutoff", async () => {
|
|
26
|
+
let clock = 5000;
|
|
27
|
+
const store = await RevocationStore.create({ now: () => clock });
|
|
28
|
+
await store.revokeSubject("bob");
|
|
29
|
+
// Issued before the cutoff → revoked.
|
|
30
|
+
assert.equal(store.isRevoked({ sub: "bob", iat: 4999, sid: "old" }), true);
|
|
31
|
+
// Issued in the same second → revoked (intent: kill what exists now).
|
|
32
|
+
assert.equal(store.isRevoked({ sub: "bob", iat: 5000, sid: "same" }), true);
|
|
33
|
+
// Issued after the cutoff (fresh login) → valid again.
|
|
34
|
+
assert.equal(store.isRevoked({ sub: "bob", iat: 5001, sid: "new" }), false);
|
|
35
|
+
// A different subject is unaffected.
|
|
36
|
+
assert.equal(store.isRevoked({ sub: "carol", iat: 1, sid: "x" }), false);
|
|
37
|
+
});
|
|
38
|
+
test("re-revoking a subject only widens the cutoff window", async () => {
|
|
39
|
+
let clock = 100;
|
|
40
|
+
const store = await RevocationStore.create({ now: () => clock });
|
|
41
|
+
await store.revokeSubject("dave"); // cutoff 100
|
|
42
|
+
clock = 200;
|
|
43
|
+
await store.revokeSubject("dave"); // cutoff 200
|
|
44
|
+
assert.equal(store.isRevoked({ sub: "dave", iat: 150, sid: "mid" }), true);
|
|
45
|
+
assert.equal(store.isRevoked({ sub: "dave", iat: 201, sid: "after" }), false);
|
|
46
|
+
});
|
|
47
|
+
test("entries persist to disk and reload into a fresh store", async () => {
|
|
48
|
+
const path = tmpFile();
|
|
49
|
+
const store = await RevocationStore.create({ path, now: () => 7000 });
|
|
50
|
+
await store.revokeSession("sid-a", { reason: "stolen laptop", by: "admin" });
|
|
51
|
+
await store.revokeSubject("eve", { reason: "offboarded" });
|
|
52
|
+
assert.equal(store.size, 2);
|
|
53
|
+
const reloaded = await RevocationStore.create({ path });
|
|
54
|
+
assert.equal(reloaded.size, 2);
|
|
55
|
+
assert.equal(reloaded.isRevoked({ sub: "x", iat: 1, sid: "sid-a" }), true);
|
|
56
|
+
assert.equal(reloaded.isRevoked({ sub: "eve", iat: 6999, sid: "z" }), true);
|
|
57
|
+
assert.equal(reloaded.isRevoked({ sub: "eve", iat: 7001, sid: "z" }), false);
|
|
58
|
+
});
|
|
59
|
+
test("persisted file is JSONL and carries the metadata", async () => {
|
|
60
|
+
const path = tmpFile();
|
|
61
|
+
const store = await RevocationStore.create({ path, now: () => 42 });
|
|
62
|
+
await store.revokeSession("sid-meta", { reason: "test", by: "root" });
|
|
63
|
+
const lines = readFileSync(path, "utf8").trim().split("\n");
|
|
64
|
+
assert.equal(lines.length, 1);
|
|
65
|
+
const entry = JSON.parse(lines[0]);
|
|
66
|
+
assert.deepEqual(entry, {
|
|
67
|
+
kind: "session",
|
|
68
|
+
value: "sid-meta",
|
|
69
|
+
revokedAt: 42,
|
|
70
|
+
reason: "test",
|
|
71
|
+
by: "root",
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
test("malformed and partial lines are skipped on load", async () => {
|
|
75
|
+
const path = tmpFile();
|
|
76
|
+
writeFileSync(path, [
|
|
77
|
+
JSON.stringify({ kind: "session", value: "good", revokedAt: 1 }),
|
|
78
|
+
"not json at all",
|
|
79
|
+
JSON.stringify({ kind: "bogus", value: "x", revokedAt: 1 }), // bad kind
|
|
80
|
+
JSON.stringify({ kind: "subject", value: "", revokedAt: 1 }), // empty value
|
|
81
|
+
JSON.stringify({ kind: "subject", value: "u", revokedAt: "nope" }), // bad ts
|
|
82
|
+
"", // blank
|
|
83
|
+
JSON.stringify({ kind: "subject", value: "frank", revokedAt: 500 }),
|
|
84
|
+
].join("\n"));
|
|
85
|
+
const store = await RevocationStore.create({ path });
|
|
86
|
+
// Only the two well-formed entries survived.
|
|
87
|
+
assert.equal(store.size, 2);
|
|
88
|
+
assert.equal(store.isRevoked({ sub: "x", iat: 1, sid: "good" }), true);
|
|
89
|
+
assert.equal(store.isRevoked({ sub: "frank", iat: 400, sid: "y" }), true);
|
|
90
|
+
});
|
|
91
|
+
test("missing file is treated as an empty blocklist", async () => {
|
|
92
|
+
const path = join(mkdtempSync(join(tmpdir(), "omcp-revoke-")), "does-not-exist.jsonl");
|
|
93
|
+
const store = await RevocationStore.create({ path });
|
|
94
|
+
assert.equal(store.size, 0);
|
|
95
|
+
assert.equal(store.isRevoked({ sub: "a", iat: 1, sid: "s" }), false);
|
|
96
|
+
// First write creates the file.
|
|
97
|
+
await store.revokeSession("s");
|
|
98
|
+
assert.equal(readFileSync(path, "utf8").trim().split("\n").length, 1);
|
|
99
|
+
});
|
|
100
|
+
test("file under a non-existent directory is created on first write", async () => {
|
|
101
|
+
const dir = mkdtempSync(join(tmpdir(), "omcp-revoke-"));
|
|
102
|
+
const path = join(dir, "nested", "deep", "revocations.jsonl");
|
|
103
|
+
const store = await RevocationStore.create({ path });
|
|
104
|
+
await store.revokeSession("s");
|
|
105
|
+
assert.equal(readFileSync(path, "utf8").trim().split("\n").length, 1);
|
|
106
|
+
});
|
|
107
|
+
test("list() returns a defensive copy in file order", async () => {
|
|
108
|
+
const store = await RevocationStore.create({ now: () => 9 });
|
|
109
|
+
await store.revokeSession("a");
|
|
110
|
+
await store.revokeSubject("b");
|
|
111
|
+
const list = store.list();
|
|
112
|
+
assert.equal(list.length, 2);
|
|
113
|
+
assert.equal(list[0].kind, "session");
|
|
114
|
+
assert.equal(list[1].kind, "subject");
|
|
115
|
+
// Mutating the returned array/objects must not corrupt the store.
|
|
116
|
+
list[0].value = "tampered";
|
|
117
|
+
list.push({ kind: "session", value: "ghost", revokedAt: 0 });
|
|
118
|
+
assert.equal(store.list().length, 2);
|
|
119
|
+
assert.equal(store.list()[0].value, "a");
|
|
120
|
+
});
|
|
121
|
+
test("concurrent revokes all land on disk without interleaving", async () => {
|
|
122
|
+
const path = tmpFile();
|
|
123
|
+
const store = await RevocationStore.create({ path, now: () => 1 });
|
|
124
|
+
await Promise.all(Array.from({ length: 20 }, (_, i) => store.revokeSession(`s${i}`)));
|
|
125
|
+
const lines = readFileSync(path, "utf8").trim().split("\n");
|
|
126
|
+
assert.equal(lines.length, 20);
|
|
127
|
+
// Every line is independently parseable (no torn writes).
|
|
128
|
+
for (const line of lines) {
|
|
129
|
+
assert.doesNotThrow(() => JSON.parse(line));
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
test("reason/by are omitted from the entry when not supplied", async () => {
|
|
133
|
+
const store = await RevocationStore.create({ now: () => 3 });
|
|
134
|
+
const entry = await store.revokeSession("s");
|
|
135
|
+
assert.deepEqual(entry, { kind: "session", value: "s", revokedAt: 3 });
|
|
136
|
+
});
|
package/dist/auth/session.d.ts
CHANGED
|
@@ -25,6 +25,13 @@ export interface SessionPayload {
|
|
|
25
25
|
tenant?: string;
|
|
26
26
|
/** Optional list of role identifiers — used by later phases for RBAC. */
|
|
27
27
|
roles?: string[];
|
|
28
|
+
/** Per-session random identifier. Minted at issue time. Lets the
|
|
29
|
+
* revocation blocklist (see ./revocation.ts) drop a single session
|
|
30
|
+
* without rotating the signing secret. Optional for backward compat:
|
|
31
|
+
* cookies issued before this field existed still verify and simply
|
|
32
|
+
* can't be revoked individually (a subject-wide revocation still
|
|
33
|
+
* catches them via `iat`). */
|
|
34
|
+
sid?: string;
|
|
28
35
|
/** Issued-at, seconds since epoch. */
|
|
29
36
|
iat: number;
|
|
30
37
|
/** Hard expiry, seconds since epoch. */
|
package/dist/auth/session.js
CHANGED
|
@@ -37,6 +37,10 @@ export function issueSession(identity, cfg, now = Math.floor(Date.now() / 1000))
|
|
|
37
37
|
email: identity.email,
|
|
38
38
|
tenant: identity.tenant,
|
|
39
39
|
roles: identity.roles,
|
|
40
|
+
// 16 random bytes ≈ 128 bits — collision-free across any realistic
|
|
41
|
+
// session population, and enough entropy that a sid can't be guessed
|
|
42
|
+
// to forge a revocation target for another user's session.
|
|
43
|
+
sid: randomBytes(16).toString("base64url"),
|
|
40
44
|
iat: now,
|
|
41
45
|
exp: now + ttl,
|
|
42
46
|
};
|
|
@@ -93,6 +97,8 @@ function isSessionPayload(v) {
|
|
|
93
97
|
return false;
|
|
94
98
|
if (o.roles !== undefined && !(Array.isArray(o.roles) && o.roles.every((r) => typeof r === "string")))
|
|
95
99
|
return false;
|
|
100
|
+
if (o.sid !== undefined && typeof o.sid !== "string")
|
|
101
|
+
return false;
|
|
96
102
|
if (o.email !== undefined && typeof o.email !== "string")
|
|
97
103
|
return false;
|
|
98
104
|
if (o.tenant !== undefined && typeof o.tenant !== "string")
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { test } from "node:test";
|
|
2
2
|
import assert from "node:assert/strict";
|
|
3
|
+
import { createHmac } from "node:crypto";
|
|
3
4
|
import { issueSession, verifySession, setCookieHeader, clearCookieHeader, readCookie, generateSecret, DEFAULT_COOKIE_NAME, } from "./session.js";
|
|
4
5
|
const secret = "a".repeat(48);
|
|
5
6
|
test("issueSession + verifySession — round-trips identity", () => {
|
|
@@ -12,6 +13,26 @@ test("issueSession + verifySession — round-trips identity", () => {
|
|
|
12
13
|
assert.equal(verified.sub, "alice");
|
|
13
14
|
assert.deepEqual(verified.roles, ["operator"]);
|
|
14
15
|
});
|
|
16
|
+
test("issueSession mints a unique sid that round-trips through verify", () => {
|
|
17
|
+
const now = 1_700_000_000;
|
|
18
|
+
const a = issueSession({ sub: "alice", name: "Alice" }, { secret }, now);
|
|
19
|
+
const b = issueSession({ sub: "alice", name: "Alice" }, { secret }, now);
|
|
20
|
+
assert.ok(a.payload.sid, "expected a sid");
|
|
21
|
+
assert.notEqual(a.payload.sid, b.payload.sid, "each session gets its own sid");
|
|
22
|
+
const verified = verifySession(a.cookie, { secret }, now + 1);
|
|
23
|
+
assert.ok(verified);
|
|
24
|
+
assert.equal(verified.sid, a.payload.sid);
|
|
25
|
+
});
|
|
26
|
+
test("verifySession — a legacy cookie without a sid still verifies", () => {
|
|
27
|
+
const now = 1_700_000_000;
|
|
28
|
+
// Hand-craft a payload lacking sid (pre-Q17 shape) and sign it.
|
|
29
|
+
const payload = { sub: "alice", name: "Alice", iat: now, exp: now + 3600 };
|
|
30
|
+
const payloadStr = Buffer.from(JSON.stringify(payload)).toString("base64url");
|
|
31
|
+
const sig = createHmac("sha256", secret).update(payloadStr).digest("base64url");
|
|
32
|
+
const verified = verifySession(`${payloadStr}.${sig}`, { secret }, now + 1);
|
|
33
|
+
assert.ok(verified, "legacy cookie should still verify");
|
|
34
|
+
assert.equal(verified.sid, undefined);
|
|
35
|
+
});
|
|
15
36
|
test("issueSession + verifySession — round-trips email when present", () => {
|
|
16
37
|
const now = 1_700_000_000;
|
|
17
38
|
const { cookie } = issueSession({ sub: "alice", name: "Alice", email: "alice@example.test", roles: ["operator"] }, { secret }, now);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { SignalType, ConnectorHealth, ServiceInfo, MetricInfo, MetricQuery, MetricResult, LogQuery, LogResult, TraceQuery, TraceResult, SourceConfig, MetricDefinition, Resource, Edge, TopologySnapshot, TopologyChangeListener } from "../types.js";
|
|
1
|
+
import type { SignalType, ConnectorHealth, ServiceInfo, MetricInfo, MetricQuery, MetricResult, LogQuery, LogResult, LogAggregateQuery, LogAggregateResult, TraceQuery, TraceResult, SourceConfig, MetricDefinition, Resource, Edge, TopologySnapshot, TopologyChangeListener } from "../types.js";
|
|
2
2
|
export interface ObservabilityConnector {
|
|
3
3
|
readonly name: string;
|
|
4
4
|
readonly type: string;
|
|
@@ -14,6 +14,10 @@ export interface ObservabilityConnector {
|
|
|
14
14
|
listAvailableMetrics?(service: string): Promise<MetricInfo[]>;
|
|
15
15
|
queryMetrics?(params: MetricQuery): Promise<MetricResult>;
|
|
16
16
|
queryLogs?(params: LogQuery): Promise<LogResult>;
|
|
17
|
+
/** Optional server-side log aggregation (count/sum/topk). Backends that
|
|
18
|
+
* can push aggregation down (Loki → LogQL metric queries) implement it;
|
|
19
|
+
* the query_logs tool routes here when an `aggregate` arg is present. */
|
|
20
|
+
queryLogAggregate?(params: LogAggregateQuery): Promise<LogAggregateResult>;
|
|
17
21
|
/** Optional traces capability — Tempo / Jaeger / OTLP backends
|
|
18
22
|
* implement this. The MCP `query_traces` tool fans out to every
|
|
19
23
|
* connector that has it. */
|
|
@@ -1,5 +1,48 @@
|
|
|
1
1
|
import type { ObservabilityConnector } from "./interface.js";
|
|
2
|
-
import type { SourceConfig, ConnectorHealth, ServiceInfo, MetricDefinition, LogQuery, LogResult, SignalType } from "../types.js";
|
|
2
|
+
import type { SourceConfig, ConnectorHealth, ServiceInfo, MetricDefinition, LogQuery, LogResult, LogAggregateQuery, LogAggregateResult, SignalType } from "../types.js";
|
|
3
|
+
/** Escape a value for a double-quoted LogQL string literal. Backslash and
|
|
4
|
+
* quote first (breakout chars), then control chars — a raw newline/tab in
|
|
5
|
+
* a Go-style `"..."` literal is a parse error, so emit the escape sequence. */
|
|
6
|
+
export declare function escapeLogQLValue(value: string): string;
|
|
7
|
+
/**
|
|
8
|
+
* Compile a `labels` equality map into LogQL label-filter expressions that
|
|
9
|
+
* run AFTER `| json`, e.g. `{...} | json | method="GET" | status="200"`.
|
|
10
|
+
* Placed after the json parse so fields the pipeline extracts (not just
|
|
11
|
+
* stream labels) are filterable. Keys are sorted for deterministic output.
|
|
12
|
+
* Reusable, side-effect-free unit — also the basis for the Q-LOG2
|
|
13
|
+
* aggregation path and a future named-LogQL catalog. Callers validate the
|
|
14
|
+
* map (see validateLogLabels); this only escapes values.
|
|
15
|
+
*/
|
|
16
|
+
export declare function logqlLabelFilters(labels: Record<string, string> | undefined): string;
|
|
17
|
+
/**
|
|
18
|
+
* Derive a log level from an HTTP status code when the line carries no
|
|
19
|
+
* explicit level: 5xx → error, 4xx → warn. Returns undefined otherwise so
|
|
20
|
+
* the caller keeps its existing fallback chain.
|
|
21
|
+
*/
|
|
22
|
+
export declare function levelFromStatus(status: unknown): "error" | "warn" | undefined;
|
|
23
|
+
/** Parse a `<n><m|h|d>` duration into seconds. Returns null when malformed. */
|
|
24
|
+
export declare function parseDurationSeconds(duration: string): number | null;
|
|
25
|
+
/** Pick a bucket size (seconds) that yields ~60 points across the window,
|
|
26
|
+
* floored at 60s, so a count_over_time range query isn't absurdly dense. */
|
|
27
|
+
export declare function defaultBucketSeconds(durationSeconds: number): number;
|
|
28
|
+
export interface AggregateLogQL {
|
|
29
|
+
logql: string;
|
|
30
|
+
/** instant (vector) for sum/topk; range (matrix) for count_over_time. */
|
|
31
|
+
mode: "instant" | "range";
|
|
32
|
+
/** Step for the range query, e.g. "300s". Only set when mode === "range". */
|
|
33
|
+
step?: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Wrap a stream+pipeline expression (`{sel} | json | …`) in a LogQL metric
|
|
37
|
+
* aggregation. Pure + side-effect-free so it's unit-testable without a
|
|
38
|
+
* backend. `by` labels are assumed pre-validated (label-name shape).
|
|
39
|
+
*/
|
|
40
|
+
export declare function buildAggregateLogQL(streamPipeline: string, agg: {
|
|
41
|
+
op: "count_over_time" | "sum" | "topk";
|
|
42
|
+
by?: string[];
|
|
43
|
+
k?: number;
|
|
44
|
+
step?: string;
|
|
45
|
+
}, duration: string): AggregateLogQL;
|
|
3
46
|
export declare class LokiConnector implements ObservabilityConnector {
|
|
4
47
|
readonly type = "loki";
|
|
5
48
|
readonly signalType: SignalType;
|
|
@@ -17,6 +60,7 @@ export declare class LokiConnector implements ObservabilityConnector {
|
|
|
17
60
|
disconnect(): Promise<void>;
|
|
18
61
|
listServices(): Promise<ServiceInfo[]>;
|
|
19
62
|
queryLogs(params: LogQuery): Promise<LogResult>;
|
|
63
|
+
queryLogAggregate(params: LogAggregateQuery): Promise<LogAggregateResult>;
|
|
20
64
|
private getLabelValues;
|
|
21
65
|
private resolveServiceSelector;
|
|
22
66
|
private parseLine;
|
package/dist/connectors/loki.js
CHANGED
|
@@ -1,6 +1,84 @@
|
|
|
1
1
|
import { buildTlsAgent } from "./tls.js";
|
|
2
2
|
const DEFAULT_SERVICE_LABELS = ["service_name", "service", "job", "app", "container"];
|
|
3
3
|
const LABEL_CACHE_TTL_MS = 60_000;
|
|
4
|
+
/** Escape a value for a double-quoted LogQL string literal. Backslash and
|
|
5
|
+
* quote first (breakout chars), then control chars — a raw newline/tab in
|
|
6
|
+
* a Go-style `"..."` literal is a parse error, so emit the escape sequence. */
|
|
7
|
+
export function escapeLogQLValue(value) {
|
|
8
|
+
return value
|
|
9
|
+
.replace(/\\/g, "\\\\")
|
|
10
|
+
.replace(/"/g, '\\"')
|
|
11
|
+
.replace(/\n/g, "\\n")
|
|
12
|
+
.replace(/\r/g, "\\r")
|
|
13
|
+
.replace(/\t/g, "\\t");
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Compile a `labels` equality map into LogQL label-filter expressions that
|
|
17
|
+
* run AFTER `| json`, e.g. `{...} | json | method="GET" | status="200"`.
|
|
18
|
+
* Placed after the json parse so fields the pipeline extracts (not just
|
|
19
|
+
* stream labels) are filterable. Keys are sorted for deterministic output.
|
|
20
|
+
* Reusable, side-effect-free unit — also the basis for the Q-LOG2
|
|
21
|
+
* aggregation path and a future named-LogQL catalog. Callers validate the
|
|
22
|
+
* map (see validateLogLabels); this only escapes values.
|
|
23
|
+
*/
|
|
24
|
+
export function logqlLabelFilters(labels) {
|
|
25
|
+
if (!labels)
|
|
26
|
+
return "";
|
|
27
|
+
return Object.keys(labels)
|
|
28
|
+
.sort()
|
|
29
|
+
.map((k) => ` | ${k}="${escapeLogQLValue(labels[k])}"`)
|
|
30
|
+
.join("");
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Derive a log level from an HTTP status code when the line carries no
|
|
34
|
+
* explicit level: 5xx → error, 4xx → warn. Returns undefined otherwise so
|
|
35
|
+
* the caller keeps its existing fallback chain.
|
|
36
|
+
*/
|
|
37
|
+
export function levelFromStatus(status) {
|
|
38
|
+
const n = typeof status === "number" ? status : parseInt(String(status ?? ""), 10);
|
|
39
|
+
if (!Number.isFinite(n))
|
|
40
|
+
return undefined;
|
|
41
|
+
if (n >= 500 && n <= 599)
|
|
42
|
+
return "error";
|
|
43
|
+
if (n >= 400 && n <= 499)
|
|
44
|
+
return "warn";
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
/** Parse a `<n><m|h|d>` duration into seconds. Returns null when malformed. */
|
|
48
|
+
export function parseDurationSeconds(duration) {
|
|
49
|
+
const m = /^(\d+)([mhd])$/.exec(duration);
|
|
50
|
+
if (!m)
|
|
51
|
+
return null;
|
|
52
|
+
const v = parseInt(m[1], 10);
|
|
53
|
+
return m[2] === "m" ? v * 60 : m[2] === "h" ? v * 3600 : v * 86400;
|
|
54
|
+
}
|
|
55
|
+
/** Pick a bucket size (seconds) that yields ~60 points across the window,
|
|
56
|
+
* floored at 60s, so a count_over_time range query isn't absurdly dense. */
|
|
57
|
+
export function defaultBucketSeconds(durationSeconds) {
|
|
58
|
+
return Math.max(60, Math.floor(durationSeconds / 60));
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Wrap a stream+pipeline expression (`{sel} | json | …`) in a LogQL metric
|
|
62
|
+
* aggregation. Pure + side-effect-free so it's unit-testable without a
|
|
63
|
+
* backend. `by` labels are assumed pre-validated (label-name shape).
|
|
64
|
+
*/
|
|
65
|
+
export function buildAggregateLogQL(streamPipeline, agg, duration) {
|
|
66
|
+
const durSec = parseDurationSeconds(duration) ?? 3600;
|
|
67
|
+
const byClause = agg.by && agg.by.length ? ` by (${agg.by.join(", ")})` : "";
|
|
68
|
+
if (agg.op === "count_over_time") {
|
|
69
|
+
const stepSec = (agg.step && parseDurationSeconds(agg.step)) || defaultBucketSeconds(durSec);
|
|
70
|
+
const inner = `count_over_time(${streamPipeline} [${stepSec}s])`;
|
|
71
|
+
const logql = byClause ? `sum${byClause} (${inner})` : inner;
|
|
72
|
+
return { logql, mode: "range", step: `${stepSec}s` };
|
|
73
|
+
}
|
|
74
|
+
// sum / topk: count over the whole window, then aggregate → instant vector.
|
|
75
|
+
const totals = `sum${byClause} (count_over_time(${streamPipeline} [${durSec}s]))`;
|
|
76
|
+
if (agg.op === "topk") {
|
|
77
|
+
const k = agg.k && agg.k > 0 ? Math.floor(agg.k) : 10;
|
|
78
|
+
return { logql: `topk(${k}, ${totals})`, mode: "instant" };
|
|
79
|
+
}
|
|
80
|
+
return { logql: totals, mode: "instant" };
|
|
81
|
+
}
|
|
4
82
|
export class LokiConnector {
|
|
5
83
|
type = "loki";
|
|
6
84
|
signalType = "logs";
|
|
@@ -94,14 +172,13 @@ export class LokiConnector {
|
|
|
94
172
|
// matches the real stream.
|
|
95
173
|
const { label: matchedLabel, value: rawValue } = await this.resolveServiceSelector(params.service);
|
|
96
174
|
const service = this.escapeLogQLValue(rawValue);
|
|
97
|
-
let logql = `{${matchedLabel}="${service}"}`;
|
|
175
|
+
let logql = `{${matchedLabel}="${service}"} | json`;
|
|
98
176
|
if (params.level) {
|
|
99
|
-
|
|
100
|
-
logql += ` | json | level="${level}"`;
|
|
101
|
-
}
|
|
102
|
-
else {
|
|
103
|
-
logql += ` | json`;
|
|
177
|
+
logql += ` | level="${this.escapeLogQLValue(params.level)}"`;
|
|
104
178
|
}
|
|
179
|
+
// Structured equality filters (method/status/url/environment/…) — run
|
|
180
|
+
// after `| json` so backend-extracted fields are selectable.
|
|
181
|
+
logql += logqlLabelFilters(params.labels);
|
|
105
182
|
if (params.query) {
|
|
106
183
|
const query = this.escapeLogQLRegex(params.query);
|
|
107
184
|
logql += ` |~ \`${query}\``;
|
|
@@ -114,9 +191,16 @@ export class LokiConnector {
|
|
|
114
191
|
const labels = stream.stream;
|
|
115
192
|
for (const [ts, line] of stream.values) {
|
|
116
193
|
const parsed = this.parseLine(line);
|
|
194
|
+
// Prefer an explicit level; otherwise derive one from an HTTP
|
|
195
|
+
// status field (5xx→error, 4xx→warn) so structured access logs
|
|
196
|
+
// that carry `status` but no `level` are still filterable/triaged.
|
|
197
|
+
const level = parsed.level ||
|
|
198
|
+
labels.level ||
|
|
199
|
+
levelFromStatus(parsed.status ?? labels.status) ||
|
|
200
|
+
"unknown";
|
|
117
201
|
entries.push({
|
|
118
202
|
timestamp: new Date(parseInt(ts) / 1_000_000).toISOString(),
|
|
119
|
-
level
|
|
203
|
+
level,
|
|
120
204
|
message: parsed.msg || line,
|
|
121
205
|
labels,
|
|
122
206
|
});
|
|
@@ -140,6 +224,54 @@ export class LokiConnector {
|
|
|
140
224
|
},
|
|
141
225
|
};
|
|
142
226
|
}
|
|
227
|
+
async queryLogAggregate(params) {
|
|
228
|
+
const { start, end } = this.parseTimeRange(params.duration);
|
|
229
|
+
const { label: matchedLabel, value: rawValue } = await this.resolveServiceSelector(params.service);
|
|
230
|
+
const service = this.escapeLogQLValue(rawValue);
|
|
231
|
+
// Same stream + pipeline prefix as queryLogs (reuses the Q-LOG1 unit),
|
|
232
|
+
// minus the level filter (aggregation groups, it doesn't level-filter).
|
|
233
|
+
let pipeline = `{${matchedLabel}="${service}"} | json`;
|
|
234
|
+
pipeline += logqlLabelFilters(params.labels);
|
|
235
|
+
if (params.query) {
|
|
236
|
+
pipeline += ` |~ \`${this.escapeLogQLRegex(params.query)}\``;
|
|
237
|
+
}
|
|
238
|
+
const { logql, mode, step } = buildAggregateLogQL(pipeline, { op: params.op, by: params.by, k: params.k, step: params.step }, params.duration);
|
|
239
|
+
const by = params.by ?? [];
|
|
240
|
+
const series = [];
|
|
241
|
+
if (mode === "instant") {
|
|
242
|
+
const url = `/loki/api/v1/query?query=${encodeURIComponent(logql)}&time=${end}000000000`;
|
|
243
|
+
const data = await this.apiGet(url);
|
|
244
|
+
for (const r of data?.data?.result || []) {
|
|
245
|
+
const v = Array.isArray(r.value) ? Number(r.value[1]) : NaN;
|
|
246
|
+
series.push({ labels: r.metric || {}, value: Number.isFinite(v) ? v : 0 });
|
|
247
|
+
}
|
|
248
|
+
// topk is already ordered by Loki; sort sum desc for a stable, useful view.
|
|
249
|
+
series.sort((a, b) => (b.value ?? 0) - (a.value ?? 0));
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
// `step` from the builder is `<n>s`; the query_range step param wants seconds.
|
|
253
|
+
const stepSec = step ? parseInt(step, 10) || 60 : 60;
|
|
254
|
+
const url = `/loki/api/v1/query_range?query=${encodeURIComponent(logql)}` +
|
|
255
|
+
`&start=${start}000000000&end=${end}000000000&step=${stepSec}`;
|
|
256
|
+
const data = await this.apiGet(url);
|
|
257
|
+
for (const r of data?.data?.result || []) {
|
|
258
|
+
const points = (r.values || []).map(([ts, val]) => ({
|
|
259
|
+
t: Math.round(Number(ts) * 1000),
|
|
260
|
+
value: Number(val),
|
|
261
|
+
}));
|
|
262
|
+
series.push({ labels: r.metric || {}, points });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return {
|
|
266
|
+
source: this.name,
|
|
267
|
+
op: params.op,
|
|
268
|
+
by,
|
|
269
|
+
step: mode === "range" ? step : undefined,
|
|
270
|
+
mode,
|
|
271
|
+
series,
|
|
272
|
+
note: "Aggregate mode: `limit` does not apply (results are grouped counts, not raw rows).",
|
|
273
|
+
};
|
|
274
|
+
}
|
|
143
275
|
// --- Private helpers ---
|
|
144
276
|
async getLabelValues(label) {
|
|
145
277
|
const cached = this.labelValuesCache.get(label);
|
|
@@ -204,7 +336,8 @@ export class LokiConnector {
|
|
|
204
336
|
return { start: now - seconds, end: now };
|
|
205
337
|
}
|
|
206
338
|
escapeLogQLValue(value) {
|
|
207
|
-
|
|
339
|
+
// Delegate to the canonical module-level escaper (single source of truth).
|
|
340
|
+
return escapeLogQLValue(value);
|
|
208
341
|
}
|
|
209
342
|
escapeLogQLRegex(value) {
|
|
210
343
|
// Escape backslash first (so we don't double-escape sequences we add),
|
|
@@ -1,7 +1,177 @@
|
|
|
1
1
|
import { describe, it } from "node:test";
|
|
2
2
|
import assert from "node:assert/strict";
|
|
3
|
-
import { LokiConnector } from "./loki.js";
|
|
3
|
+
import { LokiConnector, logqlLabelFilters, levelFromStatus, escapeLogQLValue, buildAggregateLogQL, parseDurationSeconds, defaultBucketSeconds, } from "./loki.js";
|
|
4
4
|
const proto = LokiConnector.prototype;
|
|
5
|
+
function jsonRes(obj) {
|
|
6
|
+
return { ok: true, status: 200, statusText: "OK", json: async () => obj, text: async () => "" };
|
|
7
|
+
}
|
|
8
|
+
describe("Q-LOG1: logqlLabelFilters", () => {
|
|
9
|
+
it("returns empty string for undefined / empty", () => {
|
|
10
|
+
assert.equal(logqlLabelFilters(undefined), "");
|
|
11
|
+
assert.equal(logqlLabelFilters({}), "");
|
|
12
|
+
});
|
|
13
|
+
it("compiles a single filter", () => {
|
|
14
|
+
assert.equal(logqlLabelFilters({ method: "GET" }), ' | method="GET"');
|
|
15
|
+
});
|
|
16
|
+
it("compiles multiple filters, keys sorted for determinism", () => {
|
|
17
|
+
assert.equal(logqlLabelFilters({ status: "200", method: "GET", url: "/" }), ' | method="GET" | status="200" | url="/"');
|
|
18
|
+
});
|
|
19
|
+
it("escapes double quotes and backslashes in values", () => {
|
|
20
|
+
assert.equal(logqlLabelFilters({ path: 'a"b\\c' }), ' | path="a\\"b\\\\c"');
|
|
21
|
+
});
|
|
22
|
+
});
|
|
23
|
+
describe("Q-LOG1: levelFromStatus", () => {
|
|
24
|
+
it("maps 5xx → error", () => {
|
|
25
|
+
assert.equal(levelFromStatus(500), "error");
|
|
26
|
+
assert.equal(levelFromStatus("503"), "error");
|
|
27
|
+
assert.equal(levelFromStatus(599), "error");
|
|
28
|
+
});
|
|
29
|
+
it("maps 4xx → warn", () => {
|
|
30
|
+
assert.equal(levelFromStatus(404), "warn");
|
|
31
|
+
assert.equal(levelFromStatus("400"), "warn");
|
|
32
|
+
});
|
|
33
|
+
it("returns undefined for 2xx/3xx and non-numeric", () => {
|
|
34
|
+
assert.equal(levelFromStatus(200), undefined);
|
|
35
|
+
assert.equal(levelFromStatus(301), undefined);
|
|
36
|
+
assert.equal(levelFromStatus("abc"), undefined);
|
|
37
|
+
assert.equal(levelFromStatus(undefined), undefined);
|
|
38
|
+
});
|
|
39
|
+
});
|
|
40
|
+
describe("Q-LOG1: escapeLogQLValue", () => {
|
|
41
|
+
it("escapes backslash then quote", () => {
|
|
42
|
+
assert.equal(escapeLogQLValue('he said "hi"\\'), 'he said \\"hi\\"\\\\');
|
|
43
|
+
});
|
|
44
|
+
it("escapes control chars (newline/return/tab) into LogQL escape sequences", () => {
|
|
45
|
+
assert.equal(escapeLogQLValue("a\nb\rc\td"), "a\\nb\\rc\\td");
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
describe("Q-LOG1: queryLogs LogQL assembly", () => {
|
|
49
|
+
async function captureQuery(params) {
|
|
50
|
+
const conn = new LokiConnector();
|
|
51
|
+
await conn.connect({ name: "loki", type: "loki", url: "http://loki:3100", enabled: true });
|
|
52
|
+
let captured = "";
|
|
53
|
+
const orig = globalThis.fetch;
|
|
54
|
+
globalThis.fetch = (async (url) => {
|
|
55
|
+
const u = String(url);
|
|
56
|
+
if (u.includes("/label/") && u.includes("/values"))
|
|
57
|
+
return jsonRes({ data: ["payment"] });
|
|
58
|
+
if (u.includes("/query_range")) {
|
|
59
|
+
captured = decodeURIComponent((u.match(/query=([^&]+)/) || [])[1] || "");
|
|
60
|
+
return jsonRes({ data: { result: [] } });
|
|
61
|
+
}
|
|
62
|
+
return jsonRes({ data: [] });
|
|
63
|
+
});
|
|
64
|
+
try {
|
|
65
|
+
await conn.queryLogs({ service: "payment", duration: "5m", ...params });
|
|
66
|
+
}
|
|
67
|
+
finally {
|
|
68
|
+
globalThis.fetch = orig;
|
|
69
|
+
}
|
|
70
|
+
return captured;
|
|
71
|
+
}
|
|
72
|
+
it("AND's label filters after | json, with level and line filter", async () => {
|
|
73
|
+
const q = await captureQuery({ level: "error", labels: { method: "GET", status: "200" }, query: "timeout" });
|
|
74
|
+
assert.equal(q, '{service_name="payment"} | json | level="error" | method="GET" | status="200" |~ `timeout`');
|
|
75
|
+
});
|
|
76
|
+
it("works with labels only (no level/query)", async () => {
|
|
77
|
+
const q = await captureQuery({ labels: { environment: "prod" } });
|
|
78
|
+
assert.equal(q, '{service_name="payment"} | json | environment="prod"');
|
|
79
|
+
});
|
|
80
|
+
it("plain query (no labels) is unchanged from prior behaviour", async () => {
|
|
81
|
+
const q = await captureQuery({});
|
|
82
|
+
assert.equal(q, '{service_name="payment"} | json');
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
describe("Q-LOG2: parseDurationSeconds / defaultBucketSeconds", () => {
|
|
86
|
+
it("parses m/h/d", () => {
|
|
87
|
+
assert.equal(parseDurationSeconds("5m"), 300);
|
|
88
|
+
assert.equal(parseDurationSeconds("2h"), 7200);
|
|
89
|
+
assert.equal(parseDurationSeconds("1d"), 86400);
|
|
90
|
+
assert.equal(parseDurationSeconds("bad"), null);
|
|
91
|
+
});
|
|
92
|
+
it("buckets to ~60 points, floored at 60s", () => {
|
|
93
|
+
assert.equal(defaultBucketSeconds(3600), 60); // 1h → 60s
|
|
94
|
+
assert.equal(defaultBucketSeconds(86400), 1440); // 24h → 1440s
|
|
95
|
+
assert.equal(defaultBucketSeconds(60), 60); // tiny window floors at 60s
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
describe("Q-LOG2: buildAggregateLogQL", () => {
|
|
99
|
+
const PIPE = '{service_name="app"} | json | method="GET"';
|
|
100
|
+
it("count_over_time with by → sum by + range mode + step", () => {
|
|
101
|
+
const r = buildAggregateLogQL(PIPE, { op: "count_over_time", by: ["url"], step: "15m" }, "1h");
|
|
102
|
+
assert.equal(r.mode, "range");
|
|
103
|
+
assert.equal(r.step, "900s");
|
|
104
|
+
assert.equal(r.logql, `sum by (url) (count_over_time(${PIPE} [900s]))`);
|
|
105
|
+
});
|
|
106
|
+
it("count_over_time without by → bare count_over_time, default step", () => {
|
|
107
|
+
const r = buildAggregateLogQL(PIPE, { op: "count_over_time" }, "1h");
|
|
108
|
+
assert.equal(r.mode, "range");
|
|
109
|
+
assert.equal(r.step, "60s");
|
|
110
|
+
assert.equal(r.logql, `count_over_time(${PIPE} [60s])`);
|
|
111
|
+
});
|
|
112
|
+
it("sum → instant total per group over the whole window", () => {
|
|
113
|
+
const r = buildAggregateLogQL(PIPE, { op: "sum", by: ["status"] }, "1h");
|
|
114
|
+
assert.equal(r.mode, "instant");
|
|
115
|
+
assert.equal(r.logql, `sum by (status) (count_over_time(${PIPE} [3600s]))`);
|
|
116
|
+
});
|
|
117
|
+
it("topk → instant topk(k, sum by) with default k=10", () => {
|
|
118
|
+
const r = buildAggregateLogQL(PIPE, { op: "topk", by: ["url"] }, "1h");
|
|
119
|
+
assert.equal(r.mode, "instant");
|
|
120
|
+
assert.equal(r.logql, `topk(10, sum by (url) (count_over_time(${PIPE} [3600s])))`);
|
|
121
|
+
});
|
|
122
|
+
it("topk honours explicit k", () => {
|
|
123
|
+
const r = buildAggregateLogQL(PIPE, { op: "topk", by: ["url"], k: 3 }, "30m");
|
|
124
|
+
assert.equal(r.logql, `topk(3, sum by (url) (count_over_time(${PIPE} [1800s])))`);
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
describe("Q-LOG2: queryLogAggregate", () => {
|
|
128
|
+
async function run(agg) {
|
|
129
|
+
const conn = new LokiConnector();
|
|
130
|
+
await conn.connect({ name: "loki", type: "loki", url: "http://loki:3100", enabled: true });
|
|
131
|
+
let capturedUrl = "";
|
|
132
|
+
const orig = globalThis.fetch;
|
|
133
|
+
globalThis.fetch = (async (url) => {
|
|
134
|
+
const u = String(url);
|
|
135
|
+
if (u.includes("/label/") && u.includes("/values"))
|
|
136
|
+
return jsonRes({ data: ["app"] });
|
|
137
|
+
if (u.includes("/query_range")) {
|
|
138
|
+
capturedUrl = u;
|
|
139
|
+
return jsonRes({ data: { resultType: "matrix", result: [
|
|
140
|
+
{ metric: { url: "/" }, values: [[1000, "3"], [1060, "5"]] },
|
|
141
|
+
] } });
|
|
142
|
+
}
|
|
143
|
+
if (u.includes("/query")) {
|
|
144
|
+
capturedUrl = u;
|
|
145
|
+
return jsonRes({ data: { resultType: "vector", result: [
|
|
146
|
+
{ metric: { url: "/a" }, value: [2000, "7"] },
|
|
147
|
+
{ metric: { url: "/b" }, value: [2000, "12"] },
|
|
148
|
+
] } });
|
|
149
|
+
}
|
|
150
|
+
return jsonRes({ data: [] });
|
|
151
|
+
});
|
|
152
|
+
try {
|
|
153
|
+
return await conn.queryLogAggregate({ service: "app", duration: "1h", ...agg });
|
|
154
|
+
}
|
|
155
|
+
finally {
|
|
156
|
+
globalThis.fetch = orig;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
it("topk → instant vector parsed + sorted desc, note set", async () => {
|
|
160
|
+
const res = await run({ op: "topk", by: ["url"], k: 2 });
|
|
161
|
+
assert.equal(res.mode, "instant");
|
|
162
|
+
assert.equal(res.op, "topk");
|
|
163
|
+
assert.deepEqual(res.by, ["url"]);
|
|
164
|
+
assert.deepEqual(res.series.map((s) => [s.labels.url, s.value]), [["/b", 12], ["/a", 7]]);
|
|
165
|
+
assert.match(res.note, /limit/);
|
|
166
|
+
});
|
|
167
|
+
it("count_over_time → range matrix parsed into points", async () => {
|
|
168
|
+
const res = await run({ op: "count_over_time", by: ["url"], step: "1m" });
|
|
169
|
+
assert.equal(res.mode, "range");
|
|
170
|
+
assert.equal(res.step, "60s");
|
|
171
|
+
assert.equal(res.series.length, 1);
|
|
172
|
+
assert.deepEqual(res.series[0].points, [{ t: 1000000, value: 3 }, { t: 1060000, value: 5 }]);
|
|
173
|
+
});
|
|
174
|
+
});
|
|
5
175
|
describe("LokiConnector", () => {
|
|
6
176
|
describe("parseLine", () => {
|
|
7
177
|
it("parses valid JSON", () => {
|