@camstack/system 1.2.456 → 1.2.458

package/dist/builtins/loki-logging/index.mjs

@@ -1,4 +1,6 @@
-import { $r as BaseAddon, In as logDestinationCapability } from "../../dist-BSF_C8W2.mjs";
+import { ni as BaseAddon, zn as logDestinationCapability } from "../../dist-B_GRPhnY.mjs";
+import { n as compileLogFilter, o as lineMatches } from "../../log-read-filter-D7Ev3zm-.mjs";
+import { z } from "zod";
 //#region src/builtins/loki-logging/loki-payload.ts
 /**
 * Loki rejects a label name that is not a valid Prometheus label
@@ -36,6 +38,7 @@ function toUnixNano(timestamp) {
 */
 function formatLokiLine(entry) {
 	const line = { message: entry.message };
+	if (entry.id !== void 0) line["id"] = entry.id;
 	if (entry.scope !== void 0 && entry.scope.length > 0) line["scope"] = entry.scope;
 	if (entry.tags !== void 0 && Object.keys(entry.tags).length > 0) line["tags"] = entry.tags;
 	if (entry.meta !== void 0 && Object.keys(entry.meta).length > 0) line["meta"] = entry.meta;
@@ -86,6 +89,231 @@ function buildPushBody(entries, labels) {
 	return { streams };
 }
 //#endregion
+//#region src/builtins/loki-logging/loki-read.ts
+/**
+* Reading history back from Loki (D898): `GET /loki/api/v1/query_range`,
+* newest first.
+*
+* The label selector narrows by what IS a label (`job`, `node`, `level` —
+* see `loki-payload.ts` for why nothing else is); line filters (`|=`, `|~`)
+* narrow by device, addon and text on the JSON body; the shared
+* `lineMatches` then decides exactly, so a line filter only has to be a
+* superset.
+*
+* The cursor is the nanosecond timestamp of the oldest line returned plus a
+* short hash of each line returned AT that nanosecond: entries share
+* timestamps (they are millisecond-precise), so "older than t" would drop the
+* siblings a page cut through, and "t or older" would repeat them. The next
+* page asks for `end = t + 1` (Loki's end is exclusive) and skips the hashes.
+*/
+var LEVELS = [
+	"debug",
+	"info",
+	"warn",
+	"error"
+];
+/** One query_range window. A busy stream fills `limit` long before this. */
+var LOKI_WINDOW_MS = 360 * 60 * 1e3;
+var MS_NS = 1000000n;
+var CURSOR_PREFIX = "l1.";
+var LokiCursorSchema = z.object({
+	/** Nanosecond epoch, decimal string. */
+	n: z.string().regex(/^\d{1,22}$/),
+	/** Hashes of the lines already returned at exactly `n`. */
+	h: z.array(z.string().max(16)).max(1e3)
+});
+function encodeLokiCursor(cursor) {
+	return `${CURSOR_PREFIX}${Buffer.from(JSON.stringify(cursor)).toString("base64url")}`;
+}
+function decodeLokiCursor(raw) {
+	if (!raw.startsWith(CURSOR_PREFIX)) return null;
+	try {
+		const parsed = LokiCursorSchema.safeParse(JSON.parse(Buffer.from(raw.slice(3), "base64url").toString("utf8")));
+		return parsed.success ? parsed.data : null;
+	} catch {
+		return null;
+	}
+}
+/** FNV-1a, 32 bit, hex — a dedupe key, not a security property. */
+function lineHash(line) {
+	let h = 2166136261;
+	for (let i = 0; i < line.length; i += 1) {
+		h ^= line.charCodeAt(i);
+		h = Math.imul(h, 16777619);
+	}
+	return (h >>> 0).toString(16);
+}
+/** A LogQL double-quoted string literal. */
+function quote(value) {
+	return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"")}"`;
+}
+function escapeRegex(value) {
+	return value.replaceAll(/[\\.+*?()|[\]{}^$]/g, (c) => `\\${c}`);
+}
+/** The LogQL for a read. Pure. */
+function buildLogQl(query, labels) {
+	const selectors = [`job=${quote(labels.job)}`];
+	if (query.fromNodeId !== void 0) selectors.push(`node=${quote(query.fromNodeId)}`);
+	if (query.level !== void 0 && query.level !== "debug") {
+		const floor = LEVELS.indexOf(query.level);
+		selectors.push(`level=~${quote(LEVELS.slice(floor).join("|"))}`);
+	}
+	let logql = `{${selectors.join(",")}}`;
+	if (query.deviceId !== void 0) logql += ` |= ${quote(`"deviceId":${String(query.deviceId)}`)}`;
+	if (query.fromAddonId !== void 0) {
+		const bare = query.fromAddonId.split("@")[0] ?? query.fromAddonId;
+		logql += ` |= ${quote(`"addonId":"${bare}`)}`;
+	}
+	const text = query.text?.trim();
+	if (text !== void 0 && text.length > 0) logql += ` |~ ${quote(`(?i)${escapeRegex(text)}`)}`;
+	return logql;
+}
+var LokiResponseSchema = z.object({ data: z.object({ result: z.array(z.object({
+	stream: z.record(z.string(), z.string()),
+	values: z.array(z.tuple([z.string(), z.string()]))
+})) }) });
+var LokiLineSchema = z.object({
+	id: z.string().optional(),
+	message: z.string(),
+	scope: z.string().optional(),
+	tags: z.record(z.string(), z.unknown()).optional(),
+	meta: z.record(z.string(), z.unknown()).optional()
+});
+function levelOf(label) {
+	return label === "debug" || label === "warn" || label === "error" ? label : "info";
+}
+/** A query_range body → rows, newest first. Lines that are not ours are dropped. */
+function parseQueryRange(body) {
+	const parsed = LokiResponseSchema.safeParse(body);
+	if (!parsed.success) throw new Error("Loki answered with an unexpected query_range shape");
+	const rows = [];
+	for (const stream of parsed.data.data.result) {
+		const level = levelOf(stream.stream["level"]);
+		for (const [nsText, text] of stream.values) {
+			let json;
+			try {
+				json = JSON.parse(text);
+			} catch {
+				continue;
+			}
+			const line = LokiLineSchema.safeParse(json);
+			if (!line.success) continue;
+			const ns = BigInt(nsText);
+			const timestamp = Number(ns / MS_NS);
+			const tags = {};
+			for (const [k, v] of Object.entries(line.data.tags ?? {})) if (typeof v === "string" || typeof v === "number") tags[k] = v;
+			const hasTags = Object.keys(tags).length > 0;
+			const entry = {
+				...line.data.id !== void 0 ? { id: line.data.id } : {},
+				timestamp,
+				level,
+				...line.data.scope !== void 0 ? { scope: line.data.scope } : {},
+				message: line.data.message,
+				...hasTags ? { tags } : {},
+				...line.data.meta !== void 0 ? { meta: line.data.meta } : {}
+			};
+			rows.push({
+				ns,
+				hash: lineHash(text),
+				entry,
+				line: {
+					timestamp,
+					level,
+					message: line.data.message,
+					scope: line.data.scope,
+					tags: hasTags ? tags : void 0,
+					meta: line.data.meta
+				}
+			});
+		}
+	}
+	return rows.toSorted((a, b) => a.ns < b.ns ? 1 : a.ns > b.ns ? -1 : 0);
+}
+/**
+* One backward slice over Loki: windows of {@link LOKI_WINDOW_MS}, newest
+* first, until `limit` matches, the horizon, the budget or a cancel.
+*/
+async function readLokiBackward(args) {
+	const { query } = args;
+	const now = args.now ?? Date.now;
+	const filter = compileLogFilter(query);
+	const limit = query.limit ?? 200;
+	const deadline = now() + Math.min(query.budgetMs ?? 2e3, 1e4);
+	const logql = buildLogQl(query, args.labels);
+	const horizonMs = query.since ?? now() - 30 * 864e5;
+	const horizonNs = BigInt(Math.trunc(horizonMs)) * MS_NS;
+	const cursor = query.before !== void 0 ? decodeLokiCursor(query.before) : null;
+	const untilNs = query.until !== void 0 ? BigInt(Math.trunc(query.until) + 1) * MS_NS : null;
+	let endNs = cursor !== null ? BigInt(cursor.n) + 1n : untilNs ?? BigInt(now() + 1) * MS_NS;
+	let seenAtBoundary = new Set(cursor?.h ?? []);
+	let boundaryNs = cursor !== null ? BigInt(cursor.n) : null;
+	const entries = [];
+	let bytes = 0;
+	while (endNs > horizonNs) {
+		if (args.isCancelled()) return {
+			entries,
+			nextBefore: encodeLokiCursor({
+				n: String(endNs - 1n),
+				h: [...seenAtBoundary]
+			}),
+			nextAfter: null,
+			scannedBytes: bytes,
+			done: false,
+			cancelled: true
+		};
+		const windowStart = endNs - BigInt(LOKI_WINDOW_MS) * MS_NS;
+		const startNs = windowStart > horizonNs ? windowStart : horizonNs;
+		const want = limit - entries.length + seenAtBoundary.size;
+		const { rows, bytes: got } = await args.range({
+			logql,
+			startNs,
+			endNs,
+			limit: want
+		});
+		bytes += got;
+		let oldest = null;
+		for (const row of rows) {
+			if (boundaryNs !== null && row.ns === boundaryNs && seenAtBoundary.has(row.hash)) continue;
+			if (oldest === null || row.ns < oldest) {
+				if (oldest !== null && row.ns < oldest) seenAtBoundary = /* @__PURE__ */ new Set();
+				oldest = row.ns;
+			}
+			if (row.ns === oldest) seenAtBoundary.add(row.hash);
+			boundaryNs = oldest;
+			if (!lineMatches(row.line, filter)) continue;
+			entries.push(row.entry);
+			if (entries.length >= limit) return {
+				entries,
+				nextBefore: encodeLokiCursor({
+					n: String(row.ns),
+					h: [...seenAtBoundary]
+				}),
+				nextAfter: null,
+				scannedBytes: bytes,
+				done: false
+			};
+		}
+		if (rows.length >= want && oldest !== null) endNs = oldest + 1n;
+		else {
+			endNs = startNs;
+			seenAtBoundary = /* @__PURE__ */ new Set();
+			boundaryNs = null;
+		}
+		if (now() >= deadline) break;
+	}
+	const done = endNs <= horizonNs;
+	return {
+		entries,
+		nextBefore: done ? null : encodeLokiCursor({
+			n: String(endNs - 1n),
+			h: [...seenAtBoundary]
+		}),
+		nextAfter: null,
+		scannedBytes: bytes,
+		done
+	};
+}
+//#endregion
 //#region src/builtins/loki-logging/loki-destination.ts
 var LEVEL_ORDER = {
 	debug: 0,
@@ -109,6 +337,8 @@ var LokiDestination = class {
 	failures = 0;
 	lastError = null;
 	lastPushAtMs = null;
+	/** In-flight history reads, by requestId, so `cancelRead` can abort the fetch. */
+	reads = /* @__PURE__ */ new Map();
 	fetchFn;
 	setIntervalFn;
 	clearIntervalFn;
@@ -116,7 +346,7 @@ var LokiDestination = class {
 		this.fetchFn = deps?.fetchFn ?? ((url, init) => globalThis.fetch(url, {
 			method: init.method,
 			headers: { ...init.headers },
-			body: init.body,
+			...init.body !== void 0 ? { body: init.body } : {},
 			...init.signal ? { signal: init.signal } : {}
 		}));
 		this.setIntervalFn = deps?.setIntervalFn ?? setInterval;
@@ -208,9 +438,10 @@ var LokiDestination = class {
 	}
 	async push(body, config) {
 		const endpoint = `${config.url.replace(/\/+$/, "")}/loki/api/v1/push`;
-		const headers = { "content-type": "application/json" };
-		if (config.tenantId.trim().length > 0) headers["X-Scope-OrgID"] = config.tenantId.trim();
-		if (config.username.length > 0) headers["authorization"] = `Basic ${Buffer.from(`${config.username}:${config.password}`).toString("base64")}`;
+		const headers = {
+			"content-type": "application/json",
+			...this.authHeaders(config)
+		};
 		const controller = new AbortController();
 		const timeout = setTimeout(() => controller.abort(), Math.max(1e3, config.timeoutMs));
 		try {
@@ -278,8 +509,93 @@ var LokiDestination = class {
 			};
 		}
 	}
-	async query(_filter) {
-		return [];
+	/** Headers every call to this Loki carries: tenant + basic auth. */
+	authHeaders(config) {
+		const headers = {};
+		if (config.tenantId.trim().length > 0) headers["X-Scope-OrgID"] = config.tenantId.trim();
+		if (config.username.length > 0) headers["authorization"] = `Basic ${Buffer.from(`${config.username}:${config.password}`).toString("base64")}`;
+		return headers;
+	}
+	/**
+	* Readable when it is configured (D898): the same URL and credentials the
+	* push uses answer `query_range`. Labels are `job`/`node`/`level`, so those
+	* filter at the index; device, addon and text are line filters.
+	*/
+	async describeRead() {
+		const configured = this.isActive();
+		return {
+			readable: configured,
+			reason: configured ? null : "Loki is not enabled or has no URL",
+			filters: {
+				deviceId: true,
+				addonId: true,
+				nodeId: true,
+				tags: true,
+				level: true,
+				text: true,
+				timeRange: true
+			},
+			retentionDays: null
+		};
+	}
+	/** One backward slice from `query_range`, newest first. Never logs. */
+	async read(query) {
+		const config = this.config;
+		if (config === null || !this.isActive()) throw new Error("Loki is not enabled or has no URL");
+		const controller = new AbortController();
+		const requestId = query.requestId;
+		if (requestId !== void 0) this.reads.set(requestId, controller);
+		const base = config.url.replace(/\/+$/, "");
+		const range = async ({ logql, startNs, endNs, limit }) => {
+			const params = new URLSearchParams({
+				query: logql,
+				start: String(startNs),
+				end: String(endNs),
+				limit: String(limit),
+				direction: "backward"
+			});
+			const timeout = setTimeout(() => controller.abort(), Math.max(1e3, config.timeoutMs));
+			try {
+				const response = await this.fetchFn(`${base}/loki/api/v1/query_range?${params.toString()}`, {
+					method: "GET",
+					headers: this.authHeaders(config),
+					signal: controller.signal
+				});
+				const text = await response.text();
+				if (!response.ok) throw new Error(`Loki query failed: ${String(response.status)}${text.length > 0 ? ` — ${text.slice(0, 200)}` : ""}`);
+				return {
+					rows: parseQueryRange(JSON.parse(text)),
+					bytes: Buffer.byteLength(text)
+				};
+			} finally {
+				clearTimeout(timeout);
+			}
+		};
+		try {
+			return await readLokiBackward({
+				query,
+				labels: { job: config.job },
+				range,
+				isCancelled: () => controller.signal.aborted
+			});
+		} catch (error) {
+			if (controller.signal.aborted) return {
+				entries: [],
+				nextBefore: query.before ?? null,
+				nextAfter: null,
+				scannedBytes: 0,
+				done: false,
+				cancelled: true
+			};
+			throw error;
+		} finally {
+			if (requestId !== void 0) this.reads.delete(requestId);
+		}
+	}
+	async cancelRead(input) {
+		const controller = this.reads.get(input.requestId);
+		controller?.abort();
+		return { cancelled: controller !== void 0 };
 	}
 	async shutdown() {
 		this.stopTimer();

package/dist/builtins/loki-logging/loki-destination.d.ts

@@ -24,7 +24,7 @@
 * `MAX_QUEUE` and drops OLDEST-first when full (a burst during an incident is
 * exactly when the newest lines matter most). Drops are counted and surfaced.
 */
-import type { ILogDestination, LogEntry, LogFilter, LogLevel } from "@camstack/types";
+import type { ILogDestination, LogEntry, LogLevel, LogReadCapabilities, LogReadQuery, LogReadResult } from "@camstack/types";
 export interface LokiConfig {
 	readonly enabled: boolean;
 	/** Base URL of the Loki instance, e.g. `http://loki:3100`. */
@@ -59,7 +59,8 @@ export interface LokiStatus {
 export type LokiFetch = (url: string, init: {
 	readonly method: string;
 	readonly headers: Readonly<Record<string, string>>;
-	readonly body: string;
+	/** Absent on a GET (the history read). */
+	readonly body?: string;
 	readonly signal?: AbortSignal;
 }) => Promise<{
 	readonly ok: boolean;
@@ -81,6 +82,8 @@ export declare class LokiDestination implements ILogDestination {
 	private failures;
 	private lastError;
 	private lastPushAtMs;
+	/** In-flight history reads, by requestId, so `cancelRead` can abort the fetch. */
+	private readonly reads;
 	private readonly fetchFn;
 	private readonly setIntervalFn;
 	private readonly clearIntervalFn;
@@ -120,6 +123,20 @@ export declare class LokiDestination implements ILogDestination {
 		ok: boolean;
 		error: string | null;
 	}>;
-	query(_filter: LogFilter): Promise<readonly LogEntry[]>;
+	/** Headers every call to this Loki carries: tenant + basic auth. */
+	private authHeaders;
+	/**
+	* Readable when it is configured (D898): the same URL and credentials the
+	* push uses answer `query_range`. Labels are `job`/`node`/`level`, so those
+	* filter at the index; device, addon and text are line filters.
+	*/
+	describeRead(): Promise<LogReadCapabilities>;
+	/** One backward slice from `query_range`, newest first. Never logs. */
+	read(query: LogReadQuery): Promise<LogReadResult>;
+	cancelRead(input: {
+		readonly requestId: string;
+	}): Promise<{
+		readonly cancelled: boolean;
+	}>;
 	shutdown(): Promise<void>;
 }

package/dist/builtins/loki-logging/loki-read.d.ts

@@ -0,0 +1,51 @@
+import type { LogReadEntry, LogReadQuery, LogReadResult } from "@camstack/types";
+import { type ReadableLine } from "../../logging/log-read-filter.js";
+/** How far back a read may go when the query sets no `since`. */
+export declare const LOKI_READ_HORIZON_DAYS = 30;
+/** One query_range window. A busy stream fills `limit` long before this. */
+export declare const LOKI_WINDOW_MS: number;
+/** Where a Loki page stopped. */
+export interface LokiCursor {
+	/** Nanosecond epoch, decimal string. */
+	readonly n: string;
+	/** Hashes of the lines already returned at exactly `n`. */
+	readonly h: string[];
+}
+export declare function encodeLokiCursor(cursor: LokiCursor): string;
+export declare function decodeLokiCursor(raw: string): LokiCursor | null;
+/** FNV-1a, 32 bit, hex — a dedupe key, not a security property. */
+export declare function lineHash(line: string): string;
+export interface LokiQueryLabels {
+	readonly job: string;
+}
+/** The LogQL for a read. Pure. */
+export declare function buildLogQl(query: LogReadQuery, labels: LokiQueryLabels): string;
+export interface LokiRow {
+	readonly ns: bigint;
+	readonly hash: string;
+	readonly entry: LogReadEntry;
+	readonly line: ReadableLine;
+}
+/** A query_range body → rows, newest first. Lines that are not ours are dropped. */
+export declare function parseQueryRange(body: unknown): LokiRow[];
+/** One query_range call: newest-first rows in [startNs, endNs). */
+export type LokiRangeQuery = (args: {
+	readonly logql: string;
+	readonly startNs: bigint;
+	readonly endNs: bigint;
+	readonly limit: number;
+}) => Promise<{
+	readonly rows: LokiRow[];
+	readonly bytes: number;
+}>;
+/**
+* One backward slice over Loki: windows of {@link LOKI_WINDOW_MS}, newest
+* first, until `limit` matches, the horizon, the budget or a cancel.
+*/
+export declare function readLokiBackward(args: {
+	readonly query: LogReadQuery;
+	readonly labels: LokiQueryLabels;
+	readonly range: LokiRangeQuery;
+	readonly isCancelled: () => boolean;
+	readonly now?: () => number;
+}): Promise<LogReadResult>;