@sincpro/mobile 0.3.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +234 -2
- package/dist/entrypoints/cron/EBuiltinCron.d.ts +4 -0
- package/dist/entrypoints/cron/EBuiltinCron.js +5 -0
- package/dist/entrypoints/cron/checkNetworkStatus.cron.js +2 -1
- package/dist/entrypoints/db/migrations.d.ts +3 -1
- package/dist/entrypoints/db/migrations.js +47 -0
- package/dist/entrypoints/queue/QueueProcessor.js +1 -25
- package/dist/entrypoints/queue/subscriber_filter.d.ts +9 -0
- package/dist/entrypoints/queue/subscriber_filter.js +25 -0
- package/dist/entrypoints/ui/common_provider.js +2 -2
- package/dist/framework/base_module.d.ts +1 -0
- package/dist/framework/base_module.js +6 -1
- package/dist/framework/createApp.d.ts +14 -0
- package/dist/framework/createApp.js +6 -0
- package/dist/framework/kernel.d.ts +2 -1
- package/dist/framework/kernel.js +8 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +4 -0
- package/dist/infrastructure/context_manager/context.d.ts +16 -0
- package/dist/infrastructure/context_manager/context.js +37 -0
- package/dist/infrastructure/context_manager/context_api.d.ts +28 -0
- package/dist/infrastructure/context_manager/context_api.js +55 -0
- package/dist/infrastructure/context_manager/context_key.d.ts +15 -0
- package/dist/infrastructure/context_manager/context_key.js +3 -0
- package/dist/infrastructure/context_manager/context_manager_facade.d.ts +107 -0
- package/dist/infrastructure/context_manager/context_manager_facade.js +113 -0
- package/dist/infrastructure/context_manager/decorators/propagate_context.d.ts +24 -0
- package/dist/infrastructure/context_manager/decorators/propagate_context.js +52 -0
- package/dist/infrastructure/context_manager/decorators/with_context.d.ts +34 -0
- package/dist/infrastructure/context_manager/decorators/with_context.js +44 -0
- package/dist/infrastructure/context_manager/index.d.ts +11 -0
- package/dist/infrastructure/context_manager/index.js +12 -0
- package/dist/infrastructure/context_manager/managed_context.d.ts +87 -0
- package/dist/infrastructure/context_manager/managed_context.js +144 -0
- package/dist/infrastructure/context_manager/stack_context_manager.d.ts +39 -0
- package/dist/infrastructure/context_manager/stack_context_manager.js +56 -0
- package/dist/infrastructure/database/connector.js +10 -1
- package/dist/infrastructure/interceptor/index.d.ts +4 -0
- package/dist/infrastructure/interceptor/index.js +2 -0
- package/dist/infrastructure/interceptor/intercept.d.ts +57 -0
- package/dist/infrastructure/interceptor/intercept.js +112 -0
- package/dist/infrastructure/interceptor/logging_hooks.d.ts +22 -0
- package/dist/infrastructure/interceptor/logging_hooks.js +27 -0
- package/dist/infrastructure/logger.d.ts +11 -16
- package/dist/infrastructure/logger.js +61 -18
- package/dist/infrastructure/telemetry/buffer_registry.d.ts +15 -0
- package/dist/infrastructure/telemetry/buffer_registry.js +26 -0
- package/dist/infrastructure/telemetry/connectivity_state.d.ts +20 -0
- package/dist/infrastructure/telemetry/connectivity_state.js +28 -0
- package/dist/infrastructure/telemetry/context_keys.d.ts +7 -0
- package/dist/infrastructure/telemetry/context_keys.js +7 -0
- package/dist/infrastructure/telemetry/fetch_with_timeout.d.ts +8 -0
- package/dist/infrastructure/telemetry/fetch_with_timeout.js +17 -0
- package/dist/infrastructure/telemetry/flush_telemetry.d.ts +40 -0
- package/dist/infrastructure/telemetry/flush_telemetry.js +72 -0
- package/dist/infrastructure/telemetry/index.d.ts +17 -0
- package/dist/infrastructure/telemetry/index.js +17 -0
- package/dist/infrastructure/telemetry/init_telemetry.d.ts +21 -0
- package/dist/infrastructure/telemetry/init_telemetry.js +85 -0
- package/dist/infrastructure/telemetry/logging/flush_job.d.ts +44 -0
- package/dist/infrastructure/telemetry/logging/flush_job.js +69 -0
- package/dist/infrastructure/telemetry/logging/log_queue_repository.d.ts +76 -0
- package/dist/infrastructure/telemetry/logging/log_queue_repository.js +147 -0
- package/dist/infrastructure/telemetry/logging/loki_client.d.ts +32 -0
- package/dist/infrastructure/telemetry/logging/loki_client.js +118 -0
- package/dist/infrastructure/telemetry/logging/loki_registry.d.ts +12 -0
- package/dist/infrastructure/telemetry/logging/loki_registry.js +17 -0
- package/dist/infrastructure/telemetry/telemetry_flush_worker.d.ts +35 -0
- package/dist/infrastructure/telemetry/telemetry_flush_worker.js +66 -0
- package/dist/infrastructure/telemetry/telemetry_signal.d.ts +18 -0
- package/dist/infrastructure/telemetry/telemetry_signal.js +33 -0
- package/dist/infrastructure/telemetry/tracing/active_span.d.ts +22 -0
- package/dist/infrastructure/telemetry/tracing/active_span.js +40 -0
- package/dist/infrastructure/telemetry/tracing/otlp_client.d.ts +13 -0
- package/dist/infrastructure/telemetry/tracing/otlp_client.js +34 -0
- package/dist/infrastructure/telemetry/tracing/otlp_serializer.d.ts +67 -0
- package/dist/infrastructure/telemetry/tracing/otlp_serializer.js +101 -0
- package/dist/infrastructure/telemetry/tracing/span_flush_job.d.ts +25 -0
- package/dist/infrastructure/telemetry/tracing/span_flush_job.js +42 -0
- package/dist/infrastructure/telemetry/tracing/span_queue_repository.d.ts +82 -0
- package/dist/infrastructure/telemetry/tracing/span_queue_repository.js +134 -0
- package/dist/infrastructure/telemetry/tracing/span_sampler.d.ts +40 -0
- package/dist/infrastructure/telemetry/tracing/span_sampler.js +75 -0
- package/dist/infrastructure/telemetry/tracing/span_serializer.d.ts +48 -0
- package/dist/infrastructure/telemetry/tracing/span_serializer.js +36 -0
- package/dist/infrastructure/telemetry/tracing/sqlite_span_exporter.d.ts +26 -0
- package/dist/infrastructure/telemetry/tracing/sqlite_span_exporter.js +47 -0
- package/dist/infrastructure/telemetry/tracing/tracer.d.ts +22 -0
- package/dist/infrastructure/telemetry/tracing/tracer.js +45 -0
- package/dist/infrastructure/telemetry/tracing/tracing_hooks.d.ts +10 -0
- package/dist/infrastructure/telemetry/tracing/tracing_hooks.js +59 -0
- package/dist/infrastructure/telemetry/tracing_facade.d.ts +50 -0
- package/dist/infrastructure/telemetry/tracing_facade.js +139 -0
- package/dist/infrastructure/telemetry/types.d.ts +120 -0
- package/dist/infrastructure/telemetry/types.js +1 -0
- package/dist/infrastructure/workers/CronWorker.d.ts +10 -2
- package/dist/infrastructure/workers/CronWorker.js +18 -5
- package/dist/infrastructure/workers/index.d.ts +1 -1
- package/dist/ui/components/molecules/TimeZoneSelectorModal.js +7 -1
- package/package.json +17 -11
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
let _logQueue = null;
|
|
2
|
+
let _spanQueue = null;
|
|
3
|
+
/** @internal — wired by initTelemetry so consumers can inspect buffer pressure. */
|
|
4
|
+
export function _registerBuffers(logQueue, spanQueue) {
|
|
5
|
+
_logQueue = logQueue;
|
|
6
|
+
_spanQueue = spanQueue;
|
|
7
|
+
}
|
|
8
|
+
/** @internal — test use only. */
|
|
9
|
+
export function _resetBuffers() {
|
|
10
|
+
_logQueue = null;
|
|
11
|
+
_spanQueue = null;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Current size and drop pressure of the telemetry buffers.
|
|
15
|
+
*
|
|
16
|
+
* Use this to surface offline backlog in dashboards or health checks — a high
|
|
17
|
+
* `dropped` count means the device has been offline long enough to lose data.
|
|
18
|
+
* Returns null buffers when the corresponding signal is disabled.
|
|
19
|
+
*/
|
|
20
|
+
export async function bufferStats() {
|
|
21
|
+
const [logs, spans] = await Promise.all([
|
|
22
|
+
_logQueue?.stats() ?? Promise.resolve(null),
|
|
23
|
+
_spanQueue?.stats() ?? Promise.resolve(null),
|
|
24
|
+
]);
|
|
25
|
+
return { logs, spans };
|
|
26
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Self-correcting online/offline flag for the telemetry pipeline.
|
|
3
|
+
*
|
|
4
|
+
* The framework does NOT depend on the app's connectivity cron — that is
|
|
5
|
+
* client-specific and optional. Instead this flag is corrected from two cheap,
|
|
6
|
+
* always-available sources:
|
|
7
|
+
* - the outcome of each delivery attempt (success → online, network failure → offline)
|
|
8
|
+
* - optional connectivity events, when the host app emits them
|
|
9
|
+
*
|
|
10
|
+
* It exists to skip delivery attempts while known-offline (avoiding hung HTTP
|
|
11
|
+
* and wasted work); the send timeout in the HTTP clients is the backstop when
|
|
12
|
+
* the flag is optimistically wrong.
|
|
13
|
+
*/
|
|
14
|
+
export declare class ConnectivityState {
|
|
15
|
+
private online;
|
|
16
|
+
constructor(initiallyOnline?: boolean);
|
|
17
|
+
isOnline(): boolean;
|
|
18
|
+
markOnline(): void;
|
|
19
|
+
markOffline(): void;
|
|
20
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Self-correcting online/offline flag for the telemetry pipeline.
|
|
3
|
+
*
|
|
4
|
+
* The framework does NOT depend on the app's connectivity cron — that is
|
|
5
|
+
* client-specific and optional. Instead this flag is corrected from two cheap,
|
|
6
|
+
* always-available sources:
|
|
7
|
+
* - the outcome of each delivery attempt (success → online, network failure → offline)
|
|
8
|
+
* - optional connectivity events, when the host app emits them
|
|
9
|
+
*
|
|
10
|
+
* It exists to skip delivery attempts while known-offline (avoiding hung HTTP
|
|
11
|
+
* and wasted work); the send timeout in the HTTP clients is the backstop when
|
|
12
|
+
* the flag is optimistically wrong.
|
|
13
|
+
*/
|
|
14
|
+
export class ConnectivityState {
|
|
15
|
+
online;
|
|
16
|
+
constructor(initiallyOnline = true) {
|
|
17
|
+
this.online = initiallyOnline;
|
|
18
|
+
}
|
|
19
|
+
isOnline() {
|
|
20
|
+
return this.online;
|
|
21
|
+
}
|
|
22
|
+
markOnline() {
|
|
23
|
+
this.online = true;
|
|
24
|
+
}
|
|
25
|
+
markOffline() {
|
|
26
|
+
this.online = false;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Context as OtelContext } from "@opentelemetry/api";
|
|
2
|
+
/**
|
|
3
|
+
* Key under which the active OTel Context (carrying the current Span) is
|
|
4
|
+
* stored inside the framework Context. Shared between tracing_hooks (writes)
|
|
5
|
+
* and active_span (reads) so both sides reference the same Symbol.
|
|
6
|
+
*/
|
|
7
|
+
export declare const OTEL_CTX_KEY: import("../context_manager/context_key").ContextKey<OtelContext>;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { createContextKey } from "../context_manager/index";
|
|
2
|
+
/**
|
|
3
|
+
* Key under which the active OTel Context (carrying the current Span) is
|
|
4
|
+
* stored inside the framework Context. Shared between tracing_hooks (writes)
|
|
5
|
+
* and active_span (reads) so both sides reference the same Symbol.
|
|
6
|
+
*/
|
|
7
|
+
export const OTEL_CTX_KEY = createContextKey("sincpro.telemetry.otel_context");
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** Default send timeout — fail fast instead of hanging when the network is flaky/offline. */
|
|
2
|
+
export declare const DEFAULT_SEND_TIMEOUT_MS = 4000;
|
|
3
|
+
/**
|
|
4
|
+
* `fetch` with a hard timeout via AbortController. On timeout the request is
|
|
5
|
+
* aborted and the returned promise rejects, so an offline/slow network never
|
|
6
|
+
* blocks the telemetry pipeline indefinitely.
|
|
7
|
+
*/
|
|
8
|
+
export declare function fetchWithTimeout(input: string, init: RequestInit, timeoutMs?: number): Promise<Response>;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** Default send timeout — fail fast instead of hanging when the network is flaky/offline. */
|
|
2
|
+
export const DEFAULT_SEND_TIMEOUT_MS = 4000;
|
|
3
|
+
/**
|
|
4
|
+
* `fetch` with a hard timeout via AbortController. On timeout the request is
|
|
5
|
+
* aborted and the returned promise rejects, so an offline/slow network never
|
|
6
|
+
* blocks the telemetry pipeline indefinitely.
|
|
7
|
+
*/
|
|
8
|
+
export async function fetchWithTimeout(input, init, timeoutMs = DEFAULT_SEND_TIMEOUT_MS) {
|
|
9
|
+
const controller = new AbortController();
|
|
10
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
11
|
+
try {
|
|
12
|
+
return await fetch(input, { ...init, signal: controller.signal });
|
|
13
|
+
}
|
|
14
|
+
finally {
|
|
15
|
+
clearTimeout(timer);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { ConnectivityState } from "./connectivity_state";
|
|
2
|
+
import type { FlushClient, FlushQueue } from "./logging/flush_job";
|
|
3
|
+
import type { SpanFlushClient, SpanFlushQueue } from "./tracing/span_flush_job";
|
|
4
|
+
export interface FlushTelemetryDeps {
|
|
5
|
+
connectivity: ConnectivityState;
|
|
6
|
+
logClient?: FlushClient | null;
|
|
7
|
+
logQueue?: FlushQueue | null;
|
|
8
|
+
spanClient?: SpanFlushClient | null;
|
|
9
|
+
spanQueue?: SpanFlushQueue | null;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The single unit of telemetry delivery. Every trigger — production signal,
|
|
13
|
+
* reconnect event, background cron — funnels through `run()`, which drains BOTH
|
|
14
|
+
* the log and span queues.
|
|
15
|
+
*
|
|
16
|
+
* Throttling and offline recovery are delegated to the jobs' own exponential
|
|
17
|
+
* backoff: after a failed delivery a job skips a growing number of runs, then
|
|
18
|
+
* retries. There is NO sticky offline gate, so the pipeline always recovers
|
|
19
|
+
* once the network returns — even with no cron and no connectivity events. The
|
|
20
|
+
* HTTP send timeout prevents a flush from hanging while offline.
|
|
21
|
+
*
|
|
22
|
+
* `ConnectivityState` is kept informational (surfaced in stats, fed by events);
|
|
23
|
+
* it never blocks delivery.
|
|
24
|
+
*/
|
|
25
|
+
export declare class FlushTelemetry {
|
|
26
|
+
private readonly deps;
|
|
27
|
+
private logState;
|
|
28
|
+
private spanState;
|
|
29
|
+
private running;
|
|
30
|
+
private pending;
|
|
31
|
+
constructor(deps: FlushTelemetryDeps);
|
|
32
|
+
/**
|
|
33
|
+
* @param resetBackoff - clear the jobs' backoff and retry immediately. Used by
|
|
34
|
+
* the reconnect event and the background cron so recovery is instant instead
|
|
35
|
+
* of waiting for the current backoff window to elapse.
|
|
36
|
+
*/
|
|
37
|
+
run({ resetBackoff }?: {
|
|
38
|
+
resetBackoff?: boolean;
|
|
39
|
+
}): Promise<void>;
|
|
40
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { createLogFlushJobState, runLogFlushJob } from "./logging/flush_job";
|
|
2
|
+
import { createSpanFlushJobState, runSpanFlushJob } from "./tracing/span_flush_job";
|
|
3
|
+
/**
|
|
4
|
+
* The single unit of telemetry delivery. Every trigger — production signal,
|
|
5
|
+
* reconnect event, background cron — funnels through `run()`, which drains BOTH
|
|
6
|
+
* the log and span queues.
|
|
7
|
+
*
|
|
8
|
+
* Throttling and offline recovery are delegated to the jobs' own exponential
|
|
9
|
+
* backoff: after a failed delivery a job skips a growing number of runs, then
|
|
10
|
+
* retries. There is NO sticky offline gate, so the pipeline always recovers
|
|
11
|
+
* once the network returns — even with no cron and no connectivity events. The
|
|
12
|
+
* HTTP send timeout prevents a flush from hanging while offline.
|
|
13
|
+
*
|
|
14
|
+
* `ConnectivityState` is kept informational (surfaced in stats, fed by events);
|
|
15
|
+
* it never blocks delivery.
|
|
16
|
+
*/
|
|
17
|
+
export class FlushTelemetry {
|
|
18
|
+
deps;
|
|
19
|
+
logState = createLogFlushJobState();
|
|
20
|
+
spanState = createSpanFlushJobState();
|
|
21
|
+
running = false;
|
|
22
|
+
pending = false;
|
|
23
|
+
constructor(deps) {
|
|
24
|
+
this.deps = deps;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* @param resetBackoff - clear the jobs' backoff and retry immediately. Used by
|
|
28
|
+
* the reconnect event and the background cron so recovery is instant instead
|
|
29
|
+
* of waiting for the current backoff window to elapse.
|
|
30
|
+
*/
|
|
31
|
+
async run({ resetBackoff = false } = {}) {
|
|
32
|
+
// Re-entrancy guard. A trigger arriving mid-flush sets `pending` so the
|
|
33
|
+
// current run loops once more — no produced telemetry is silently dropped.
|
|
34
|
+
if (this.running) {
|
|
35
|
+
this.pending = true;
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
this.running = true;
|
|
39
|
+
try {
|
|
40
|
+
let reset = resetBackoff;
|
|
41
|
+
do {
|
|
42
|
+
this.pending = false;
|
|
43
|
+
if (reset) {
|
|
44
|
+
this.logState = createLogFlushJobState();
|
|
45
|
+
this.spanState = createSpanFlushJobState();
|
|
46
|
+
reset = false;
|
|
47
|
+
}
|
|
48
|
+
let delivered = false;
|
|
49
|
+
let failed = false;
|
|
50
|
+
if (this.deps.logClient && this.deps.logQueue) {
|
|
51
|
+
const r = await runLogFlushJob(this.deps.logClient, this.deps.logQueue, this.logState);
|
|
52
|
+
delivered ||= r.delivered > 0;
|
|
53
|
+
failed ||= r.failed;
|
|
54
|
+
}
|
|
55
|
+
if (this.deps.spanClient && this.deps.spanQueue) {
|
|
56
|
+
const r = await runSpanFlushJob(this.deps.spanClient, this.deps.spanQueue, this.spanState);
|
|
57
|
+
delivered ||= r.delivered > 0;
|
|
58
|
+
failed ||= r.failed;
|
|
59
|
+
}
|
|
60
|
+
// A working destination means we are online; only a total failure
|
|
61
|
+
// (nothing delivered) flips the informational flag offline.
|
|
62
|
+
if (delivered)
|
|
63
|
+
this.deps.connectivity.markOnline();
|
|
64
|
+
else if (failed)
|
|
65
|
+
this.deps.connectivity.markOffline();
|
|
66
|
+
} while (this.pending);
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
this.running = false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export { Tracing } from "./tracing_facade";
|
|
2
|
+
export { initTelemetry } from "./init_telemetry";
|
|
3
|
+
export type { BufferStats, FlushConfig, LokiAuth, LokiConfig, OtlpConfig, TelemetryBufferStats, TelemetryConfig, } from "./types";
|
|
4
|
+
export { bufferStats } from "./buffer_registry";
|
|
5
|
+
export type { LogEntry } from "./logging/log_queue_repository";
|
|
6
|
+
export { LogQueueRepository } from "./logging/log_queue_repository";
|
|
7
|
+
export { LokiClient } from "./logging/loki_client";
|
|
8
|
+
export { getLokiClient } from "./logging/loki_registry";
|
|
9
|
+
export { activeTraceLabel, getActiveSpanContext } from "./tracing/active_span";
|
|
10
|
+
export { OtlpClient } from "./tracing/otlp_client";
|
|
11
|
+
export type { OtlpExportRequest } from "./tracing/otlp_serializer";
|
|
12
|
+
export { serializeToOtlp } from "./tracing/otlp_serializer";
|
|
13
|
+
export type { SpanInput, SpanRow } from "./tracing/span_queue_repository";
|
|
14
|
+
export { SpanQueueRepository } from "./tracing/span_queue_repository";
|
|
15
|
+
export { SQLiteSpanExporter } from "./tracing/sqlite_span_exporter";
|
|
16
|
+
export { getTracer } from "./tracing/tracer";
|
|
17
|
+
export { tracingHooks } from "./tracing/tracing_hooks";
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// Main entry point — public facade for framework consumers
|
|
2
|
+
export { Tracing } from "./tracing_facade";
|
|
3
|
+
// Initialization
|
|
4
|
+
export { initTelemetry } from "./init_telemetry";
|
|
5
|
+
// Buffer observability
|
|
6
|
+
export { bufferStats } from "./buffer_registry";
|
|
7
|
+
export { LogQueueRepository } from "./logging/log_queue_repository";
|
|
8
|
+
export { LokiClient } from "./logging/loki_client";
|
|
9
|
+
export { getLokiClient } from "./logging/loki_registry";
|
|
10
|
+
// Tracing
|
|
11
|
+
export { activeTraceLabel, getActiveSpanContext } from "./tracing/active_span";
|
|
12
|
+
export { OtlpClient } from "./tracing/otlp_client";
|
|
13
|
+
export { serializeToOtlp } from "./tracing/otlp_serializer";
|
|
14
|
+
export { SpanQueueRepository } from "./tracing/span_queue_repository";
|
|
15
|
+
export { SQLiteSpanExporter } from "./tracing/sqlite_span_exporter";
|
|
16
|
+
export { getTracer } from "./tracing/tracer";
|
|
17
|
+
export { tracingHooks } from "./tracing/tracing_hooks";
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { TelemetryConfig } from "./types";
|
|
2
|
+
/**
|
|
3
|
+
* Initializes telemetry with offline-first delivery.
|
|
4
|
+
*
|
|
5
|
+
* Delivery is event-driven, not a fixed timer:
|
|
6
|
+
* - Producing telemetry schedules a debounced flush (the primary trigger).
|
|
7
|
+
* - A single {@link FlushTelemetry} use case drains BOTH the log and span
|
|
8
|
+
* queues, gated by a self-correcting connectivity flag + HTTP send timeout.
|
|
9
|
+
* - Optional triggers (none required): reconnect events drain the backlog
|
|
10
|
+
* immediately; a long-interval background cron probes as a safety net.
|
|
11
|
+
*
|
|
12
|
+
* Spans always buffer to SQLite only when an OTLP destination is configured;
|
|
13
|
+
* the TracerProvider is registered regardless so `@Trace`/`withSpan` work.
|
|
14
|
+
*
|
|
15
|
+
* Idempotent: safe to call again (hot-reload, config change). The previous
|
|
16
|
+
* worker (signal + events + cron) is fully torn down — awaited — before the new
|
|
17
|
+
* one registers, so the shared background task name is never double-registered.
|
|
18
|
+
*
|
|
19
|
+
* Call once at app startup after migrations have run (`createApp` handles this).
|
|
20
|
+
*/
|
|
21
|
+
export declare function initTelemetry(config: TelemetryConfig): Promise<void>;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { DBCursor } from "../../infrastructure/database";
|
|
2
|
+
import { _registerBuffers } from "./buffer_registry";
|
|
3
|
+
import { ConnectivityState } from "./connectivity_state";
|
|
4
|
+
import { FlushTelemetry } from "./flush_telemetry";
|
|
5
|
+
import { LogQueueRepository } from "./logging/log_queue_repository";
|
|
6
|
+
import { _resetLokiClient, getLokiClient, initLokiClient } from "./logging/loki_registry";
|
|
7
|
+
import { TelemetryFlushWorker } from "./telemetry_flush_worker";
|
|
8
|
+
import { TelemetrySignal } from "./telemetry_signal";
|
|
9
|
+
import { OtlpClient } from "./tracing/otlp_client";
|
|
10
|
+
import { EVICT_CHECK_EVERY, MAX_SPANS_QUEUE_BYTES, MAX_SPANS_QUEUE_SIZE, SpanQueueRepository, } from "./tracing/span_queue_repository";
|
|
11
|
+
import { SpanSampler } from "./tracing/span_sampler";
|
|
12
|
+
import { initTracing } from "./tracing/tracer";
|
|
13
|
+
let _worker = null;
|
|
14
|
+
/**
|
|
15
|
+
* Initializes telemetry with offline-first delivery.
|
|
16
|
+
*
|
|
17
|
+
* Delivery is event-driven, not a fixed timer:
|
|
18
|
+
* - Producing telemetry schedules a debounced flush (the primary trigger).
|
|
19
|
+
* - A single {@link FlushTelemetry} use case drains BOTH the log and span
|
|
20
|
+
* queues, gated by a self-correcting connectivity flag + HTTP send timeout.
|
|
21
|
+
* - Optional triggers (none required): reconnect events drain the backlog
|
|
22
|
+
* immediately; a long-interval background cron probes as a safety net.
|
|
23
|
+
*
|
|
24
|
+
* Spans always buffer to SQLite only when an OTLP destination is configured;
|
|
25
|
+
* the TracerProvider is registered regardless so `@Trace`/`withSpan` work.
|
|
26
|
+
*
|
|
27
|
+
* Idempotent: safe to call again (hot-reload, config change). The previous
|
|
28
|
+
* worker (signal + events + cron) is fully torn down — awaited — before the new
|
|
29
|
+
* one registers, so the shared background task name is never double-registered.
|
|
30
|
+
*
|
|
31
|
+
* Call once at app startup after migrations have run (`createApp` handles this).
|
|
32
|
+
*/
|
|
33
|
+
export async function initTelemetry(config) {
|
|
34
|
+
// Tear down the previous worker before re-wiring (hot-reload / config change).
|
|
35
|
+
// Awaited so the old background task is unregistered before we register a new
|
|
36
|
+
// one under the same task name.
|
|
37
|
+
await _worker?.stop().catch(() => { });
|
|
38
|
+
_worker = null;
|
|
39
|
+
const connectivity = new ConnectivityState();
|
|
40
|
+
const timeoutMs = config.flush?.sendTimeoutMs;
|
|
41
|
+
// Logs — opportunistic push (with fallback to SQLite) + batch drain on flush.
|
|
42
|
+
let logQueue = null;
|
|
43
|
+
if (config.loki) {
|
|
44
|
+
logQueue = new LogQueueRepository(DBCursor);
|
|
45
|
+
initLokiClient(config.loki, logQueue, timeoutMs);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
// Drop a stale client from a previous init that had Loki configured.
|
|
49
|
+
_resetLokiClient();
|
|
50
|
+
}
|
|
51
|
+
const logClient = config.loki ? getLokiClient() : null;
|
|
52
|
+
// Spans — always buffered to SQLite; OTLP client only when a destination exists.
|
|
53
|
+
// Head sampling (default on) drops whole traces at ingest once the buffer is
|
|
54
|
+
// near full, so sustained offline pressure degrades gracefully instead of
|
|
55
|
+
// thrashing insert→evict.
|
|
56
|
+
const spanQueue = new SpanQueueRepository(DBCursor, MAX_SPANS_QUEUE_SIZE, MAX_SPANS_QUEUE_BYTES, EVICT_CHECK_EVERY, new SpanSampler());
|
|
57
|
+
const spanClient = config.otlp ? new OtlpClient(config.otlp, timeoutMs) : null;
|
|
58
|
+
// The single delivery use case — all triggers funnel here.
|
|
59
|
+
const flush = new FlushTelemetry({
|
|
60
|
+
connectivity,
|
|
61
|
+
logClient,
|
|
62
|
+
logQueue,
|
|
63
|
+
spanClient,
|
|
64
|
+
spanQueue,
|
|
65
|
+
});
|
|
66
|
+
// Production signal: producing telemetry schedules a debounced flush.
|
|
67
|
+
const signal = new TelemetrySignal(() => {
|
|
68
|
+
flush.run().catch(() => { });
|
|
69
|
+
}, config.flush?.debounceMs);
|
|
70
|
+
// Register the TracerProvider; persist spans (and fire the signal) only when
|
|
71
|
+
// an OTLP destination is configured.
|
|
72
|
+
initTracing(spanQueue, !!config.otlp, () => signal.notify(), {
|
|
73
|
+
serviceName: config.serviceName,
|
|
74
|
+
serviceVersion: config.serviceVersion,
|
|
75
|
+
deploymentEnvironment: config.deploymentEnvironment,
|
|
76
|
+
extra: config.resourceAttributes,
|
|
77
|
+
});
|
|
78
|
+
_worker = new TelemetryFlushWorker(flush, signal, connectivity, {
|
|
79
|
+
backgroundIntervalMin: config.flush?.backgroundIntervalMin,
|
|
80
|
+
onConnectivityEvents: config.flush?.onConnectivityEvents,
|
|
81
|
+
});
|
|
82
|
+
_worker.start();
|
|
83
|
+
// Expose buffer pressure for health checks / dashboards.
|
|
84
|
+
_registerBuffers(logQueue, spanQueue);
|
|
85
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { LogEntry } from "./log_queue_repository";
|
|
2
|
+
/** Max entries per Loki push request. Keeps payloads well under the 4MB Loki limit. */
|
|
3
|
+
export declare const FLUSH_BATCH_SIZE = 100;
|
|
4
|
+
/**
|
|
5
|
+
* Max batches drained in a single cron tick.
|
|
6
|
+
* After an offline period the queue can hold thousands of entries; without this cap
|
|
7
|
+
* a single tick could run for minutes blocking the CronWorker job slot.
|
|
8
|
+
* 5 batches × 100 entries = 500 entries max per minute — enough to drain hours of
|
|
9
|
+
* backlog within a few ticks without saturating the cron.
|
|
10
|
+
*/
|
|
11
|
+
export declare const MAX_BATCHES_PER_TICK = 5;
|
|
12
|
+
/** Mutable backoff state — one instance held by FlushTelemetry. */
|
|
13
|
+
export interface LogFlushJobState {
|
|
14
|
+
consecutiveFailures: number;
|
|
15
|
+
skipTicksRemaining: number;
|
|
16
|
+
}
|
|
17
|
+
export declare function createLogFlushJobState(): LogFlushJobState;
|
|
18
|
+
/** Outcome of one flush run — lets callers update connectivity state. */
|
|
19
|
+
export interface FlushResult {
|
|
20
|
+
/** Batches successfully delivered this run. */
|
|
21
|
+
delivered: number;
|
|
22
|
+
/** True if a delivery failed (network/HTTP) — caller may mark offline. */
|
|
23
|
+
failed: boolean;
|
|
24
|
+
}
|
|
25
|
+
export interface FlushClient {
|
|
26
|
+
deliver(entries: LogEntry[]): Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
export interface FlushQueue {
|
|
29
|
+
pruneExpired(): Promise<void>;
|
|
30
|
+
findPending(limit: number): Promise<LogEntry[]>;
|
|
31
|
+
removeMany(ids: number[]): Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* One tick of the flush cron.
|
|
35
|
+
*
|
|
36
|
+
* Ordering invariant: `deliver()` always runs before `removeMany()`.
|
|
37
|
+
* A crash between the two means the same entries are retried next tick (at-least-once
|
|
38
|
+
* delivery). Duplicates in Loki are acceptable for log telemetry; data loss is not.
|
|
39
|
+
*
|
|
40
|
+
* Transactions: impossible here — an HTTP call sits between the SQLite read and delete.
|
|
41
|
+
* The mutex inside DBCursor serialises concurrent DB access, but two overlapping cron
|
|
42
|
+
* ticks can still deliver the same batch twice. Again: acceptable for logs.
|
|
43
|
+
*/
|
|
44
|
+
export declare function runLogFlushJob(client: FlushClient, queue: FlushQueue, state: LogFlushJobState): Promise<FlushResult>;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// Constants
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
/** Max entries per Loki push request. Keeps payloads well under the 4MB Loki limit. */
|
|
5
|
+
export const FLUSH_BATCH_SIZE = 100;
|
|
6
|
+
/**
|
|
7
|
+
* Max batches drained in a single cron tick.
|
|
8
|
+
* After an offline period the queue can hold thousands of entries; without this cap
|
|
9
|
+
* a single tick could run for minutes blocking the CronWorker job slot.
|
|
10
|
+
* 5 batches × 100 entries = 500 entries max per minute — enough to drain hours of
|
|
11
|
+
* backlog within a few ticks without saturating the cron.
|
|
12
|
+
*/
|
|
13
|
+
export const MAX_BATCHES_PER_TICK = 5;
|
|
14
|
+
/**
|
|
15
|
+
* Exponential backoff cap in ticks. After repeated Loki failures the cron backs off
|
|
16
|
+
* up to 32 skipped ticks (32 min at 1-min interval) before retrying.
|
|
17
|
+
*/
|
|
18
|
+
const MAX_SKIP_TICKS = 32;
|
|
19
|
+
export function createLogFlushJobState() {
|
|
20
|
+
return { consecutiveFailures: 0, skipTicksRemaining: 0 };
|
|
21
|
+
}
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
// Core flush logic
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
/**
|
|
26
|
+
* One tick of the flush cron.
|
|
27
|
+
*
|
|
28
|
+
* Ordering invariant: `deliver()` always runs before `removeMany()`.
|
|
29
|
+
* A crash between the two means the same entries are retried next tick (at-least-once
|
|
30
|
+
* delivery). Duplicates in Loki are acceptable for log telemetry; data loss is not.
|
|
31
|
+
*
|
|
32
|
+
* Transactions: impossible here — an HTTP call sits between the SQLite read and delete.
|
|
33
|
+
* The mutex inside DBCursor serialises concurrent DB access, but two overlapping cron
|
|
34
|
+
* ticks can still deliver the same batch twice. Again: acceptable for logs.
|
|
35
|
+
*/
|
|
36
|
+
export async function runLogFlushJob(client, queue, state) {
|
|
37
|
+
// Backoff — skip this tick if previous deliveries failed repeatedly
|
|
38
|
+
if (state.skipTicksRemaining > 0) {
|
|
39
|
+
state.skipTicksRemaining -= 1;
|
|
40
|
+
return { delivered: 0, failed: false };
|
|
41
|
+
}
|
|
42
|
+
await queue.pruneExpired();
|
|
43
|
+
let batchesDelivered = 0;
|
|
44
|
+
while (batchesDelivered < MAX_BATCHES_PER_TICK) {
|
|
45
|
+
const entries = await queue.findPending(FLUSH_BATCH_SIZE);
|
|
46
|
+
if (entries.length === 0)
|
|
47
|
+
break;
|
|
48
|
+
try {
|
|
49
|
+
await client.deliver(entries);
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
|
+
// Loki is unavailable — keep entries in queue and back off.
|
|
53
|
+
state.consecutiveFailures += 1;
|
|
54
|
+
// 1st fail → skip 1 tick (retry in 2 min)
|
|
55
|
+
// 2nd fail → skip 2 ticks (retry in 3 min) …capped at 32
|
|
56
|
+
state.skipTicksRemaining = Math.min(Math.pow(2, state.consecutiveFailures - 1), MAX_SKIP_TICKS);
|
|
57
|
+
return { delivered: batchesDelivered, failed: true };
|
|
58
|
+
}
|
|
59
|
+
await queue.removeMany(entries.map((e) => e.id));
|
|
60
|
+
batchesDelivered += 1;
|
|
61
|
+
// Fewer entries than the batch limit → queue is now empty
|
|
62
|
+
if (entries.length < FLUSH_BATCH_SIZE)
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
if (batchesDelivered > 0) {
|
|
66
|
+
state.consecutiveFailures = 0;
|
|
67
|
+
}
|
|
68
|
+
return { delivered: batchesDelivered, failed: false };
|
|
69
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { DBCursor } from "../../../infrastructure/database";
|
|
2
|
+
import type { BufferStats } from "../types";
|
|
3
|
+
export interface LogEntry {
|
|
4
|
+
id: number;
|
|
5
|
+
level: string;
|
|
6
|
+
message: string;
|
|
7
|
+
created_at: string;
|
|
8
|
+
}
|
|
9
|
+
/** Max rows kept in telemetry_queue before eviction kicks in. */
|
|
10
|
+
export declare const MAX_LOG_QUEUE_SIZE = 5000;
|
|
11
|
+
/** Max approximate payload bytes kept in telemetry_queue (~2 MB). */
|
|
12
|
+
export declare const MAX_LOG_QUEUE_BYTES = 2000000;
|
|
13
|
+
/** Run the (scanning) eviction check once every N enqueues — see class doc. */
|
|
14
|
+
export declare const EVICT_CHECK_EVERY = 32;
|
|
15
|
+
/**
|
|
16
|
+
* Drop priority by level — LOWER is evicted FIRST. So under pressure debug/info
|
|
17
|
+
* go before warn, and error survives longest. Unknown levels fall back to
|
|
18
|
+
* {@link DEFAULT_DROP_PRIORITY}. Keys are the lowercase level strings the logger
|
|
19
|
+
* persists.
|
|
20
|
+
*/
|
|
21
|
+
export declare const LOG_LEVEL_DROP_PRIORITY: Readonly<Record<string, number>>;
|
|
22
|
+
/**
|
|
23
|
+
* Offline-first log buffer, bounded by BOTH a row count and a payload-byte
|
|
24
|
+
* budget. When either is exceeded, rows are evicted **lowest-severity-first**
|
|
25
|
+
* (debug → info → warn → error), oldest-first within a severity — so error/warn
|
|
26
|
+
* logs survive longest under sustained offline pressure.
|
|
27
|
+
*
|
|
28
|
+
* The budget check scans the table, so it runs **amortized** — once every
|
|
29
|
+
* `evictEvery` enqueues — to keep the hot path cheap. The bound is therefore
|
|
30
|
+
* soft: the queue may transiently exceed the cap by up to `evictEvery` rows.
|
|
31
|
+
*
|
|
32
|
+
* Telemetry is best-effort: under sustained offline pressure logs ARE dropped,
|
|
33
|
+
* but the loss is bounded, severity-aware, and observable (see {@link stats}).
|
|
34
|
+
* Business-critical data must NOT live here.
|
|
35
|
+
*/
|
|
36
|
+
export declare class LogQueueRepository {
|
|
37
|
+
private readonly db;
|
|
38
|
+
private readonly maxRows;
|
|
39
|
+
private readonly maxBytes;
|
|
40
|
+
private readonly evictEvery;
|
|
41
|
+
private ops;
|
|
42
|
+
private dropped;
|
|
43
|
+
constructor(db: typeof DBCursor, maxRows?: number, maxBytes?: number, evictEvery?: number);
|
|
44
|
+
enqueue(level: string, message: string): Promise<void>;
|
|
45
|
+
/**
|
|
46
|
+
* Brings the queue under both budgets. The two budgets use DIFFERENT victim
|
|
47
|
+
* policies on purpose:
|
|
48
|
+
*
|
|
49
|
+
* - Row pressure → **severity-aware**: drop lowest-severity, oldest-first.
|
|
50
|
+
* Each deletion reduces the count by exactly one, so this is exact.
|
|
51
|
+
* - Byte pressure → **largest-first**, reclaiming ACTUAL bytes. Severity is
|
|
52
|
+
* deliberately NOT used here: when the overage is caused by one oversized
|
|
53
|
+
* payload, protecting it (severity-first) forces deleting everything else
|
|
54
|
+
* to free its bytes, annihilating the queue to clear an overage that row
|
|
55
|
+
* itself caused. Largest-first targets the cause directly, so the maximum
|
|
56
|
+
* number of (small, information-dense) entries survive. Corollary: a huge
|
|
57
|
+
* high-severity payload is NOT protected from byte eviction — keep large
|
|
58
|
+
* blobs out of logs.
|
|
59
|
+
*
|
|
60
|
+
* Terminates: each pass removes ≥1 row (count strictly decreases) or returns.
|
|
61
|
+
*/
|
|
62
|
+
private evict;
|
|
63
|
+
/** Deletes `n` rows lowest-severity-first, oldest-first within a tier. */
|
|
64
|
+
private dropBySeverity;
|
|
65
|
+
/**
|
|
66
|
+
* Deletes the largest rows (oldest as tiebreak) until at least `overage` bytes
|
|
67
|
+
* are reclaimed, using each row's ACTUAL size. Returns rows removed.
|
|
68
|
+
*/
|
|
69
|
+
private dropLargestUntilBytesFreed;
|
|
70
|
+
private measure;
|
|
71
|
+
stats(): Promise<BufferStats>;
|
|
72
|
+
findPending(limit?: number): Promise<LogEntry[]>;
|
|
73
|
+
removeMany(ids: number[]): Promise<void>;
|
|
74
|
+
/** Deletes entries older than `days` days. Called before each flush to bound queue growth. */
|
|
75
|
+
pruneExpired(days?: number): Promise<void>;
|
|
76
|
+
}
|