@zlink-systems/framework 0.30.3 → 0.32.0

package/src/runtime/diagnostics/message-flow-record.ts

@@ -0,0 +1,295 @@
+import type { Attributes } from '@opentelemetry/api';
+import type { ZLinkMessageFlowLogMode } from '../../contracts';
+import {
+  ZLinkDispatchErrorSurface,
+  ZLinkDispatchMessageKind,
+  ZLinkRuntimeMessageFlowOutcome as ZLinkMessageFlowOutcome,
+  type ZLinkRuntimeMessageFlowEvent,
+  type ZLinkRuntimeMessageFlowResult
+} from '../../contracts/Dispatch/ZLinkDispatchOptions';
+import { ERROR_MESSAGE_MAX_LENGTH } from './dispatch-error-details';
+import { FLOW_ORIGIN_VALUES } from './flow-context';
+export const TELEMETRY_INSTRUMENTATION_NAME = '@zlink-systems/framework';
+export const MESSAGE_FLOW_EVENT_ID = 'zlink.message_flow';
+export const DISPATCH_ERROR_EVENT_ID = 'zlink.dispatch_error';
+
+export interface ZLinkTelemetryRecord {
+  readonly severity?: 'warning';
+  readonly eventId: typeof MESSAGE_FLOW_EVENT_ID | typeof DISPATCH_ERROR_EVENT_ID;
+  readonly timestamp: Date;
+  readonly phase?: string;
+  readonly outcome: ZLinkRuntimeMessageFlowResult;
+  readonly surface:
+    | 'node'
+    | 'channel'
+    | 'spot'
+    | 'instance_spot'
+    | 'actor'
+    | 'stream'
+    | 'actor_relocation'
+    | 'classic_fanout';
+  readonly messageKind: 'request' | 'send' | 'response' | 'error' | 'control';
+  readonly reason?: string;
+  readonly action?: string;
+  readonly packetName?: string;
+  readonly channelName?: string;
+  readonly channelRouteKind?: 'route_mesh' | 'client_server';
+  readonly meshName?: string;
+  readonly topic?: string;
+  readonly correlationId?: string;
+  readonly sourceRid?: string;
+  readonly targetRid?: string;
+  readonly serverRid?: string;
+  readonly flowId?: string;
+  readonly flowOrigin?: string;
+  readonly spotId?: string;
+  readonly instanceSpotType?: string;
+  readonly activationState?: string;
+  readonly actorId?: string;
+  readonly streamSessionId?: string;
+  readonly commandId?: number;
+  readonly messageSizeBytes?: number;
+  readonly durationSeconds?: number;
+  readonly errorType?: string;
+  readonly errorMessage?: string;
+  readonly errorCauseType?: string;
+  readonly errorCauseMessage?: string;
+}
+
+export type MessageFlowInput = Omit<
+  ZLinkRuntimeMessageFlowEvent,
+  'effectiveMode' | 'flowId' | 'flowOrigin'
+> & {
+  readonly severity?: 'warning';
+  readonly effectiveMode?: ZLinkMessageFlowLogMode;
+  readonly flowId?: string;
+  readonly flowOrigin?: import('../../contracts').ZLinkFlowOrigin;
+};
+
+export function toTelemetryRecord(
+  flow: MessageFlowInput,
+  messageSizeBytes: number | undefined,
+  durationSeconds: number | undefined
+): ZLinkTelemetryRecord {
+  return {
+    severity: flow.severity,
+    eventId:
+      flow.outcome === ZLinkMessageFlowOutcome.Error
+        ? DISPATCH_ERROR_EVENT_ID
+        : MESSAGE_FLOW_EVENT_ID,
+    timestamp: new Date(),
+    phase: messageFlowPhase(flow.outcome),
+    outcome: flow.result ?? messageFlowOutcome(flow.outcome),
+    surface: messageFlowSurface(flow.surface),
+    messageKind: messageFlowKind(flow.messageKind),
+    reason: flow.errorReason,
+    action: flow.errorAction,
+    packetName: flow.packetName,
+    channelName: flow.channelName,
+    channelRouteKind: flow.channelRouteKind ?? channelRouteKind(flow.surface),
+    meshName: flow.meshName,
+    topic: flow.topic,
+    correlationId: flow.correlationId,
+    sourceRid: flow.sourceRid,
+    targetRid: flow.targetRid,
+    serverRid: flow.serverRid,
+    flowId: flow.flowId,
+    flowOrigin: flow.flowOrigin === undefined ? undefined : FLOW_ORIGIN_VALUES[flow.flowOrigin],
+    spotId: flow.spotId,
+    instanceSpotType: flow.instanceSpotType,
+    activationState: flow.activationState,
+    actorId: flow.actorId,
+    streamSessionId: flow.streamSessionId,
+    commandId: flow.commandId,
+    messageSizeBytes,
+    durationSeconds,
+    errorType: flow.errorType,
+    errorMessage: flow.errorMessage,
+    errorCauseType: flow.errorCauseType,
+    errorCauseMessage: boundedText(flow.errorCauseMessage)
+  };
+}
+
+export function traceAttributes(record: ZLinkTelemetryRecord): Attributes {
+  const attributes: Attributes = {};
+  for (const [name, value] of Object.entries(record)) {
+    if (value === undefined || name === 'timestamp' || name === 'severity') continue;
+    attributes[toSnakeCase(name)] = value instanceof Date ? value.toISOString() : value;
+  }
+  return attributes;
+}
+
+export function structuredLogAttributes(record: ZLinkTelemetryRecord): Attributes {
+  //  기록의 attribute 이름은 Message flow tracing 3.2가 고정한다. 로그 본문의
+  //  `zlink flow:` 축약 key(`kind`, `mesh` …)와 다른 집합이다 —
+  //  structuredLogBodyAttributes를 참고.
+  return compactAttributes({
+    event_id: record.eventId,
+    timestamp: record.timestamp.toISOString(),
+    phase: record.phase,
+    surface: record.surface,
+    message_kind: record.messageKind,
+    mesh_name: record.meshName,
+    channel_name: record.channelName,
+    channel_route_kind: record.channelRouteKind,
+    source_rid: record.sourceRid,
+    target_rid: record.targetRid,
+    server_rid: record.serverRid,
+    packet_name: record.packetName,
+    topic: record.topic,
+    spot_id: record.spotId,
+    instance_spot_type: record.instanceSpotType,
+    activation_state: record.activationState,
+    actor_id: record.actorId,
+    stream_session_id: record.streamSessionId,
+    correlation_id: record.correlationId,
+    flow_id: record.flowId,
+    flow_origin: record.flowOrigin,
+    outcome: record.outcome,
+    reason: record.reason,
+    action: record.action,
+    command_id: record.commandId,
+    error_type: record.errorType,
+    error_message: record.errorMessage,
+    error_cause_type: record.errorCauseType,
+    error_cause_message: record.errorCauseMessage,
+    message_size_bytes: record.messageSizeBytes,
+    duration_seconds: record.durationSeconds
+  });
+}
+
+function structuredLogBodyAttributes(record: ZLinkTelemetryRecord): Attributes {
+  //  `zlink flow:` 본문 전용 축약 key — Message flow tracing "Structured log 대체 표기".
+  return compactAttributes({
+    event: record.eventId,
+    phase: record.phase,
+    surface: record.surface,
+    kind: record.messageKind,
+    mesh: record.meshName,
+    channel: record.channelName,
+    channel_route: record.channelRouteKind,
+    source_rid: record.sourceRid,
+    target_rid: record.targetRid,
+    server_rid: record.serverRid,
+    packet: record.packetName,
+    topic: record.topic,
+    spot: record.spotId,
+    instance_type: record.instanceSpotType,
+    activation_state: record.activationState,
+    actor: record.actorId,
+    session: record.streamSessionId,
+    corr: record.correlationId,
+    flow: record.flowId,
+    origin: record.flowOrigin,
+    outcome: record.outcome,
+    reason: record.reason,
+    action: record.action,
+    error_type: record.errorType,
+    error_message: record.errorMessage,
+    size: record.messageSizeBytes
+  });
+}
+
+export function structuredLogBody(record: ZLinkTelemetryRecord): string {
+  return `zlink flow: ${Object.entries(structuredLogBodyAttributes(record))
+    .map(([key, value]) => `${key}=${String(value)}`)
+    .join(' ')}`;
+}
+
+function compactAttributes(values: Record<string, string | number | undefined>): Attributes {
+  const attributes: Attributes = {};
+  for (const [key, value] of Object.entries(values)) {
+    if (value !== undefined) attributes[key] = value;
+  }
+  return attributes;
+}
+
+function messageFlowPhase(outcome: ZLinkMessageFlowOutcome): string | undefined {
+  switch (outcome) {
+    case ZLinkMessageFlowOutcome.Received:
+      return 'received';
+    case ZLinkMessageFlowOutcome.Admitted:
+      return 'admitted';
+    case ZLinkMessageFlowOutcome.Dispatched:
+      return 'dispatched';
+    case ZLinkMessageFlowOutcome.Completed:
+      return 'completed';
+    case ZLinkMessageFlowOutcome.Replied:
+      return 'replied';
+    case ZLinkMessageFlowOutcome.Dropped:
+      return 'dropped';
+    case ZLinkMessageFlowOutcome.Sent:
+      return 'sent';
+    case ZLinkMessageFlowOutcome.ReplyReceived:
+      return 'reply_received';
+    case ZLinkMessageFlowOutcome.Backpressured:
+      return 'backpressured';
+    case ZLinkMessageFlowOutcome.Error:
+      return undefined;
+  }
+}
+
+function messageFlowOutcome(outcome: ZLinkMessageFlowOutcome): ZLinkTelemetryRecord['outcome'] {
+  if (outcome === ZLinkMessageFlowOutcome.Error) return 'failed';
+  if (outcome === ZLinkMessageFlowOutcome.Backpressured) return 'backpressured';
+  if (outcome === ZLinkMessageFlowOutcome.Dropped) return 'dropped';
+  return 'succeeded';
+}
+
+function messageFlowSurface(surface: ZLinkDispatchErrorSurface): ZLinkTelemetryRecord['surface'] {
+  switch (surface) {
+    case ZLinkDispatchErrorSurface.Node:
+      return 'node';
+    case ZLinkDispatchErrorSurface.Channel:
+    case ZLinkDispatchErrorSurface.RouteMeshChannel:
+      return 'channel';
+    case ZLinkDispatchErrorSurface.SpotRoute:
+    case ZLinkDispatchErrorSurface.SpotSubscription:
+      return 'spot';
+    case ZLinkDispatchErrorSurface.InstanceSpot:
+      return 'instance_spot';
+    case ZLinkDispatchErrorSurface.SpotActor:
+      return 'actor';
+    case ZLinkDispatchErrorSurface.StreamSession:
+      return 'stream';
+    case ZLinkDispatchErrorSurface.ActorRelocation:
+      return 'actor_relocation';
+    case ZLinkDispatchErrorSurface.ClassicFanout:
+      return 'classic_fanout';
+  }
+}
+
+function messageFlowKind(kind: ZLinkDispatchMessageKind): ZLinkTelemetryRecord['messageKind'] {
+  switch (kind) {
+    case ZLinkDispatchMessageKind.Request:
+    case ZLinkDispatchMessageKind.ActorRequest:
+      return 'request';
+    case ZLinkDispatchMessageKind.Send:
+    case ZLinkDispatchMessageKind.ActorSend:
+      return 'send';
+    case ZLinkDispatchMessageKind.Response:
+      return 'response';
+    case ZLinkDispatchMessageKind.Error:
+      return 'error';
+    case ZLinkDispatchMessageKind.Control:
+      return 'control';
+    case ZLinkDispatchMessageKind.Publish:
+      throw new Error('Classic fanout publish must not create a normal message-flow record.');
+  }
+}
+
+function channelRouteKind(
+  surface: ZLinkDispatchErrorSurface
+): ZLinkTelemetryRecord['channelRouteKind'] {
+  if (surface === ZLinkDispatchErrorSurface.RouteMeshChannel) return 'route_mesh';
+  if (surface === ZLinkDispatchErrorSurface.Channel) return 'client_server';
+  return undefined;
+}
+
+function boundedText(value: string | undefined): string | undefined {
+  return value === undefined ? undefined : value.slice(0, ERROR_MESSAGE_MAX_LENGTH);
+}
+
+function toSnakeCase(value: string): string {
+  return value.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`);
+}

package/src/runtime/diagnostics/message-flow-worker.ts

@@ -0,0 +1,146 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import {
+  metrics,
+  trace,
+  ProxyTracerProvider,
+  SpanStatusCode,
+  type Tracer,
+  type Counter
+} from '@opentelemetry/api';
+import { logs, SeverityNumber, type Logger } from '@opentelemetry/api-logs';
+import { initialLoggerProvider } from './message-flow-bootstrap';
+import {
+  TelemetryState,
+  TelemetryCounter,
+  TELEMETRY_STATE_BYTES,
+  TELEMETRY_WORKER_AVAILABLE,
+  type ZLinkTelemetryDelivery
+} from './message-flow-delivery';
+import { COUNTER_UNITS, METRIC_NAMES } from './runtime-metrics';
+import { ZLinkMeters } from '../../contracts/Eventing/Metrics';
+import {
+  DISPATCH_ERROR_EVENT_ID,
+  TELEMETRY_INSTRUMENTATION_NAME,
+  structuredLogBody,
+  structuredLogAttributes,
+  traceAttributes,
+  type ZLinkTelemetryRecord
+} from './message-flow-record';
+
+const state = new Int32Array(workerData as SharedArrayBuffer, 0, TelemetryState.Length);
+const counters = new BigUint64Array(
+  workerData as SharedArrayBuffer,
+  TELEMETRY_STATE_BYTES,
+  TelemetryCounter.Length
+);
+let logger: Logger | undefined;
+let tracer: Tracer | undefined;
+let overflow: Counter | undefined;
+const overflowAttributes = { source: 'message_flow' };
+let reportedOverflow = 0n;
+
+function failed(
+  role: TelemetryCounter.LoggerFailures | TelemetryCounter.TraceFailures,
+  error: unknown
+): void {
+  if (
+    Atomics.add(counters, role, 1n) !== 0n ||
+    Atomics.load(counters, TelemetryCounter.FallbackFailures) !== 0n
+  )
+    return;
+  try {
+    const description = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
+    process.stderr.write(
+      `ZLink telemetry ${role === TelemetryCounter.LoggerFailures ? 'logger' : 'trace'} provider failed: ${description}\n`,
+      (error) => {
+        if (error !== undefined && error !== null)
+          Atomics.store(counters, TelemetryCounter.FallbackFailures, 1n);
+      }
+    );
+  } catch {
+    Atomics.store(counters, TelemetryCounter.FallbackFailures, 1n);
+  }
+}
+
+function deliver(record: ZLinkTelemetryRecord): void {
+  if (logger !== undefined) {
+    try {
+      const severity =
+        record.severity === 'warning'
+          ? 'WARN'
+          : record.eventId === DISPATCH_ERROR_EVENT_ID
+            ? 'ERROR'
+            : 'INFO';
+      logger.emit({
+        eventName: record.eventId,
+        timestamp: record.timestamp,
+        severityNumber: SeverityNumber[severity],
+        severityText: severity,
+        body: structuredLogBody(record),
+        attributes: structuredLogAttributes(record)
+      });
+    } catch (error) {
+      failed(TelemetryCounter.LoggerFailures, error);
+    }
+  }
+  if (tracer === undefined) return;
+  try {
+    const span = tracer.startSpan(record.eventId, { attributes: traceAttributes(record) });
+    if (record.eventId === DISPATCH_ERROR_EVENT_ID) {
+      span.setStatus({ code: SpanStatusCode.ERROR, message: record.reason });
+    }
+    span.end();
+  } catch (error) {
+    failed(TelemetryCounter.TraceFailures, error);
+  }
+}
+
+process.stderr.once('error', () => Atomics.store(counters, TelemetryCounter.FallbackFailures, 1n));
+const hasLoggerProvider = logs.getLoggerProvider() !== initialLoggerProvider;
+if (hasLoggerProvider) {
+  try {
+    logger = logs.getLogger(TELEMETRY_INSTRUMENTATION_NAME);
+  } catch (error) {
+    failed(TelemetryCounter.LoggerFailures, error);
+  }
+}
+let hasTraceProvider = false;
+try {
+  const provider = trace.getTracerProvider();
+  tracer =
+    provider instanceof ProxyTracerProvider
+      ? provider.getDelegateTracer(TELEMETRY_INSTRUMENTATION_NAME)
+      : provider.getTracer(TELEMETRY_INSTRUMENTATION_NAME);
+  hasTraceProvider = tracer !== undefined;
+} catch (error) {
+  // The no-provider proxy returns undefined. A delegate throwing getTracer is
+  // still a registered provider, and its failure stays in this worker.
+  hasTraceProvider = true;
+  failed(TelemetryCounter.TraceFailures, error);
+}
+
+// enabled() is an event filter, not a provider-registration test. In particular,
+// an ERROR-only processor may reject an unspecified-severity enabled() query.
+Atomics.store(state, TelemetryState.Enabled, hasLoggerProvider || hasTraceProvider ? 1 : 0);
+parentPort!.postMessage(TELEMETRY_WORKER_AVAILABLE);
+parentPort!.on('message', (batch: ZLinkTelemetryDelivery<ZLinkTelemetryRecord>[]) => {
+  for (const item of batch) {
+    Atomics.sub(state, TelemetryState.Pending, 1);
+    if (Atomics.load(new Int32Array(item.lease), 0) === 0) deliver(item.record);
+  }
+  const dropped = Atomics.load(counters, TelemetryCounter.Overflow);
+  if (dropped !== reportedOverflow) {
+    try {
+      overflow ??= metrics
+        .getMeter(ZLinkMeters.Framework)
+        .createCounter(METRIC_NAMES.ObservabilityEventsOverflow, {
+          unit: COUNTER_UNITS[METRIC_NAMES.ObservabilityEventsOverflow]
+        });
+      overflow.add(Number(dropped - reportedOverflow), overflowAttributes);
+    } catch (error) {
+      failed(TelemetryCounter.TraceFailures, error);
+    }
+    reportedOverflow = dropped;
+  }
+  parentPort!.postMessage(TELEMETRY_WORKER_AVAILABLE);
+});