@temelj/workflow 0.0.0-alpha.1

package/dist/otel.mjs

@@ -0,0 +1,588 @@
+import { optionalDurationBetweenMilliseconds } from "./temporal.mjs";
+import { WORKFLOW_ATTRIBUTES, WORKFLOW_INSTRUMENTATION_NAME, WORKFLOW_INSTRUMENTATION_VERSION, WORKFLOW_MESSAGING_ATTRIBUTES, WORKFLOW_METRIC_NAMES, WORKFLOW_SPAN_NAMES, WORKFLOW_SYSTEM } from "./telemetry.mjs";
+import { getEventDescriptor } from "./events/descriptors.mjs";
+import { WorkflowHistory } from "./history/mod.mjs";
+import { contextFromTraceContext, errorRecordToException, spanLinkFromTraceContext, streamStatusFromEventKind, toException, withRunSpanStatus, withSpanStatus } from "./otel/support.mjs";
+import { SpanStatusCode, context, metrics, propagation, trace } from "@opentelemetry/api";
+//#region src/otel.ts
+function createOpenTelemetryTelemetry(options = {}) {
+	return new OpenTelemetryWorkflowTelemetry(options);
+}
+var OpenTelemetryWorkflowTelemetry = class {
+	#tracer = trace.getTracer(WORKFLOW_INSTRUMENTATION_NAME, WORKFLOW_INSTRUMENTATION_VERSION);
+	#meter = metrics.getMeter(WORKFLOW_INSTRUMENTATION_NAME, WORKFLOW_INSTRUMENTATION_VERSION);
+	#options;
+	#runTransitions = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.runTransitions, { description: "Count of workflow run transitions." });
+	#runDuration = this.#meter.createHistogram(WORKFLOW_METRIC_NAMES.runDuration, {
+		description: "Workflow run duration in milliseconds.",
+		unit: "ms"
+	});
+	#eventsAppended = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.eventsAppended, { description: "Count of durable workflow events appended." });
+	#stepAttempts = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.stepAttempts, { description: "Count of workflow step attempt updates." });
+	#stepDuration = this.#meter.createHistogram(WORKFLOW_METRIC_NAMES.stepDuration, {
+		description: "Workflow step attempt duration in milliseconds.",
+		unit: "ms"
+	});
+	#messagesSent = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.messagesSent, { description: "Count of workflow messages sent." });
+	#messageWaitLatency = this.#meter.createHistogram(WORKFLOW_METRIC_NAMES.messageWaitLatency, {
+		description: "Latency from workflow message wait creation to message receipt.",
+		unit: "ms"
+	});
+	#hooksResumed = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.hooksResumed, { description: "Count of workflow hooks resumed." });
+	#hookResumeLatency = this.#meter.createHistogram(WORKFLOW_METRIC_NAMES.hookResumeLatency, {
+		description: "Latency from workflow hook wait creation to hook resume message receipt.",
+		unit: "ms"
+	});
+	#streamUpdates = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.streamUpdates, { description: "Count of workflow stream updates." });
+	#cleanupDeletedRuns = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.cleanupDeletedRuns, { description: "Count of workflow runs deleted by cleanup or retention operations." });
+	#runGauge;
+	#workerActiveExecutionsGauge;
+	#workerClaims = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.workerClaims, { description: "Count of workflow run claims by workers." });
+	#workerClaimFailures = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.workerClaimFailures, { description: "Count of failed targeted workflow run claims by workers." });
+	#workerHeartbeatFailures = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.workerHeartbeatFailures, { description: "Count of failed workflow worker heartbeat lease extensions." });
+	#workerLeaseExtensions = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.workerLeaseExtensions, { description: "Count of workflow run lease extension attempts." });
+	#workerLeaseReleases = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.workerLeaseReleases, { description: "Count of workflow run lease releases." });
+	#pendingToClaimedLatency = this.#meter.createHistogram(WORKFLOW_METRIC_NAMES.pendingToClaimedLatency, {
+		description: "Latency from workflow availability to worker claim in milliseconds.",
+		unit: "ms"
+	});
+	#waitingToResumedLatency = this.#meter.createHistogram(WORKFLOW_METRIC_NAMES.runWaitingToResumedLatency, {
+		description: "Latency from workflow waiting state to resumed execution in milliseconds.",
+		unit: "ms"
+	});
+	#retryScheduled = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.retryScheduled, { description: "Count of workflow retries scheduled." });
+	#retryExhausted = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.retryExhausted, { description: "Count of workflow retries exhausted into permanent failure." });
+	#retryDelay = this.#meter.createHistogram(WORKFLOW_METRIC_NAMES.retryDelay, {
+		description: "Delay until a scheduled workflow retry in milliseconds.",
+		unit: "ms"
+	});
+	#scheduleDueToRunCreatedLatency = this.#meter.createHistogram(WORKFLOW_METRIC_NAMES.scheduleDueToRunCreatedLatency, {
+		description: "Latency from recurring schedule fire time to scheduled run creation.",
+		unit: "ms"
+	});
+	#storageOperationDuration = this.#meter.createHistogram(WORKFLOW_METRIC_NAMES.storageOperationDuration, {
+		description: "Workflow storage/backend operation duration in milliseconds.",
+		unit: "ms"
+	});
+	#storageOperationErrors = this.#meter.createCounter(WORKFLOW_METRIC_NAMES.storageOperationErrors, { description: "Count of workflow storage/backend operation failures." });
+	#engine;
+	constructor(options) {
+		this.#options = options;
+		this.#runGauge = this.#meter.createObservableGauge(WORKFLOW_METRIC_NAMES.runs, { description: "Current count of workflow runs by status." });
+		this.#workerActiveExecutionsGauge = this.#meter.createObservableGauge(WORKFLOW_METRIC_NAMES.workerActiveExecutions, { description: "Current count of workflow runs actively held by workers." });
+		this.#runGauge.addCallback(async (result) => {
+			if (this.#engine?.getRunSummary === void 0) return;
+			try {
+				const summary = await this.#engine.getRunSummary();
+				for (const [status, count] of Object.entries(summary.status)) result.observe(count, this.#attributes({ [WORKFLOW_ATTRIBUTES.runStatus]: status }));
+			} catch {
+				return;
+			}
+		});
+		this.#workerActiveExecutionsGauge.addCallback(async (result) => {
+			if (this.#engine?.getRunSummary === void 0) return;
+			try {
+				const summary = await this.#engine.getRunSummary();
+				result.observe(summary.status.running, this.#attributes({ [WORKFLOW_ATTRIBUTES.runStatus]: "running" }));
+			} catch {
+				return;
+			}
+		});
+	}
+	captureContext() {
+		const carrier = {};
+		propagation.inject(context.active(), carrier);
+		return Object.keys(carrier).length === 0 ? void 0 : {
+			...carrier.traceparent === void 0 ? {} : { traceparent: carrier.traceparent },
+			...carrier.tracestate === void 0 ? {} : { tracestate: carrier.tracestate }
+		};
+	}
+	async withRunExecution(telemetryContext, callback) {
+		const parent = contextFromTraceContext(telemetryContext.traceContext);
+		return await this.#tracer.startActiveSpan(this.#runExecutionSpanName(telemetryContext), { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.workflowName]: telemetryContext.run.workflowName,
+			...telemetryContext.run.workflowVersion === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.workflowVersion]: telemetryContext.run.workflowVersion },
+			[WORKFLOW_ATTRIBUTES.runId]: telemetryContext.run.id,
+			[WORKFLOW_ATTRIBUTES.namespace]: telemetryContext.run.namespace,
+			...telemetryContext.run.attempts === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.runAttempt]: telemetryContext.run.attempts },
+			...telemetryContext.run.workerId === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.workerId]: telemetryContext.run.workerId }
+		}) }, parent, async (span) => await withRunSpanStatus(span, callback));
+	}
+	async withStepExecution(telemetryContext, callback) {
+		return await this.#tracer.startActiveSpan(this.#stepExecutionSpanName(telemetryContext), { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.workflowName]: telemetryContext.run.workflowName,
+			...telemetryContext.run.workflowVersion === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.workflowVersion]: telemetryContext.run.workflowVersion },
+			[WORKFLOW_ATTRIBUTES.runId]: telemetryContext.run.id,
+			[WORKFLOW_ATTRIBUTES.stepId]: telemetryContext.step.id,
+			[WORKFLOW_ATTRIBUTES.stepName]: telemetryContext.step.name,
+			[WORKFLOW_ATTRIBUTES.stepKind]: telemetryContext.step.kind,
+			[WORKFLOW_ATTRIBUTES.stepAttempt]: telemetryContext.attempt,
+			...telemetryContext.timeoutAt === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.stepTimeoutAt]: telemetryContext.timeoutAt.toString() }
+		}) }, async (span) => await withSpanStatus(span, callback));
+	}
+	async withStorageOperation(telemetryContext, callback) {
+		const startedAt = Date.now();
+		const attributes = this.#attributes({
+			[WORKFLOW_ATTRIBUTES.storageOperation]: telemetryContext.operation,
+			[WORKFLOW_ATTRIBUTES.storageBackend]: telemetryContext.backend
+		});
+		if (this.#options.includeStorageOperationSpans === false) try {
+			const result = await callback();
+			this.#recordStorageOperationResult(startedAt, attributes, "success");
+			return result;
+		} catch (error) {
+			this.#recordStorageOperationResult(startedAt, attributes, "failure");
+			throw error;
+		}
+		return await this.#tracer.startActiveSpan(WORKFLOW_SPAN_NAMES.storageOperation, { attributes }, async (span) => {
+			try {
+				const result = await callback();
+				this.#recordStorageOperationResult(startedAt, attributes, "success");
+				span.setAttribute(WORKFLOW_ATTRIBUTES.storageOperationStatus, "success");
+				span.setStatus({ code: SpanStatusCode.OK });
+				return result;
+			} catch (error) {
+				this.#recordStorageOperationResult(startedAt, attributes, "failure");
+				span.setAttribute(WORKFLOW_ATTRIBUTES.storageOperationStatus, "failure");
+				span.recordException(toException(error));
+				span.setStatus({ code: SpanStatusCode.ERROR });
+				throw error;
+			} finally {
+				span.end();
+			}
+		});
+	}
+	recordRunExecution(telemetryContext) {
+		const result = telemetryContext.result;
+		if (result?.kind === "completed" || result?.kind === "failed") {
+			const duration = optionalDurationBetweenMilliseconds(result.run.startedAt ?? result.run.createdAt, result.run.finishedAt);
+			if (duration !== void 0) this.#runDuration.record(duration, this.#attributes({
+				[WORKFLOW_ATTRIBUTES.workflowName]: result.run.workflowName,
+				...result.run.workflowVersion === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.workflowVersion]: result.run.workflowVersion },
+				[WORKFLOW_ATTRIBUTES.runStatus]: result.run.status
+			}));
+		}
+	}
+	observe(name, payload) {
+		switch (name) {
+			case "workflow:run-created":
+			case "workflow:run-waiting":
+			case "workflow:run-rescheduled":
+			case "workflow:run-manual-retry":
+			case "workflow:run-completed":
+			case "workflow:run-canceled": {
+				const event = payload;
+				this.#recordRunTransition(event);
+				break;
+			}
+			case "workflow:run-failed": {
+				const event = payload;
+				this.#recordRunTransition(event);
+				this.#recordRetryExhausted(event);
+				break;
+			}
+			case "workflow:run-permanently-failed": {
+				const event = payload;
+				this.#recordRunTransition(event);
+				this.#recordRetryExhausted(event);
+				break;
+			}
+			case "workflow:run-claimed": {
+				const event = payload;
+				this.#recordRunTransition(event);
+				const attributes = this.#runMetricAttributes(event, { [WORKFLOW_ATTRIBUTES.workerOperation]: event.reason });
+				this.#workerClaims.add(1, attributes);
+				const latency = optionalDurationBetweenMilliseconds(event.previousRun?.availableAt ?? event.run.availableAt ?? event.run.createdAt, event.timestamp);
+				if (latency !== void 0) this.#pendingToClaimedLatency.record(latency, attributes);
+				break;
+			}
+			case "workflow:run-started": {
+				const event = payload;
+				this.#recordRunTransition(event);
+				if (event.previousRun?.status !== "waiting") break;
+				const latency = optionalDurationBetweenMilliseconds(event.previousRun.availableAt ?? event.previousRun.lastTransitionAt, event.timestamp);
+				if (latency !== void 0) this.#waitingToResumedLatency.record(latency, this.#runMetricAttributes(event));
+				break;
+			}
+			case "workflow:run-lease-released": {
+				const event = payload;
+				this.#recordRunTransition(event);
+				this.#workerLeaseReleases.add(1, this.#runMetricAttributes(event));
+				break;
+			}
+			case "workflow:run-retrying": {
+				const event = payload;
+				this.#recordRunTransition(event);
+				const attributes = event.run.retryReason === void 0 ? this.#runMetricAttributes(event) : this.#runMetricAttributes(event, { [WORKFLOW_ATTRIBUTES.retryReason]: event.run.retryReason });
+				this.#retryScheduled.add(1, attributes);
+				const delay = optionalDurationBetweenMilliseconds(event.timestamp, event.run.retryAt);
+				if (delay !== void 0) this.#retryDelay.record(delay, attributes);
+				break;
+			}
+			case "workflow:durable-event-appended": {
+				const event = payload;
+				const descriptor = getEventDescriptor(event.event.kind);
+				this.#eventsAppended.add(1, this.#attributes({
+					[WORKFLOW_ATTRIBUTES.eventKind]: descriptor.kind,
+					[WORKFLOW_ATTRIBUTES.eventCategory]: descriptor.category
+				}));
+				const history = new WorkflowHistory(event.events);
+				this.#recordDurableEventLatency(event, history);
+				this.#recordDurableSleepSpan(event, history);
+				this.#recordChildWorkflowOperationSpan(event, history);
+				break;
+			}
+			case "workflow:step-attempt-updated": {
+				const event = payload;
+				const attributes = this.#attributes({
+					[WORKFLOW_ATTRIBUTES.stepKind]: event.attempt.kind,
+					[WORKFLOW_ATTRIBUTES.stepStatus]: event.attempt.status,
+					...this.#options.includeStepNameMetricAttribute ? { [WORKFLOW_ATTRIBUTES.stepName]: event.attempt.stepName } : {}
+				});
+				this.#stepAttempts.add(1, attributes);
+				if (event.attempt.status === "completed" || event.attempt.status === "failed") {
+					const duration = optionalDurationBetweenMilliseconds(event.attempt.startedAt, event.attempt.finishedAt);
+					if (duration !== void 0) this.#stepDuration.record(duration, attributes);
+				}
+				break;
+			}
+			case "workflow:message-sent": {
+				const event = payload;
+				this.#messagesSent.add(1, this.#attributes({}));
+				this.#recordMessageSentSpan(event);
+				break;
+			}
+			case "workflow:hook-resumed": {
+				const event = payload;
+				this.#hooksResumed.add(1, this.#attributes({ [WORKFLOW_ATTRIBUTES.hookKind]: "hook" }));
+				this.#recordHookResumeSpan("hook", event);
+				break;
+			}
+			case "workflow:webhook-resumed": {
+				const event = payload;
+				this.#hooksResumed.add(1, this.#attributes({ [WORKFLOW_ATTRIBUTES.hookKind]: "webhook" }));
+				this.#recordHookResumeSpan("webhook", event);
+				break;
+			}
+			case "workflow:stream-updated": {
+				const event = payload;
+				this.#streamUpdates.add(1, this.#attributes({
+					[WORKFLOW_ATTRIBUTES.streamEventKind]: event.eventKind,
+					[WORKFLOW_ATTRIBUTES.streamStatus]: streamStatusFromEventKind(event.eventKind)
+				}));
+				this.#recordStreamOperationSpan(event);
+				break;
+			}
+			case "workflow:child-workflow-started": {
+				const event = payload;
+				this.#recordChildWorkflowStartedSpan(event);
+				break;
+			}
+			case "workflow:cleanup-completed": {
+				const event = payload;
+				this.#cleanupDeletedRuns.add(event.result.deletedRuns, this.#attributes({ [WORKFLOW_ATTRIBUTES.cleanupSource]: "cleanup" }));
+				this.#recordCleanupOperationSpan("cleanup", event);
+				break;
+			}
+			case "workflow:retention-applied": {
+				const event = payload;
+				this.#cleanupDeletedRuns.add(event.result.deletedRuns, this.#attributes({ [WORKFLOW_ATTRIBUTES.cleanupSource]: "retention" }));
+				this.#recordCleanupOperationSpan("retention", event);
+				break;
+			}
+			case "workflow:worker-operation": {
+				const event = payload;
+				const attributes = this.#attributes({
+					[WORKFLOW_ATTRIBUTES.workerOperation]: event.operation,
+					[WORKFLOW_ATTRIBUTES.workerOperationStatus]: event.status
+				});
+				if (event.operation === "claim" && event.status === "failure") this.#workerClaimFailures.add(1, attributes);
+				if (event.operation === "heartbeat" && event.status === "failure") this.#workerHeartbeatFailures.add(1, attributes);
+				if (event.operation === "lease_extension") this.#workerLeaseExtensions.add(1, attributes);
+				if (event.operation === "lease_release") this.#workerLeaseReleases.add(1, attributes);
+				this.#recordWorkerOperationSpan(event);
+				break;
+			}
+			case "workflow:schedule-ticked": {
+				const event = payload;
+				this.#recordScheduleTickSpan(event);
+				break;
+			}
+			case "workflow:schedule-run-created": {
+				const event = payload;
+				const attributes = this.#attributes({
+					[WORKFLOW_ATTRIBUTES.workflowName]: event.workflowName,
+					...event.workflowVersion === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.workflowVersion]: event.workflowVersion }
+				});
+				const latency = optionalDurationBetweenMilliseconds(event.fireAt, event.timestamp);
+				if (latency !== void 0) this.#scheduleDueToRunCreatedLatency.record(latency, attributes);
+				this.#recordScheduleRunCreatedSpan(event);
+				break;
+			}
+		}
+	}
+	bindEngine(engine) {
+		this.#engine = engine;
+	}
+	async shutdown() {
+		const errors = [];
+		try {
+			await this.#options.forceFlush?.();
+		} catch (error) {
+			errors.push(error);
+		}
+		try {
+			await this.#options.shutdown?.();
+		} catch (error) {
+			errors.push(error);
+		}
+		if (errors.length > 0) throw errors[0];
+	}
+	#recordRunTransition(event) {
+		this.#runTransitions.add(1, this.#runMetricAttributes(event));
+	}
+	#recordRetryExhausted(event) {
+		const retryReason = event.previousRun?.retryReason ?? ((event.previousRun?.attempts ?? event.run.attempts ?? 1) > 1 ? "workflow" : void 0);
+		if (retryReason === void 0) return;
+		this.#retryExhausted.add(1, this.#runMetricAttributes(event, {
+			[WORKFLOW_ATTRIBUTES.retryReason]: retryReason,
+			[WORKFLOW_ATTRIBUTES.retryExhausted]: true
+		}));
+	}
+	#attributes(attributes) {
+		return {
+			...this.#options.serviceNamespace === void 0 ? {} : { "service.namespace": this.#options.serviceNamespace },
+			[WORKFLOW_ATTRIBUTES.system]: WORKFLOW_SYSTEM,
+			...attributes
+		};
+	}
+	#runExecutionSpanName(telemetryContext) {
+		return this.#options.includeNamesInSpanNames === true ? `${WORKFLOW_SPAN_NAMES.runExecution} ${telemetryContext.run.workflowName}` : WORKFLOW_SPAN_NAMES.runExecution;
+	}
+	#stepExecutionSpanName(telemetryContext) {
+		return this.#options.includeNamesInSpanNames === true ? `${WORKFLOW_SPAN_NAMES.stepExecution} ${telemetryContext.step.name}` : WORKFLOW_SPAN_NAMES.stepExecution;
+	}
+	#recordStorageOperationResult(startedAt, attributes, status) {
+		const resultAttributes = {
+			...attributes,
+			[WORKFLOW_ATTRIBUTES.storageOperationStatus]: status
+		};
+		this.#storageOperationDuration.record(Date.now() - startedAt, resultAttributes);
+		if (status === "failure") this.#storageOperationErrors.add(1, resultAttributes);
+	}
+	#runMetricAttributes(event, attributes = {}) {
+		return this.#attributes({
+			[WORKFLOW_ATTRIBUTES.workflowName]: event.workflowName,
+			...event.workflowVersion === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.workflowVersion]: event.workflowVersion },
+			[WORKFLOW_ATTRIBUTES.runStatus]: event.status,
+			[WORKFLOW_ATTRIBUTES.runTransition]: event.reason,
+			...attributes
+		});
+	}
+	#recordDurableEventLatency(event, history) {
+		if (event.event.kind !== "message_wait_completed") return;
+		const completed = event.event;
+		const started = history.startedMessageWaitForCompleted(completed);
+		if (started === void 0) return;
+		const latency = optionalDurationBetweenMilliseconds(started.timestamp, completed.messageTimestamp);
+		if (latency === void 0) return;
+		const source = started.source ?? "message";
+		const attributes = this.#attributes({ [WORKFLOW_ATTRIBUTES.messageSource]: source });
+		this.#messageWaitLatency.record(latency, attributes);
+		if (source === "hook" || source === "webhook") this.#hookResumeLatency.record(latency, this.#attributes({ [WORKFLOW_ATTRIBUTES.hookKind]: source }));
+		this.#recordMessageWaitSpan(completed, started, history);
+	}
+	#recordDurableSleepSpan(event, history) {
+		if (event.event.kind !== "sleep_completed") return;
+		const completed = event.event;
+		const started = history.startedSleepForCompleted(completed);
+		if (started === void 0) return;
+		const requestedDuration = started.duration === void 0 ? void 0 : started.duration.total({ unit: "millisecond" });
+		const actualDuration = optionalDurationBetweenMilliseconds(started.timestamp, completed.timestamp);
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.sleep, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.runId]: event.runId,
+			[WORKFLOW_ATTRIBUTES.stepId]: completed.stepId,
+			[WORKFLOW_ATTRIBUTES.stepName]: completed.stepName,
+			[WORKFLOW_ATTRIBUTES.stepKind]: "sleep",
+			[WORKFLOW_ATTRIBUTES.sleepUntil]: started.until.toString(),
+			...requestedDuration === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.sleepDurationMs]: requestedDuration },
+			...actualDuration === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.sleepActualMs]: actualDuration }
+		}) });
+		span.setStatus({ code: SpanStatusCode.OK });
+		span.end();
+	}
+	#recordChildWorkflowOperationSpan(event, history) {
+		if (event.event.kind !== "child_workflow_completed" && event.event.kind !== "child_workflow_failed") return;
+		const terminal = event.event;
+		const started = history.startedChildWorkflowForTerminal(terminal);
+		const duration = started === void 0 ? void 0 : optionalDurationBetweenMilliseconds(started.timestamp, terminal.timestamp);
+		const failed = terminal.kind === "child_workflow_failed";
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.childWorkflowOperation, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.runId]: event.runId,
+			[WORKFLOW_ATTRIBUTES.stepId]: terminal.stepId,
+			[WORKFLOW_ATTRIBUTES.stepName]: terminal.stepName,
+			[WORKFLOW_ATTRIBUTES.stepKind]: "workflow",
+			[WORKFLOW_ATTRIBUTES.stepAttempt]: terminal.attempt,
+			[WORKFLOW_ATTRIBUTES.childWorkflowOperation]: failed ? "fail" : "complete",
+			[WORKFLOW_ATTRIBUTES.childWorkflowRunId]: terminal.childRunId,
+			...started === void 0 ? {} : {
+				[WORKFLOW_ATTRIBUTES.childWorkflowName]: started.workflowName,
+				...started.workflowVersion === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.childWorkflowVersion]: started.workflowVersion }
+			},
+			[WORKFLOW_ATTRIBUTES.childWorkflowStatus]: failed ? "failed" : "completed",
+			...duration === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.childWorkflowDurationMs]: duration },
+			...failed ? { [WORKFLOW_ATTRIBUTES.errorKind]: terminal.error.name } : {}
+		}) });
+		if (failed) {
+			span.recordException(errorRecordToException(terminal.error));
+			span.setStatus({
+				code: SpanStatusCode.ERROR,
+				message: terminal.error.message
+			});
+		} else span.setStatus({ code: SpanStatusCode.OK });
+		span.end();
+	}
+	#recordChildWorkflowStartedSpan(event) {
+		const started = event.event;
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.childWorkflowOperation, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.runId]: event.runId,
+			[WORKFLOW_ATTRIBUTES.stepId]: started.stepId,
+			[WORKFLOW_ATTRIBUTES.stepName]: started.stepName,
+			[WORKFLOW_ATTRIBUTES.stepKind]: "workflow",
+			[WORKFLOW_ATTRIBUTES.stepAttempt]: started.attempt,
+			[WORKFLOW_ATTRIBUTES.childWorkflowOperation]: "start",
+			[WORKFLOW_ATTRIBUTES.childWorkflowRunId]: started.childRunId,
+			[WORKFLOW_ATTRIBUTES.childWorkflowName]: started.workflowName,
+			...started.workflowVersion === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.childWorkflowVersion]: started.workflowVersion },
+			[WORKFLOW_ATTRIBUTES.childWorkflowStatus]: "started"
+		}) });
+		span.setStatus({ code: SpanStatusCode.OK });
+		span.end();
+	}
+	#recordMessageSentSpan(event) {
+		const parent = contextFromTraceContext(event.traceContext);
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.messageSend, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.runId]: event.runId,
+			[WORKFLOW_ATTRIBUTES.messageId]: event.messageId,
+			[WORKFLOW_MESSAGING_ATTRIBUTES.system]: "temelj.workflow",
+			[WORKFLOW_MESSAGING_ATTRIBUTES.destinationName]: event.runId,
+			[WORKFLOW_MESSAGING_ATTRIBUTES.messageId]: event.messageId,
+			[WORKFLOW_MESSAGING_ATTRIBUTES.operationName]: "send",
+			[WORKFLOW_MESSAGING_ATTRIBUTES.operationType]: "publish"
+		}) }, parent);
+		span.setStatus({ code: SpanStatusCode.OK });
+		span.end();
+	}
+	#recordHookResumeSpan(kind, event) {
+		const parent = contextFromTraceContext(event.traceContext);
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.hookResume, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.hookKind]: kind,
+			[WORKFLOW_ATTRIBUTES.runId]: event.runId,
+			[WORKFLOW_ATTRIBUTES.messageId]: event.messageId,
+			[WORKFLOW_MESSAGING_ATTRIBUTES.system]: "temelj.workflow",
+			[WORKFLOW_MESSAGING_ATTRIBUTES.destinationName]: event.runId,
+			[WORKFLOW_MESSAGING_ATTRIBUTES.messageId]: event.messageId,
+			[WORKFLOW_MESSAGING_ATTRIBUTES.operationName]: "resume",
+			[WORKFLOW_MESSAGING_ATTRIBUTES.operationType]: "publish"
+		}) }, parent);
+		span.setStatus({ code: SpanStatusCode.OK });
+		span.end();
+	}
+	#recordScheduleRunCreatedSpan(event) {
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.scheduleRunCreate, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.workflowName]: event.workflowName,
+			...event.workflowVersion === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.workflowVersion]: event.workflowVersion },
+			[WORKFLOW_ATTRIBUTES.scheduleId]: event.scheduleId,
+			[WORKFLOW_ATTRIBUTES.scheduleFireAt]: event.fireAt.toString(),
+			[WORKFLOW_ATTRIBUTES.runId]: event.runId
+		}) });
+		span.setStatus({ code: SpanStatusCode.OK });
+		span.end();
+	}
+	#recordScheduleTickSpan(event) {
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.scheduleTick, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.scheduleTickedRuns]: event.result.ticked,
+			[WORKFLOW_ATTRIBUTES.scheduleTickedSchedules]: event.result.scheduleIds.length,
+			...event.result.nextScheduleAt === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.scheduleNextAt]: event.result.nextScheduleAt.toString() }
+		}) });
+		span.setStatus({ code: SpanStatusCode.OK });
+		span.end();
+	}
+	#recordWorkerOperationSpan(event) {
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.workerOperation, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.workerId]: event.workerId,
+			...event.runId === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.runId]: event.runId },
+			...event.run === void 0 ? {} : {
+				[WORKFLOW_ATTRIBUTES.workflowName]: event.run.workflowName,
+				...event.run.workflowVersion === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.workflowVersion]: event.run.workflowVersion },
+				[WORKFLOW_ATTRIBUTES.runStatus]: event.run.status
+			},
+			[WORKFLOW_ATTRIBUTES.workerOperation]: event.operation,
+			[WORKFLOW_ATTRIBUTES.workerOperationStatus]: event.status
+		}) });
+		if (event.status === "failure") {
+			span.setStatus({ code: SpanStatusCode.ERROR });
+			if (event.error !== void 0) span.recordException(toException(event.error));
+		} else span.setStatus({ code: SpanStatusCode.OK });
+		span.end();
+	}
+	#recordCleanupOperationSpan(source, event) {
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.cleanupOperation, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.cleanupSource]: source,
+			[WORKFLOW_ATTRIBUTES.cleanupDeletedRuns]: event.result.deletedRuns,
+			[WORKFLOW_ATTRIBUTES.cleanupCreatedMarkers]: event.result.createdCleanupMarkers,
+			[WORKFLOW_ATTRIBUTES.cleanupDeletedMarkers]: event.result.deletedCleanupMarkers,
+			[WORKFLOW_ATTRIBUTES.cleanupDeletedEvents]: event.result.deletedEvents,
+			[WORKFLOW_ATTRIBUTES.cleanupDeletedStepAttempts]: event.result.deletedStepAttempts,
+			[WORKFLOW_ATTRIBUTES.cleanupDeletedIdempotencyKeys]: event.result.deletedIdempotencyKeys,
+			[WORKFLOW_ATTRIBUTES.cleanupDeletedMessageIdempotencyKeys]: event.result.deletedMessageIdempotencyKeys,
+			...event.result.dryRun === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.cleanupDryRun]: event.result.dryRun },
+			[WORKFLOW_ATTRIBUTES.cleanupMatchedRuns]: event.result.matchedRuns ?? event.result.runIds.length
+		}) });
+		span.setStatus({ code: SpanStatusCode.OK });
+		span.end();
+	}
+	#recordStreamOperationSpan(event) {
+		const span = this.#tracer.startSpan(WORKFLOW_SPAN_NAMES.streamOperation, { attributes: this.#attributes({
+			[WORKFLOW_ATTRIBUTES.runId]: event.runId,
+			[WORKFLOW_ATTRIBUTES.streamId]: event.streamId,
+			[WORKFLOW_ATTRIBUTES.streamEventKind]: event.eventKind,
+			[WORKFLOW_ATTRIBUTES.streamStatus]: streamStatusFromEventKind(event.eventKind),
+			[WORKFLOW_ATTRIBUTES.stepId]: event.stepId,
+			[WORKFLOW_ATTRIBUTES.stepName]: event.stepName,
+			...event.index === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.streamIndex]: event.index },
+			...event.error === void 0 ? {} : { [WORKFLOW_ATTRIBUTES.errorKind]: event.error.name }
+		}) });
+		if (event.error === void 0) span.setStatus({ code: SpanStatusCode.OK });
+		else {
+			span.recordException(errorRecordToException(event.error));
+			span.setStatus({
+				code: SpanStatusCode.ERROR,
+				message: event.error.message
+			});
+		}
+		span.end();
+	}
+	#recordMessageWaitSpan(completed, started, history) {
+		const message = history.messageSentByTimestamp(completed.messageId, completed.messageTimestamp);
+		const link = spanLinkFromTraceContext(completed.telemetryContext ?? message?.telemetryContext);
+		return this.#tracer.startActiveSpan(WORKFLOW_SPAN_NAMES.messageWait, {
+			attributes: this.#attributes({
+				[WORKFLOW_ATTRIBUTES.messageId]: completed.messageId,
+				[WORKFLOW_ATTRIBUTES.messageSource]: started.source ?? "message",
+				[WORKFLOW_ATTRIBUTES.stepId]: completed.stepId,
+				[WORKFLOW_ATTRIBUTES.stepName]: completed.stepName,
+				[WORKFLOW_MESSAGING_ATTRIBUTES.system]: "temelj.workflow",
+				[WORKFLOW_MESSAGING_ATTRIBUTES.messageId]: completed.messageId,
+				[WORKFLOW_MESSAGING_ATTRIBUTES.operationName]: "receive",
+				[WORKFLOW_MESSAGING_ATTRIBUTES.operationType]: "receive"
+			}),
+			...link === void 0 ? {} : { links: [link] }
+		}, (span) => {
+			span.setStatus({ code: SpanStatusCode.OK });
+			span.end();
+		});
+	}
+};
+//#endregion
+export { createOpenTelemetryTelemetry };

package/dist/pagination.d.mts

@@ -0,0 +1,25 @@
+import { StepId } from "./types/step-id.mjs";
+import { CursorKeyPart, PageOptions, WorkflowPage } from "./types/pagination.mjs";
+import "./types/step.mjs";
+//#region src/pagination.d.ts
+declare function pageItems<TItem>(items: readonly TItem[], options: PageOptions | undefined, label: string, cursorKey: (item: TItem, index: number) => readonly CursorKeyPart[]): WorkflowPage<TItem>;
+declare function makeWorkflowRunCursorKey(run: {
+  readonly createdAt: Temporal.Instant;
+  readonly id: string;
+}): readonly [number, string];
+declare function makeWorkflowEventCursorKey(_event: unknown, index: number): readonly [number];
+declare function makeWorkflowStepAttemptCursorKey(attempt: {
+  readonly startedAt: Temporal.Instant;
+  readonly stepId: StepId;
+  readonly attempt: number;
+}): readonly [number, StepId, number];
+declare function makeScheduleCursorKey(schedule: {
+  readonly nextFireAt: Temporal.Instant;
+  readonly createdAt: Temporal.Instant;
+  readonly id: string;
+}): readonly [number, number, string];
+declare function makeStreamChunkCursorKey(chunk: {
+  readonly index: number;
+}): readonly [number];
+//#endregion
+export { makeScheduleCursorKey, makeStreamChunkCursorKey, makeWorkflowEventCursorKey, makeWorkflowRunCursorKey, makeWorkflowStepAttemptCursorKey, pageItems };