tasker-rb 0.1.1

data/ext/tasker_core/src/observability_ffi.rs

@@ -0,0 +1,339 @@
+//! # Observability FFI Module
+//!
+//! TAS-77: Exposes worker observability services (Health, Metrics, Templates, Config)
+//! via Ruby FFI, enabling applications to access the same data available through
+//! the HTTP API without running a web server.
+//!
+//! ## Design Approach
+//!
+//! Complex response types are serialized to JSON strings that Ruby can parse with
+//! `JSON.parse`. This provides a clean interface and avoids manual hash building
+//! for deeply nested structures.
+//!
+//! ## Usage
+//!
+//! ```ruby
+//! # Get health status
+//! health = JSON.parse(TaskerCore::FFI.health_basic)
+//! health = JSON.parse(TaskerCore::FFI.health_detailed)
+//! ready = JSON.parse(TaskerCore::FFI.health_ready)
+//! live = JSON.parse(TaskerCore::FFI.health_live)
+//!
+//! # Get metrics
+//! metrics = JSON.parse(TaskerCore::FFI.metrics_worker)
+//! events = JSON.parse(TaskerCore::FFI.metrics_events)
+//! prometheus = TaskerCore::FFI.metrics_prometheus  # Already text format
+//!
+//! # Query templates
+//! templates = JSON.parse(TaskerCore::FFI.templates_list(true))
+//! template = JSON.parse(TaskerCore::FFI.template_get("namespace", "name", "version"))
+//! validation = JSON.parse(TaskerCore::FFI.template_validate("namespace", "name", "version"))
+//!
+//! # Query configuration
+//! config = JSON.parse(TaskerCore::FFI.config_runtime)
+//! env = TaskerCore::FFI.config_environment  # Simple string
+//! ```
+
+use crate::bridge::WORKER_SYSTEM;
+use magnus::{function, prelude::*, Error, ExceptionClass, RModule, Ruby};
+use tracing::{debug, error};
+
+/// Helper to get RuntimeError exception class
+fn runtime_error_class() -> ExceptionClass {
+    Ruby::get()
+        .expect("Ruby runtime should be available")
+        .exception_runtime_error()
+}
+
+/// Helper to get web_state from worker system, returning error if unavailable
+fn with_web_state<F, T>(f: F) -> Result<T, Error>
+where
+    F: FnOnce(
+        &crate::bridge::RubyBridgeHandle,
+        &tasker_worker::web::WorkerWebState,
+    ) -> Result<T, Error>,
+{
+    let handle_guard = WORKER_SYSTEM.lock().map_err(|e| {
+        error!("Failed to acquire worker system lock: {}", e);
+        Error::new(runtime_error_class(), "Lock acquisition failed")
+    })?;
+
+    let handle = handle_guard
+        .as_ref()
+        .ok_or_else(|| Error::new(runtime_error_class(), "Worker system not running"))?;
+
+    let web_state = handle.system_handle.web_state.as_ref().ok_or_else(|| {
+        Error::new(
+            runtime_error_class(),
+            "Web state not available (web API may be disabled)",
+        )
+    })?;
+
+    f(handle, web_state)
+}
+
+/// Helper to serialize a value to JSON, with error conversion
+fn to_json<T: serde::Serialize>(value: &T) -> Result<String, Error> {
+    serde_json::to_string(value).map_err(|e| {
+        Error::new(
+            runtime_error_class(),
+            format!("Failed to serialize response: {}", e),
+        )
+    })
+}
+
+// =============================================================================
+// Health Service FFI Functions
+// =============================================================================
+
+/// Get basic health status as JSON string
+///
+/// Returns JSON with:
+/// - `status`: "healthy" or "unhealthy"
+/// - `worker_id`: Worker identifier
+/// - `timestamp`: ISO-8601 timestamp
+pub fn health_basic() -> Result<String, Error> {
+    debug!("FFI: health_basic called");
+
+    with_web_state(|_handle, web_state| {
+        let response = web_state.health_service().basic_health();
+        to_json(&response)
+    })
+}
+
+/// Get liveness status as JSON (Kubernetes liveness probe equivalent)
+pub fn health_live() -> Result<String, Error> {
+    debug!("FFI: health_live called");
+
+    with_web_state(|_handle, web_state| {
+        let response = web_state.health_service().liveness();
+        to_json(&response)
+    })
+}
+
+/// Get readiness status as JSON (Kubernetes readiness probe equivalent)
+///
+/// Returns JSON with detailed health checks.
+/// Check the `status` field to determine if the worker is ready.
+pub fn health_ready() -> Result<String, Error> {
+    debug!("FFI: health_ready called");
+
+    with_web_state(|handle, web_state| {
+        let response = handle
+            .runtime
+            .block_on(async { web_state.health_service().readiness().await });
+
+        // Return the response regardless of Ok/Err - both contain DetailedHealthResponse
+        match response {
+            Ok(detailed) => to_json(&detailed),
+            Err(detailed) => to_json(&detailed),
+        }
+    })
+}
+
+/// Get detailed health information as JSON
+pub fn health_detailed() -> Result<String, Error> {
+    debug!("FFI: health_detailed called");
+
+    with_web_state(|handle, web_state| {
+        let response = handle
+            .runtime
+            .block_on(async { web_state.health_service().detailed_health().await });
+
+        to_json(&response)
+    })
+}
+
+// =============================================================================
+// Metrics Service FFI Functions
+// =============================================================================
+
+/// Get worker metrics as JSON string
+pub fn metrics_worker() -> Result<String, Error> {
+    debug!("FFI: metrics_worker called");
+
+    with_web_state(|handle, web_state| {
+        let response = handle
+            .runtime
+            .block_on(async { web_state.metrics_service().worker_metrics().await });
+
+        to_json(&response)
+    })
+}
+
+/// Get domain event statistics as JSON
+pub fn metrics_events() -> Result<String, Error> {
+    debug!("FFI: metrics_events called");
+
+    with_web_state(|handle, web_state| {
+        let response = handle
+            .runtime
+            .block_on(async { web_state.metrics_service().domain_event_stats().await });
+
+        to_json(&response)
+    })
+}
+
+/// Get Prometheus-formatted metrics string
+///
+/// This returns the raw Prometheus text format, not JSON.
+pub fn metrics_prometheus() -> Result<String, Error> {
+    debug!("FFI: metrics_prometheus called");
+
+    with_web_state(|handle, web_state| {
+        let response = handle
+            .runtime
+            .block_on(async { web_state.metrics_service().prometheus_format().await });
+
+        Ok(response)
+    })
+}
+
+// =============================================================================
+// Template Query Service FFI Functions
+// =============================================================================
+
+/// List all available templates as JSON
+///
+/// # Arguments
+/// * `include_cache_stats` - Whether to include cache statistics
+pub fn templates_list(include_cache_stats: bool) -> Result<String, Error> {
+    debug!("FFI: templates_list called");
+
+    with_web_state(|handle, web_state| {
+        let response = handle.runtime.block_on(async {
+            web_state
+                .template_query_service()
+                .list_templates(include_cache_stats)
+                .await
+        });
+
+        to_json(&response)
+    })
+}
+
+/// Get a specific template by namespace/name/version as JSON
+pub fn template_get(namespace: String, name: String, version: String) -> Result<String, Error> {
+    debug!(
+        "FFI: template_get called for {}/{}/{}",
+        namespace, name, version
+    );
+
+    with_web_state(|handle, web_state| {
+        let response = handle.runtime.block_on(async {
+            web_state
+                .template_query_service()
+                .get_template(&namespace, &name, &version)
+                .await
+        });
+
+        match response {
+            Ok(template) => to_json(&template),
+            Err(e) => Err(Error::new(runtime_error_class(), e.to_string())),
+        }
+    })
+}
+
+/// Validate a template for worker execution as JSON
+pub fn template_validate(
+    namespace: String,
+    name: String,
+    version: String,
+) -> Result<String, Error> {
+    debug!(
+        "FFI: template_validate called for {}/{}/{}",
+        namespace, name, version
+    );
+
+    with_web_state(|handle, web_state| {
+        let response = handle.runtime.block_on(async {
+            web_state
+                .template_query_service()
+                .validate_template(&namespace, &name, &version)
+                .await
+        });
+
+        match response {
+            Ok(validation) => to_json(&validation),
+            Err(e) => Err(Error::new(runtime_error_class(), e.to_string())),
+        }
+    })
+}
+
+/// Get template cache statistics as JSON
+pub fn templates_cache_stats() -> Result<String, Error> {
+    debug!("FFI: templates_cache_stats called");
+
+    with_web_state(|handle, web_state| {
+        let stats = handle
+            .runtime
+            .block_on(async { web_state.template_query_service().cache_stats().await });
+
+        to_json(&stats)
+    })
+}
+
+// TAS-169: Removed templates_cache_clear and template_refresh FFI functions.
+// Cache operations are now internal-only. Restart worker to refresh templates.
+// Distributed cache status is available via health_detailed().
+
+// =============================================================================
+// Config Query Service FFI Functions
+// =============================================================================
+
+/// Get runtime configuration as JSON (safe fields only, no secrets)
+pub fn config_runtime() -> Result<String, Error> {
+    debug!("FFI: config_runtime called");
+
+    with_web_state(|_handle, web_state| {
+        let response = web_state.config_query_service().runtime_config();
+
+        match response {
+            Ok(config) => to_json(&config),
+            Err(e) => Err(Error::new(runtime_error_class(), e.to_string())),
+        }
+    })
+}
+
+/// Get the current environment name (simple string, not JSON)
+pub fn config_environment() -> Result<String, Error> {
+    debug!("FFI: config_environment called");
+
+    with_web_state(|_handle, web_state| {
+        Ok(web_state.config_query_service().environment().to_string())
+    })
+}
+
+// =============================================================================
+// Module Initialization
+// =============================================================================
+
+/// Initialize the observability FFI module
+pub fn init_observability_ffi(module: &RModule) -> Result<(), Error> {
+    tracing::info!("🔍 Initializing observability FFI module (TAS-77)");
+
+    // Health endpoints
+    module.define_singleton_method("health_basic", function!(health_basic, 0))?;
+    module.define_singleton_method("health_live", function!(health_live, 0))?;
+    module.define_singleton_method("health_ready", function!(health_ready, 0))?;
+    module.define_singleton_method("health_detailed", function!(health_detailed, 0))?;
+
+    // Metrics endpoints
+    module.define_singleton_method("metrics_worker", function!(metrics_worker, 0))?;
+    module.define_singleton_method("metrics_events", function!(metrics_events, 0))?;
+    module.define_singleton_method("metrics_prometheus", function!(metrics_prometheus, 0))?;
+
+    // Template endpoints
+    // TAS-169: Removed templates_cache_clear and template_refresh (cache ops are internal-only)
+    module.define_singleton_method("templates_list", function!(templates_list, 1))?;
+    module.define_singleton_method("template_get", function!(template_get, 3))?;
+    module.define_singleton_method("template_validate", function!(template_validate, 3))?;
+    module.define_singleton_method("templates_cache_stats", function!(templates_cache_stats, 0))?;
+
+    // Config endpoints
+    module.define_singleton_method("config_runtime", function!(config_runtime, 0))?;
+    module.define_singleton_method("config_environment", function!(config_environment, 0))?;
+
+    tracing::info!("✅ Observability FFI module initialized");
+    Ok(())
+}

data/lib/tasker_core/batch_processing/batch_aggregation_scenario.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module TaskerCore
+  module BatchProcessing
+    # Ruby equivalent of Rust's BatchAggregationScenario
+    #
+    # Detects whether a batchable step created batch workers or returned NoBatches outcome.
+    # Uses DependencyResultsWrapper to access parent step results.
+    #
+    # This helper distinguishes two scenarios:
+    # - **NoBatches**: Batchable step returned NoBatches outcome, only placeholder worker exists
+    # - **WithBatches**: Batchable step created real batch workers for parallel processing
+    #
+    # @example Detecting aggregation scenario
+    #   scenario = BatchAggregationScenario.detect(
+    #     sequence_step,
+    #     'analyze_dataset',
+    #     'process_batch_'
+    #   )
+    #
+    #   if scenario.no_batches?
+    #     # Handle NoBatches scenario - use batchable result directly
+    #     return scenario.batchable_result
+    #   else
+    #     # Handle WithBatches scenario - aggregate worker results
+    #     total = scenario.batch_results.values.sum
+    #     return { total: total, worker_count: scenario.worker_count }
+    #   end
+    class BatchAggregationScenario
+      attr_reader :type, :batchable_result, :batch_results, :worker_count
+
+      # Detect aggregation scenario from step dependencies
+      #
+      # @param dependency_results [DependencyResultsWrapper] Dependency results wrapper
+      # @param batchable_step_name [String] Name of the batchable step
+      # @param batch_worker_prefix [String] Prefix for batch worker step names
+      # @return [BatchAggregationScenario] Detected scenario
+      def self.detect(dependency_results, batchable_step_name, batch_worker_prefix)
+        # dependency_results is already a DependencyResultsWrapper - pass it directly
+        new(dependency_results, batchable_step_name, batch_worker_prefix)
+      end
+
+      # Initialize aggregation scenario
+      #
+      # @param dependency_results [DependencyResultsWrapper] Dependency results wrapper
+      # @param batchable_step_name [String] Name of the batchable step
+      # @param batch_worker_prefix [String] Prefix for batch worker step names
+      def initialize(dependency_results, batchable_step_name, batch_worker_prefix)
+        # Get the batchable step result (extract the 'result' field)
+        @batchable_result = dependency_results.get_results(batchable_step_name)
+
+        # Find all batch worker results by prefix matching (extract 'result' field from each)
+        @batch_results = {}
+        dependency_results.each_key do |step_name|
+          if step_name.start_with?(batch_worker_prefix)
+            @batch_results[step_name] = dependency_results.get_results(step_name)
+          end
+        end
+
+        # Determine scenario type based on batch worker count
+        if @batch_results.empty?
+          @type = :no_batches
+          @worker_count = 0
+        else
+          @type = :with_batches
+          @worker_count = @batch_results.size
+        end
+      end
+
+      # Check if scenario is NoBatches
+      #
+      # @return [Boolean] True if batchable step returned NoBatches outcome
+      def no_batches?
+        @type == :no_batches
+      end
+
+      # Check if scenario is WithBatches
+      #
+      # @return [Boolean] True if batch workers were created
+      def with_batches?
+        @type == :with_batches
+      end
+    end
+  end
+end

data/lib/tasker_core/batch_processing/batch_worker_context.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+module TaskerCore
+  module BatchProcessing
+    # Ruby equivalent of Rust's BatchWorkerContext
+    #
+    # Extracts cursor config from workflow_step.inputs via WorkflowStepWrapper.
+    # This mirrors the Rust implementation which deserializes BatchWorkerInputs
+    # from the workflow_step.inputs field.
+    #
+    # @example Checking if worker is no-op
+    #   context = BatchWorkerContext.from_step_data(sequence_step)
+    #   if context.no_op?
+    #     # Handle placeholder worker for NoBatches scenario
+    #     return success_result(message: 'No-op worker completed')
+    #   end
+    #
+    # @example Accessing cursor configuration
+    #   context = BatchWorkerContext.from_step_data(sequence_step)
+    #   start = context.start_cursor  # => 0
+    #   end_pos = context.end_cursor  # => 1000
+    #   batch_id = context.batch_id     # => "001"
+    #
+    # == Cursor Flexibility
+    #
+    # Cursors are intentionally flexible to support diverse business logic scenarios.
+    # The system validates numeric cursors when both values are integers, but cursors
+    # can also be alphanumeric strings, timestamps, UUIDs, or any comparable type that
+    # makes sense for your data partitioning strategy.
+    #
+    # With great power comes great responsibility: ensure your cursor implementation
+    # matches your data source's capabilities and that your handler logic correctly
+    # interprets the cursor boundaries.
+    #
+    # @example Numeric cursors (validated for ordering)
+    #   cursor: {
+    #     batch_id: '001',
+    #     start_cursor: 0,      # Integer cursors are validated
+    #     end_cursor: 1000
+    #   }
+    #
+    # @example Alphanumeric cursors (developer responsibility)
+    #   cursor: {
+    #     batch_id: '002',
+    #     start_cursor: 'A',    # String cursors for alphabetical ranges
+    #     end_cursor: 'M'
+    #   }
+    #
+    # @example UUID-based cursors (developer responsibility)
+    #   cursor: {
+    #     batch_id: '003',
+    #     start_cursor: '00000000-0000-0000-0000-000000000000',
+    #     end_cursor: '88888888-8888-8888-8888-888888888888'
+    #   }
+    #
+    # @example Timestamp cursors (developer responsibility)
+    #   cursor: {
+    #     batch_id: '004',
+    #     start_cursor: '2024-01-01T00:00:00Z',
+    #     end_cursor: '2024-01-31T23:59:59Z'
+    #   }
+    class BatchWorkerContext
+      attr_reader :cursor, :batch_metadata, :is_no_op, :checkpoint
+
+      # Extract context from WorkflowStepWrapper
+      #
+      # @param workflow_step [WorkflowStepWrapper] Workflow step wrapper
+      # @return [BatchWorkerContext] Extracted context
+      def self.from_step_data(workflow_step)
+        new(workflow_step)
+      end
+
+      # Initialize batch worker context from workflow step
+      #
+      # Reads BatchWorkerInputs from workflow_step.inputs (instance data)
+      # not from step_definition.handler.initialization (template data).
+      #
+      # @param workflow_step [WorkflowStepWrapper] Workflow step wrapper
+      def initialize(workflow_step)
+        # Access inputs via WorkflowStepWrapper
+        # Use ActiveSupport deep_symbolize_keys for clean key access
+        inputs = (workflow_step.inputs || {}).deep_symbolize_keys
+
+        @is_no_op = inputs[:is_no_op] == true
+
+        # TAS-125: Extract checkpoint data from workflow step
+        # Checkpoint contains persisted progress from previous yields
+        # Use with_indifferent_access for flexible string/symbol key access
+        @checkpoint = (workflow_step.checkpoint || {}).with_indifferent_access
+
+        if @is_no_op
+          # Placeholder worker - minimal context
+          @cursor = {}
+          @batch_metadata = {}
+        else
+          @cursor = inputs[:cursor] || {}
+          @batch_metadata = inputs[:batch_metadata] || {}
+
+          validate_cursor!
+        end
+      end
+
+      # Get the starting cursor in the dataset (inclusive)
+      #
+      # @return [Integer] Starting cursor position
+      def start_cursor
+        cursor[:start_cursor].to_i
+      end
+
+      # Get the ending cursor in the dataset (exclusive)
+      #
+      # @return [Integer] Ending cursor position
+      def end_cursor
+        cursor[:end_cursor].to_i
+      end
+
+      # Get the batch identifier
+      #
+      # @return [String] Batch ID (e.g., "001", "002", "000" for no-op)
+      def batch_id
+        cursor[:batch_id] || 'unknown'
+      end
+
+      # Check if this is a no-op/placeholder worker
+      #
+      # @return [Boolean] True if this worker should skip processing
+      def no_op?
+        @is_no_op
+      end
+
+      # TAS-125: Get checkpoint cursor from previous yield
+      #
+      # When a handler yields a checkpoint, the cursor position is persisted.
+      # On re-dispatch, this returns that cursor position to resume from.
+      #
+      # @return [Integer, String, Hash, nil] Last persisted cursor position, or nil if no checkpoint
+      #
+      # @example Resume from checkpoint
+      #   start = batch_ctx.checkpoint_cursor || batch_ctx.start_cursor
+      def checkpoint_cursor
+        checkpoint[:cursor]
+      end
+
+      # TAS-125: Get accumulated results from previous checkpoint yield
+      #
+      # When a handler yields a checkpoint with accumulated_results, those
+      # partial aggregations are persisted. On re-dispatch, this returns
+      # them so the handler can continue accumulating.
+      #
+      # @return [Hash, nil] Partial aggregations from previous yields
+      #
+      # @example Continue accumulating totals
+      #   accumulated = batch_ctx.accumulated_results || { 'total' => 0 }
+      #   accumulated['total'] += item.value
+      def accumulated_results
+        checkpoint[:accumulated_results]
+      end
+
+      # TAS-125: Check if checkpoint exists
+      #
+      # Use this to determine if this is a fresh execution or a resumption
+      # from a previous checkpoint yield.
+      #
+      # @return [Boolean] True if checkpoint data exists
+      #
+      # @example Conditional initialization
+      #   if batch_ctx.has_checkpoint?
+      #     # Resuming from previous checkpoint
+      #     start = batch_ctx.checkpoint_cursor
+      #   else
+      #     # Fresh start
+      #     start = batch_ctx.start_cursor
+      #   end
+      def has_checkpoint?
+        checkpoint.present? && checkpoint[:cursor].present?
+      end
+
+      # TAS-125: Get items processed count from checkpoint
+      #
+      # Returns the cumulative count of items processed across all yields.
+      #
+      # @return [Integer] Items processed so far, or 0 if no checkpoint
+      def checkpoint_items_processed
+        checkpoint[:items_processed] || 0
+      end
+
+      private
+
+      # Validate cursor configuration for real workers
+      #
+      # @raise [ArgumentError] If cursor configuration is invalid
+      def validate_cursor!
+        return if @is_no_op
+
+        raise ArgumentError, 'Missing cursor configuration' if cursor.empty?
+        raise ArgumentError, 'Missing batch_id' unless cursor[:batch_id]
+        raise ArgumentError, 'Missing start_cursor' unless cursor.key?(:start_cursor)
+        raise ArgumentError, 'Missing end_cursor' unless cursor.key?(:end_cursor)
+
+        # Extract cursor values for validation
+        start_val = cursor[:start_cursor]
+        end_val = cursor[:end_cursor]
+
+        # Validate numeric cursors if they're integers
+        # (supports both integer and other types like timestamps/UUIDs)
+        return unless start_val.is_a?(Integer) || end_val.is_a?(Integer)
+
+        # If one is integer, both should be integers
+        unless start_val.is_a?(Integer) && end_val.is_a?(Integer)
+          raise ArgumentError,
+                "Mixed cursor types not allowed: start=#{start_val.class}, end=#{end_val.class}"
+        end
+
+        # Validate non-negative values
+        if start_val.negative?
+          raise ArgumentError,
+                "start_cursor must be non-negative, got #{start_val}"
+        end
+        if end_val.negative?
+          raise ArgumentError,
+                "end_cursor must be non-negative, got #{end_val}"
+        end
+
+        # Validate logical ordering
+        if start_val > end_val
+          raise ArgumentError,
+                "start_cursor (#{start_val}) must be <= end_cursor (#{end_val})"
+        end
+
+        # Warn about zero-length ranges (likely a bug)
+        return unless start_val == end_val
+
+        warn "WARNING: Zero-length cursor range (#{start_val} == #{end_val}) " \
+             "- worker will process no data. batch_id=#{cursor[:batch_id]}"
+      end
+    end
+  end
+end