axn 0.1.0.pre.alpha.3 → 0.1.0.pre.alpha.4

data/lib/axn/async/enqueue_all_orchestrator.rb

@@ -0,0 +1,363 @@
+# frozen_string_literal: true
+
+module Axn
+  module Async
+    # Custom error for missing enqueues_each configuration
+    class MissingEnqueuesEachError < StandardError; end
+
+    # Shared trigger action for executing batch enqueueing in the background.
+    # Called by enqueue_all to iterate over configured fields asynchronously.
+    #
+    # Configure the async adapter via Axn.config.set_enqueue_all_async,
+    # or it defaults to Axn.config.set_default_async.
+    #
+    # @example Configure a specific queue for all enqueue_all jobs
+    #   Axn.configure do |c|
+    #     c.set_enqueue_all_async(:sidekiq, queue: :batch)
+    #   end
+    class EnqueueAllOrchestrator
+      include Axn
+
+      # Disable automatic before/after logging - we log the count manually
+      log_calls false
+
+      expects :target_class_name
+      expects :static_args, default: {}, allow_blank: true
+
+      def call
+        target = target_class_name.constantize
+
+        # Deserialize static_args (convert GlobalID strings back to objects)
+        deserialized_static_args = Axn::Util::GlobalIdSerialization.deserialize(static_args)
+
+        count = self.class.execute_iteration(
+          target,
+          **deserialized_static_args,
+          on_progress: method(:set_logging_context),
+        )
+
+        message_parts = ["Batch enqueued #{count} jobs for #{target.name}"]
+        message_parts << "with explicit args: #{static_args.inspect}" if static_args.any?
+
+        Axn::Util::Logging.log_at_level(
+          self.class,
+          level: :info,
+          message_parts:,
+          error_context: "logging batch enqueue completion",
+        )
+      end
+
+      class << self
+        # Entry point for enqueue_all - validates upfront, then executes async
+        #
+        # @param target [Class] The action class to batch enqueue
+        # @param static_args [Hash] Static arguments passed to each job
+        # @return [String] Job ID from the async adapter
+        def enqueue_for(target, **static_args)
+          validate_async_configured!(target)
+
+          # Handle no-expects case: just call_async directly
+          return target.call_async(**static_args) if target.internal_field_configs.empty?
+
+          # Get configs and resolved static args
+          # Kwargs are split: scalars → resolved_static, enumerables → configs
+          configs, resolved_static = resolve_configs(target, static_args:)
+
+          # Validate static args upfront (raises ArgumentError if missing)
+          validate_static_args!(target, configs, resolved_static) if configs.any?
+
+          # Check if any configs came from kwargs (these have lambdas that can't be serialized)
+          # If so, we must execute iteration synchronously
+          has_kwarg_iteration = configs.any? { |c| c.from.is_a?(Proc) && static_args.key?(c.field) }
+
+          if has_kwarg_iteration
+            # Execute iteration synchronously - kwargs with iterables can't be serialized
+            kwarg_fields = configs.select { |c| c.from.is_a?(Proc) && static_args.key?(c.field) }.map(&:field)
+            info "[enqueue_all] Running in foreground: kwargs #{kwarg_fields.join(', ')} cannot be serialized for background execution"
+            execute_iteration_without_logging(target, **static_args)
+          else
+            # Serialize static_args for Sidekiq (convert GlobalID objects, stringify keys)
+            serialized_static_args = Axn::Util::GlobalIdSerialization.serialize(resolved_static)
+
+            # Execute iteration in background via EnqueueAllOrchestrator
+            call_async(target_class_name: target.name, static_args: serialized_static_args)
+          end
+        end
+
+        # Execute the actual iteration (called from #call in background)
+        # Returns the count of jobs enqueued
+        #
+        # @param target [Class] The action class to enqueue jobs for
+        # @param on_progress [Proc, nil] Callback to track iteration progress for logging context
+        # @param static_args [Hash] Static arguments to pass to each job
+        def execute_iteration(target, on_progress: nil, **static_args)
+          configs, resolved_static = resolve_configs(target, static_args:)
+          count = { value: 0 }
+          iterate(target:, configs:, index: 0, accumulated: {}, static_args: resolved_static, count:, on_progress:)
+          count[:value]
+        end
+
+        # Execute iteration with per-job async logging suppressed (for foreground execution)
+        def execute_iteration_without_logging(target, **static_args)
+          original_log_level = target.log_calls_level
+          target.log_calls_level = nil
+          execute_iteration(target, **static_args)
+        ensure
+          target.log_calls_level = original_log_level
+        end
+
+        private
+
+        # Builds iteration sources and resolves static args from kwargs
+        #
+        # Returns [configs, resolved_static_args] where:
+        # - configs: Array of Config objects for fields to iterate
+        # - resolved_static_args: Hash of field => value for static (non-iterated) fields
+        #
+        # Kwargs handling:
+        # - Scalar values → static args (override any inferred/explicit config)
+        # - Enumerable values → iteration source (replaces inferred/explicit config source)
+        #   Exception: if field expects enumerable type (Array, etc), treat as scalar
+        def resolve_configs(target, static_args: {})
+          explicit_configs = target._batch_enqueue_configs
+          explicit_fields = explicit_configs.map(&:field)
+
+          resolved_static = {}
+          kwarg_configs = []
+
+          # Process kwargs: separate into static vs iterable
+          static_args.each do |field, value|
+            field_config = target.internal_field_configs.find { |c| c.field == field }
+
+            if should_iterate?(value, field_config)
+              # Enumerable kwarg → create a config to iterate over it
+              kwarg_configs << BatchEnqueue::Config.new(
+                field:,
+                from: -> { value },
+                via: nil,
+                filter_block: nil,
+              )
+            else
+              # Scalar kwarg → static arg
+              resolved_static[field] = value
+            end
+          end
+
+          # Fields covered by scalar kwargs shouldn't be iterated
+          scalar_fields = resolved_static.keys
+          iterable_kwarg_fields = kwarg_configs.map(&:field)
+
+          # Filter explicit configs: remove fields that are given as scalars
+          # but keep fields that are given as iterables (kwarg will override the source)
+          filtered_explicit = explicit_configs.reject { |c| scalar_fields.include?(c.field) }
+
+          # For explicit configs with matching kwarg iterables, the kwarg takes precedence
+          filtered_explicit = filtered_explicit.reject { |c| iterable_kwarg_fields.include?(c.field) }
+
+          # Infer configs for model: fields not already covered
+          exclude_from_inference = explicit_fields + scalar_fields + iterable_kwarg_fields
+          inferred = infer_configs_from_models(target, exclude: exclude_from_inference)
+
+          # Merge: inferred first (as defaults), then explicit (as overrides), then kwarg configs (as final overrides)
+          merged = inferred + filtered_explicit + kwarg_configs
+
+          # Sort for memory efficiency: model-based configs (using find_each) should be processed first
+          # to minimize memory usage in nested iterations
+          merged.sort_by! { |config| model_based_config?(config, target) ? 0 : 1 }
+
+          return [merged, resolved_static] if merged.any?
+
+          # No configs at all - error only if there are required fields not covered by static args
+          uncovered_fields = target.internal_field_configs.map(&:field) - resolved_static.keys
+          uncovered_required = uncovered_fields.reject do |field|
+            field_config = target.internal_field_configs.find { |c| c.field == field }
+            field_config&.default.present? || field_config&.validations&.dig(:allow_blank)
+          end
+
+          return [[], resolved_static] if uncovered_required.empty?
+
+          raise MissingEnqueuesEachError,
+                "#{target.name} has required fields (#{uncovered_required.join(', ')}) " \
+                "not covered by enqueues_each, model: declarations, or static args. " \
+                "Add `enqueues_each :field_name, from: -> { ... }` for fields to iterate, " \
+                "use `expects :field, model: SomeModel` where SomeModel responds to find_each, " \
+                "or pass the field as a static argument to enqueue_all."
+        end
+
+        # Infer configs from fields with model: declarations whose model responds to find_each
+        def infer_configs_from_models(target, exclude: [])
+          target.internal_field_configs.filter_map do |field_config|
+            next if exclude.include?(field_config.field)
+
+            model_config = field_config.validations&.dig(:model)
+            next unless model_config
+
+            model_class = model_config[:klass]
+            next unless model_class.respond_to?(:find_each)
+
+            # Create an inferred config (equivalent to `enqueues_each :field`)
+            BatchEnqueue::Config.new(field: field_config.field, from: nil, via: nil, filter_block: nil)
+          end
+        end
+
+        # Checks if a config is model-based (will use find_each for memory-efficient iteration)
+        def model_based_config?(config, target)
+          # Configs with nil 'from' are inferred from model declarations
+          return true if config.from.nil?
+
+          # For explicit configs, check if the field has a model declaration that supports find_each
+          field_config = target.internal_field_configs.find { |c| c.field == config.field }
+          model_config = field_config&.validations&.dig(:model)
+          return true if model_config && model_config[:klass].respond_to?(:find_each)
+
+          false
+        end
+
+        def validate_async_configured!(target)
+          # Set up default async configuration if none is set (same pattern as call_async)
+          if target._async_adapter.nil? && Axn.config._default_async_adapter.present?
+            target.async(
+              Axn.config._default_async_adapter,
+              **Axn.config._default_async_config,
+              &Axn.config._default_async_config_block
+            )
+          end
+
+          return if target._async_adapter.present? && target._async_adapter != false
+
+          raise NotImplementedError,
+                "#{target.name} does not have async configured. " \
+                "Add `async :sidekiq` or `async :active_job` to enable enqueue_all."
+        end
+
+        def validate_static_args!(target, configs, static_args)
+          enqueue_each_fields = configs.map(&:field)
+          all_expected_fields = target.internal_field_configs.map(&:field)
+          static_fields = all_expected_fields - enqueue_each_fields
+
+          # Check for required static fields (those without defaults and not optional)
+          required_static = static_fields.reject do |field|
+            field_config = target.internal_field_configs.find { |c| c.field == field }
+            next true if field_config&.default.present?
+            next true if field_config&.validations&.dig(:allow_blank)
+
+            false
+          end
+
+          missing = required_static - static_args.keys
+          return unless missing.any?
+
+          raise ArgumentError,
+                "Missing required static field(s): #{missing.join(', ')}. " \
+                "These fields are not covered by enqueues_each and must be provided."
+        end
+
+        def iterate(target:, configs:, index:, accumulated:, static_args:, count:, on_progress:)
+          # Base case: all fields accumulated, enqueue the job
+          if index >= configs.length
+            on_progress&.call(stage: :enqueueing, enqueue_args: accumulated.merge(static_args))
+            target.call_async(**accumulated, **static_args)
+            count[:value] += 1
+            return
+          end
+
+          config = configs[index]
+
+          # Track which field's source we're resolving
+          on_progress&.call(stage: :resolving_source, field: config.field)
+          source = config.resolve_source(target:)
+
+          # Use find_each if available (ActiveRecord), otherwise each
+          iterator = source.respond_to?(:find_each) ? :find_each : :each
+
+          source.public_send(iterator) do |item|
+            # Track current item being processed
+            item_id = item.try(:id) || item.to_s.truncate(100)
+            on_progress&.call(stage: :iterating, field: config.field, current_item_id: item_id)
+
+            # Apply filter block if present - swallow errors, skip item
+            if config.filter_block
+              filter_result = begin
+                config.filter_block.call(item)
+              rescue StandardError => e
+                Axn::Internal::Logging.piping_error(
+                  "filter block for :#{config.field}",
+                  exception: e,
+                )
+                false
+              end
+              next unless filter_result
+            end
+
+            # Apply via extraction if present - swallow errors, skip item
+            value = if config.via
+                      begin
+                        item.public_send(config.via)
+                      rescue StandardError => e
+                        Axn::Internal::Logging.piping_error(
+                          "via extraction (:#{config.via}) for :#{config.field}",
+                          exception: e,
+                        )
+                        next
+                      end
+                    else
+                      item
+                    end
+
+            # Recurse to next field
+            iterate(
+              target:,
+              configs:,
+              index: index + 1,
+              accumulated: accumulated.merge(config.field => value),
+              static_args:,
+              count:,
+              on_progress:,
+            )
+          end
+        end
+
+        # Determines if a kwarg value should be iterated over or used as static
+        #
+        # @param value [Object] The value passed in kwargs
+        # @param field_config [Object, nil] The field's config from internal_field_configs
+        # @return [Boolean] true if we should iterate over the value
+        def should_iterate?(value, field_config)
+          return false unless value.respond_to?(:each)
+          return false if value.is_a?(String) || value.is_a?(Hash)
+          return false if field_expects_enumerable?(field_config)
+
+          true
+        end
+
+        # Checks if a field expects an enumerable type (Array, Set, etc)
+        #
+        # @param field_config [Object, nil] The field's config from internal_field_configs
+        # @return [Boolean] true if the field expects an enumerable
+        def field_expects_enumerable?(field_config)
+          return false unless field_config
+
+          type_config = field_config.validations&.dig(:type)
+          return false unless type_config
+
+          # type: Array becomes { klass: Array } after syntactic sugar
+          expected_type = type_config[:klass]
+          return false unless expected_type
+
+          # Handle array of types (e.g., type: [Array, Hash])
+          Array(expected_type).any? do |type|
+            next false unless type.is_a?(Class)
+
+            ENUMERABLE_TYPES.any? { |enum_type| type <= enum_type }
+          end
+        rescue TypeError
+          # expected_type might be a Symbol or something that doesn't support <=
+          false
+        end
+
+        # Types that are considered enumerable for the purposes of iteration detection
+        ENUMERABLE_TYPES = [Array, Set].freeze
+      end
+    end
+  end
+end

data/lib/axn/async.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "axn/async/adapters"
+require "axn/async/batch_enqueue"
 
 module Axn
   module Async
@@ -8,6 +9,10 @@ module Axn
 
     included do
       class_attribute :_async_adapter, :_async_config, :_async_config_block, default: nil
+
+      # Include batch enqueue functionality
+      include BatchEnqueue
+      extend BatchEnqueue::DSL
     end
 
     class_methods do
@@ -29,16 +34,26 @@ module Axn
         end
       end
 
-      def call_async(**)
+      def call_async(**kwargs)
         # Set up default async configuration if none is set
         if _async_adapter.nil?
           async Axn.config._default_async_adapter, **Axn.config._default_async_config, &Axn.config._default_async_config_block
           # Call ourselves again now that the adapter is included
-          return call_async(**)
+          return call_async(**kwargs)
         end
 
-        # This will be overridden by the included adapter module
-        raise NotImplementedError, "No async adapter configured. Use e.g. `async :sidekiq` or `async :active_job` to enable background processing."
+        # Skip notification and logging for disabled adapter (it will raise immediately)
+        return _enqueue_async_job(kwargs) if _async_adapter == false
+
+        # Emit notification for async call
+        _emit_call_async_notification(kwargs)
+
+        # Log async invocation if logging is enabled
+        adapter_name = _async_adapter_name_for_logging
+        _log_async_invocation(kwargs, adapter_name:) if adapter_name && log_calls_level
+
+        # Delegate to adapter-specific enqueueing logic
+        _enqueue_async_job(kwargs)
       end
 
       # Ensure default async is applied when the class is first instantiated
@@ -50,12 +65,114 @@ module Axn
 
       private
 
+      def _emit_call_async_notification(kwargs)
+        resource = name || "AnonymousClass"
+        # Use dup to ensure kwargs modifications don't affect the notification payload
+        payload = { resource:, action_class: self, kwargs: kwargs.dup, adapter: _async_adapter_name }
+
+        ActiveSupport::Notifications.instrument("axn.call_async", payload)
+      rescue StandardError => e
+        # Don't raise in notification emission to avoid interfering with async enqueueing
+        Axn::Internal::Logging.piping_error("emitting notification for axn.call_async", action_class: self, exception: e)
+      end
+
+      def _log_async_invocation(kwargs, adapter_name:)
+        Axn::Util::Logging.log_at_level(
+          self,
+          level: log_calls_level,
+          message_parts: ["Enqueueing async execution via #{adapter_name}"],
+          join_string: " with: ",
+          before: _async_log_separator,
+          prefix: "[#{name.presence || 'Anonymous Class'}]",
+          error_context: "logging async invocation",
+          context_direction: :inbound,
+          context_data: kwargs,
+        )
+      end
+
+      def _async_log_separator
+        return if Axn.config.env.production?
+        return if Axn::Util::ExecutionContext.background?
+        return if Axn::Util::ExecutionContext.console?
+
+        "\n------\n"
+      end
+
+      # Hook method that must be implemented by async adapter modules.
+      #
+      # Adapters MUST:
+      # - Implement this method with adapter-specific enqueueing logic
+      # - NOT override `call_async` (the base implementation handles notifications, logging, and delegates here)
+      #
+      # The only exception is the Disabled adapter, which overrides `call_async` to raise immediately
+      # without emitting notifications.
+      #
+      # @param kwargs [Hash] The keyword arguments to pass to the action when it executes
+      # @return The result of enqueueing (typically a job ID or similar, adapter-specific)
+      def _enqueue_async_job(kwargs)
+        # This will be overridden by the included adapter module
+        raise NotImplementedError, "No async adapter configured. Use e.g. `async :sidekiq` or `async :active_job` to enable background processing."
+      end
+
+      def _async_adapter_name
+        if _async_adapter.nil?
+          "none"
+        elsif _async_adapter == false
+          "disabled"
+        else
+          _async_adapter.to_s
+        end
+      end
+
+      def _async_adapter_name_for_logging
+        return nil if _async_adapter.nil? || _async_adapter == false
+
+        _async_adapter_name
+      end
+
       def _ensure_default_async_configured
         return if _async_adapter.present?
         return unless Axn.config._default_async_adapter.present?
 
         async Axn.config._default_async_adapter, **Axn.config._default_async_config, &Axn.config._default_async_config_block
       end
+
+      # Extracts and normalizes _async options from kwargs.
+      # Returns normalized options hash (with string keys and converted durations) and removes _async from kwargs.
+      #
+      # @param kwargs [Hash] The keyword arguments (modified in place)
+      # @return [Hash, nil] Normalized async options hash, or nil if no _async options present
+      def _extract_and_normalize_async_options(kwargs)
+        async_options = kwargs.delete(:_async) if kwargs[:_async].is_a?(Hash)
+        _normalize_async_options(async_options) if async_options
+      end
+
+      # Normalizes _async options hash:
+      # - Converts symbol keys to string keys
+      # - Converts ActiveSupport::Duration values to integer seconds (for wait)
+      # - Preserves Time objects (for wait_until)
+      #
+      # @param async_hash [Hash, nil] The async options hash
+      # @return [Hash, nil] Normalized hash with string keys, or nil if input is not a hash
+      def _normalize_async_options(async_hash)
+        return nil unless async_hash.is_a?(Hash)
+
+        normalized = {}
+        async_hash.each do |key, value|
+          string_key = key.to_s
+
+          normalized[string_key] = case string_key
+                                   when "wait"
+                                     # Convert ActiveSupport::Duration to integer seconds
+                                     value.respond_to?(:to_i) ? value.to_i : value
+                                   else
+                                     # Preserve wait_until and other keys/values as-is
+                                     value
+                                   end
+        end
+
+        normalized
+      end
     end
   end
 end

data/lib/axn/configuration.rb

@@ -6,7 +6,7 @@ module Axn
   end
 
   class Configuration
-    attr_accessor :wrap_with_trace, :emit_metrics
+    attr_accessor :emit_metrics, :raise_piping_errors_in_dev
     attr_writer :logger, :env, :on_exception, :additional_includes, :log_level, :rails
 
     def log_level = @log_level ||= :info
@@ -23,32 +23,54 @@ module Axn
       @default_async_adapter = adapter unless adapter.nil?
       @default_async_config = config.any? ? config : {}
       @default_async_config_block = block_given? ? block : nil
+
+      _apply_async_to_enqueue_all_orchestrator
+    end
+
+    # Async configuration for EnqueueAllOrchestrator (used by enqueue_all_async)
+    # Defaults to the default async config if not explicitly set
+    def _enqueue_all_async_adapter = @enqueue_all_async_adapter || _default_async_adapter
+    def _enqueue_all_async_config = @enqueue_all_async_config || _default_async_config
+    def _enqueue_all_async_config_block = @enqueue_all_async_config_block || _default_async_config_block
+
+    def set_enqueue_all_async(adapter, **config, &block)
+      @enqueue_all_async_adapter = adapter
+      @enqueue_all_async_config = config.any? ? config : {}
+      @enqueue_all_async_config_block = block_given? ? block : nil
+
+      _apply_async_to_enqueue_all_orchestrator
     end
 
     def rails = @rails ||= RailsConfiguration.new
 
     def on_exception(e, action:, context: {})
-      msg = "Handled exception (#{e.class.name}): #{e.message}"
+      if action.respond_to?(:result) && action.result.respond_to?(:error)
+        resolved_error = action.result.error
+        # Compare with the default fallback message instead of calling default_error
+        # to avoid triggering error message resolution multiple times
+        detail = resolved_error == Axn::Core::Flow::Handlers::Resolvers::MessageResolver::DEFAULT_ERROR ? e.message : resolved_error
+      else
+        detail = e.message
+      end
+
+      msg = "Handled exception (#{e.class.name}): #{detail}"
       msg = ("#" * 10) + " #{msg} " + ("#" * 10) unless Axn.config.env.production?
       action.log(msg)
 
       return unless @on_exception
 
-      # Only pass the kwargs that the given block expects
-      kwargs = @on_exception.parameters.select { |type, _name| %i[key keyreq].include?(type) }.map(&:last)
-      kwarg_hash = {}
-      kwarg_hash[:action] = action if kwargs.include?(:action)
-      kwarg_hash[:context] = context if kwargs.include?(:context)
-      if kwarg_hash.any?
-        @on_exception.call(e, **kwarg_hash)
-      else
-        @on_exception.call(e)
-      end
+      # Only pass the args and kwargs that the given block expects
+      Axn::Util::Callable.call_with_desired_shape(@on_exception, args: [e], kwargs: { action:, context: })
     end
 
     def logger
       @logger ||= begin
-        Rails.logger
+        # Use sidekiq logger if in background
+        if Axn::Util::ExecutionContext.background? && defined?(Sidekiq)
+          Sidekiq.logger
+        else
+          Rails.logger
+        end
       rescue NameError
         Logger.new($stdout).tap do |l|
           l.level = Logger::INFO
@@ -60,6 +82,24 @@ module Axn
       @env ||= ENV["RACK_ENV"].presence || ENV["RAILS_ENV"].presence || "development"
       ActiveSupport::StringInquirer.new(@env)
     end
+
+    private
+
+    # Apply async config to EnqueueAllOrchestrator if it's already loaded.
+    # Called from set_default_async and set_enqueue_all_async to ensure the
+    # orchestrator has Sidekiq::Job included before any worker tries to process jobs.
+    def _apply_async_to_enqueue_all_orchestrator
+      return unless defined?(Axn::Async::EnqueueAllOrchestrator)
+
+      adapter = _enqueue_all_async_adapter
+      return if adapter.nil? || adapter == false
+
+      Axn::Async::EnqueueAllOrchestrator.async(
+        adapter,
+        **_enqueue_all_async_config,
+        &_enqueue_all_async_config_block
+      )
+    end
   end
 
   class << self

data/lib/axn/context.rb

@@ -40,6 +40,7 @@ module Axn
     def __record_early_completion(message)
       @early_completion_message = message unless message == Axn::Internal::EarlyCompletion.new.message
       @early_completion = true
+      @finalized = true
     end
 
     def __early_completion_message = @early_completion_message.presence