attr-gather 1.0.0

data/lib/attr/gather/workflow/dot_serializer.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'tempfile'
+
+module Attr
+  module Gather
+    module Workflow
+      # @api private
+      class DotSerializer
+        def initialize(task_graph)
+          @task_graph = task_graph
+        end
+
+        def to_dot
+          lines = @task_graph.tsort.map { |t| serialize_row(t) }
+          joined_lines = lines.flatten.map { |l| "  #{l}" }.join("\n").strip
+
+          <<~DOT
+            digraph TaskGraph {
+              #{joined_lines}
+            }
+          DOT
+        end
+
+        def preview
+          Tempfile.open(['task-graph-preview', '.svg']) do |tf|
+            IO.popen("dot -Tsvg -o #{tf.path}", 'w') { |p| p.write(to_dot) }
+            `xdg-open #{tf.path}`
+          end
+        end
+
+        private
+
+        def serialize_row(task)
+          row = all_dependants_for_task(task).map { |dt| [task, dt] }
+          lines = row.map { |item| item.map(&:name).join(' -> ') + ';' }
+          lines
+        end
+
+        def all_dependants_for_task(input_task)
+          @task_graph.to_h.keys.select { |task| task.depends_on?(input_task) }
+        end
+      end
+    end
+  end
+end

data/lib/attr/gather/workflow/dsl.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Attr
+  module Gather
+    module Workflow
+      # DSL for configuring a workflow
+      #
+      # @api public
+      module DSL
+        # @api private
+        Undefined = Object.new.freeze
+
+        # Defines a task with name and options
+        #
+        # @param task_name [Symbol] the name of the task
+        #
+        # @example
+        #   class EnhanceUserProfile
+        #     extend Attr::Gather::Workflow
+        #
+        #     # ...
+        #
+        #     task :fetch_database_info do |t|
+        #       t.depends_on = []
+        #     end
+        #
+        #     task :fetch_avatar_info do |t|
+        #       t.depends_on = [:fetch_gravatar_info]
+        #     end
+        #   end
+        #
+        # Calling `task` will yield a task object which you can configure like
+        # a PORO. Tasks will be registered for execution in the workflow.
+        #
+        # @yield [Attr::Gather::Workflow::Task] A task to configure
+        #
+        # @api public
+        def task(task_name, opts = EMPTY_HASH)
+          task = Task.new(name: task_name, **opts)
+          yield task
+          tasks << task
+          self
+        end
+
+        # Defines a container for task dependencies
+        #
+        # Using a container  makes it easy to re-use workflows with different
+        # data sources. Say one workflow was required to use a legacy DB, and
+        # one wanted to use a new DB. Using a container makes it easy to
+        # configure that dependency.
+        #
+        # @example
+        #   LegacySystem = Dry::Container.new.tap do |c|
+        #     c.register(:database) { Sequel.connect('sqlite://legacy.db')
+        #   end
+        #
+        #   class EnhanceUserProfile
+        #     extend Attr::Gather::Workflow
+        #
+        #     container LegacySystem
+        #   end
+        #
+        # @param cont [Dry::Container] the Dry::Container to use
+        #
+        # @note For more information, check out {https://dry-rb.org/gems/dry-container}
+        #
+        # @api public
+        def container(cont = nil)
+          @container = cont if cont
+          @container
+        end
+
+        # Configures the result aggregator
+        #
+        # Aggregators make is possible to build custom logic about
+        # how results should be "merged" together. For example,
+        # yuo could build and aggregator that prioritizes the
+        # values of some tasks over others.
+        #
+        # @example
+        #   class EnhanceUserProfile
+        #     extend Attr::Gather::Workflow
+        #
+        #     aggregator :deep_merge
+        #   end
+        #
+        # @param agg [#call] the aggregator to use
+        #
+        # @api public
+        def aggregator(agg = nil, opts = EMPTY_HASH)
+          if agg.nil? && !defined?(@aggregator)
+            @aggregator = Aggregators.default
+            return @aggregator
+          end
+
+          @aggregator = Aggregators.resolve(agg, filter: filter, **opts) if agg
+          @aggregator
+        end
+
+        # Defines a filter for filtering out invalid values
+        #
+        # When aggregating data from many sources, it is hard to reason about
+        # all the ways invalid data will be returned. For example, if you are
+        # pulling data from a spreadsheet, there will often be typos, etc.
+        #
+        # Defining a filter allows you to declaratively state what is valid.
+        # attr-gather will use this definition to automatically filter out
+        # invalid values, so they never make it into your system.
+        #
+        # Filtering happens during each step of the workflow, which means that
+        # every Task will receive validated input that you can rely on.
+        #
+        # @example
+        #   class UserContract < Dry::Validation::Contract do
+        #     params do
+        #       optional(:id).filled(:integer)
+        #       optional(:email).filled(:str?, format?: /@/)
+        #     end
+        #   end
+        #
+        #   class EnhanceUserProfile
+        #     extend Attr::Gather::Workflow
+        #
+        #     # Any of the key/value pairs that had validation errors will be
+        #     # filtered from the output.
+        #     filter :contract, UserContract.new
+        #   end
+        #
+        # @param filt [Symbol] the name filter to use
+        # @param args [Array<Object>] arguments for initializing the filter
+        #
+        # @api public
+        def filter(filt = Undefined, *args)
+          if filt == Undefined && !defined?(@filter)
+            @filter = Filters.default
+          elsif filt != Undefined
+            @filter = Filters.resolve(filt, *args)
+          end
+
+          @filter
+        end
+
+        # Defines a filter for filtering invalid values with an inline contract
+        #
+        # This serves as a convenience method for defining a contract filter.
+        #
+        # @example
+        #
+        #   class EnhanceUserProfile
+        #     extend Attr::Gather::Workflow
+        #
+        #     # Any of the key/value pairs that had validation errors will be
+        #     # filtered from the output.
+        #     filter_with_contract do
+        #        params do
+        #          required(:name).filled(:string)
+        #          required(:age).value(:integer)
+        #        end
+        #
+        #        rule(:age) do
+        #          key.failure('must be greater than 18') if value < 18
+        #        end
+        #     end
+        #   end
+        #
+        # @return [Dry::Validation::Contract,NilClass]
+        # @see https://dry-rb.org/gems/dry-validation
+        #
+        # @api public
+        def filter_with_contract(arg = nil, &blk)
+          contract = block_given? ? build_inline_contract_filter(&blk) : arg
+          @filter = Filters.resolve(:contract, contract)
+        end
+
+        private
+
+        def build_inline_contract_filter(&blk)
+          contract_klass = Class.new(Dry::Validation::Contract, &blk)
+          contract_klass.new
+        end
+      end
+    end
+  end
+end

data/lib/attr/gather/workflow/graphable.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Attr
+  module Gather
+    module Workflow
+      # Module containing graph functionality
+      #
+      # @api public
+      module Graphable
+        # Class methods for graph functionality
+        module ClassMethods
+          # Returns the graph of tasks
+          #
+          # @return [TaskGraph] the graph
+          #
+          # @api private
+          def tasks
+            @tasks ||= TaskGraph.new
+          end
+
+          # Returns a graphviz visualization of the workflow
+          #
+          # @param preview [Boolean] show a preview image of the Workflow
+          #
+          # @api public
+          def to_dot(preview: true)
+            tasks.to_dot(preview: preview)
+          end
+        end
+
+        # Instance methods for graph functionality
+        module InstanceMethods
+          # Returns a graphviz visualization of the workflow
+          #
+          # @param preview [Boolean] show a preview image of the Workflow
+          #
+          # @api public
+          def to_dot(preview: true)
+            self.class.to_dot(preview: preview)
+          end
+        end
+
+        def self.included(klass)
+          klass.extend(ClassMethods)
+          klass.include(InstanceMethods)
+        end
+      end
+    end
+  end
+end

data/lib/attr/gather/workflow/task.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Attr
+  module Gather
+    module Workflow
+      # @api private
+      class Task
+        attr_accessor :depends_on, :name
+
+        def initialize(name:, depends_on: [])
+          @name = name
+          @depends_on = depends_on
+        end
+
+        def depends_on?(other_task)
+          depends_on.include?(other_task.name)
+        end
+
+        def fullfilled_given_remaining_tasks?(task_list)
+          task_list.none? { |list_task| depends_on?(list_task) }
+        end
+
+        def as_json
+          { name: name, depends_on: depends_on }
+        end
+      end
+    end
+  end
+end

data/lib/attr/gather/workflow/task_execution_result.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Attr
+  module Gather
+    module Workflow
+      # A wrapper containing information and results of a task execution
+      #
+      # @!attribute [r] started_at
+      #   @return [Time] time which the execution occured
+      #
+      # @!attribute [r] task
+      #   @return [Attr::Gather::Workflow::Task] task that was run
+      #
+      # @!attribute [r] result
+      #   @return [Concurrent::Promise] the result promise of the the task
+      #
+      # @api public
+      class TaskExecutionResult
+        include Concerns::Identifiable
+
+        attr_reader :task, :result, :started_at, :uuid
+
+        def initialize(task, result)
+          @started_at = Time.now
+          @uuid = SecureRandom.uuid
+          @task = task
+          @result = result
+        end
+
+        # @!attribute [r] state
+        #   @return [:unscheduled, :pending, :processing, :rejected, :fulfilled]
+        def state
+          result.state
+        end
+
+        # Extracts the result, this is an unsafe operation that blocks the
+        # operation, and returns either the value or an exception.
+        #
+        # @note For more information, check out {https://ruby-concurrency.github.io/concurrent-ruby/1.1.5/Concurrent/Concern/Obligation.html#value!-instance_method}
+        def value!
+          result.value!
+        end
+
+        # Represents the TaskExecutionResult as a hash
+        #
+        # @return [Hash]
+        def as_json
+          value = result.value
+
+          { started_at: started_at,
+            task: task.as_json,
+            state: state,
+            value: value }
+        end
+      end
+    end
+  end
+end

data/lib/attr/gather/workflow/task_executor.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+require 'attr/gather/workflow/task_execution_result'
+
+module Attr
+  module Gather
+    module Workflow
+      # @api private
+      class TaskExecutor
+        attr_reader :batch, :container, :executor
+
+        def initialize(batch, container:)
+          @batch = batch
+          @container = container
+          @executor = :immediate
+        end
+
+        def call(input)
+          batch.map do |task|
+            task_proc = container.resolve(task.name)
+            result = Concurrent::Promise.execute(executor: executor) do
+              task_proc.call(input)
+            end
+            TaskExecutionResult.new(task, result)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/attr/gather/workflow/task_graph.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'tsort'
+require 'attr/gather/workflow/dot_serializer'
+
+module Attr
+  module Gather
+    module Workflow
+      # @api private
+      class TaskGraph
+        class UnfinishableError < StandardError; end
+        class InvalidTaskDepedencyError < StandardError; end
+
+        include TSort
+
+        attr_reader :tasks_hash
+
+        def initialize(tasks: [])
+          @tasks_hash = {}
+          tasks.each { |t| self << t }
+        end
+
+        def <<(task)
+          validate_for_insert!(task)
+
+          registered_tasks.each do |t|
+            tasks_hash[t] << task if t.depends_on?(task)
+            tasks_hash[t].uniq!
+          end
+
+          tasks_hash[task] = all_dependencies_for_task(task)
+        end
+
+        def runnable_tasks
+          tsort.take_while do |task|
+            task.fullfilled_given_remaining_tasks?(registered_tasks)
+          end
+        end
+
+        def each_batch
+          return enum_for(:each_batch) unless block_given?
+
+          to_execute = tsort
+
+          until to_execute.empty?
+            batch = to_execute.take_while do |task|
+              task.fullfilled_given_remaining_tasks?(to_execute)
+            end
+
+            to_execute -= batch
+
+            validate_finishable!(batch, to_execute)
+
+            yield batch
+          end
+        end
+
+        alias to_a tsort
+
+        def to_h
+          tasks_hash
+        end
+
+        def to_dot(preview: false)
+          serializer = DotSerializer.new(self)
+          preview ? serializer.preview : serializer.to_dot
+        end
+
+        private
+
+        def tsort_each_child(node, &blk)
+          to_h[node].each(&blk)
+        end
+
+        def tsort_each_node(&blk)
+          to_h.each_key(&blk)
+        end
+
+        def validate_finishable!(batch, to_execute)
+          return unless batch.empty? && !to_execute.empty?
+
+          # TODO: statically verify this
+          raise UnfinishableError, 'task dependencies are not fulfillable'
+        end
+
+        def validate_for_insert!(task)
+          return if depended_on_tasks_exist?(task)
+
+          raise InvalidTaskDepedencyError,
+                "could not find a matching task for #{task.name}"
+        end
+
+        def all_dependencies_for_task(input_task)
+          registered_tasks.select { |task| input_task.depends_on?(task) }
+        end
+
+        def registered_tasks
+          tasks_hash.keys
+        end
+
+        def depended_on_tasks_exist?(task)
+          task.depends_on.all? { |t| registered_tasks.map(&:name).include?(t) }
+        end
+      end
+    end
+  end
+end