attr-gather 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee9b30fd44bd8931964b53b904815224fa6a5a592ac1f22d19605a982a326d42
-  data.tar.gz: 7954a71a61dc89266358909ce189312b3b3164e4092d62ae4fae973712efca9d
+  metadata.gz: 56fa2344f15f389295db52f8201720d91bae7114d5d68472208d5509fe40d35d
+  data.tar.gz: cce04b8b3ebe30523f0b4b5528f2531d5607e1905e6869f21076ac7eeac39e34
 SHA512:
-  metadata.gz: 040b490aba781f8f851b617a365d2aea061f9d2afb189bcdf4e0e086effebc2135109a648b0f116d4be7000cbe4d3e10e369fadcb38d9e2910114c36957cce68
-  data.tar.gz: 59a93d0d6e16f9b2433822c4faf7ef4cda25f3e9e37da8a5fae816e313e1d6e0b86906a1d652b3c64125eae888607c9117ade38fc17ff11b84c361add21f4c62
+  metadata.gz: dad243cf700400e4bab5abb9134704870885c711a6a4c4eedf185860af66485493f16d2a2fda99b103e1ab7b5238c244e16a73a7b1264b9f59cee6f42bbdf2b7
+  data.tar.gz: 46441ed69c96018c33c4676e3e48b3b06bf39de8aaa2f16c20fdda026375bd66330064c11ac8ea4f7499b0038c98309a3f958ea532ab6da1476dad2e5e070de1

data/.github/mergify.yml

@@ -13,7 +13,10 @@ pull_request_rules:
     conditions:
       - author~=^dependabot(|-preview)\[bot\]$
       - "#approved-reviews-by>=1"
-      - "check-success=build-test-lint"
+      - "check-success~=build-test-lint (2.4.x)"
+      - "check-success~=build-test-lint (2.5.x)"
+      - "check-success~=build-test-lint (2.6.x)"
+      - "check-success~=build-test-lint (2.7.x)"
     actions:
       merge:
         method: squash

data/.github/workflows/ruby.yml

@@ -4,6 +4,9 @@ on: [push]
 jobs:
   build-test-lint:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: ["2.4.x", "2.5.x", "2.6.x", "2.7.x"]
     steps:
       - uses: actions/checkout@v1
       - name: Set up Ruby 2.6

data/.rubocop.yml

@@ -4,7 +4,8 @@ require:
 AllCops:
   UseCache: true
   CacheRootDirectory: './tmp/cache'
-  TargetRubyVersion: 2.5
+  TargetRubyVersion: 2.4
+  NewCops: enable
 
 Metrics/BlockLength:
   Exclude:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    attr-gather (1.2.0)
+    attr-gather (1.2.1)
       dry-container (~> 0.7)
 
 GEM
@@ -14,7 +14,7 @@ GEM
     benchmark (0.1.0)
     coderay (1.1.3)
     concurrent-ruby (1.1.7)
-    diff-lcs (1.3)
+    diff-lcs (1.4.4)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     dry-configurable (0.11.6)
@@ -76,7 +76,7 @@ GEM
     mini_portile2 (2.4.0)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
-    parallel (1.19.2)
+    parallel (1.20.1)
     parser (2.7.2.0)
       ast (~> 2.4.1)
     pry (0.13.1)
@@ -85,29 +85,35 @@ GEM
     public_suffix (4.0.6)
     rainbow (3.0.0)
     rake (13.0.1)
+    regexp_parser (2.0.0)
     reverse_markdown (2.0.0)
       nokogiri
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.0)
-      rspec-support (~> 3.9.0)
-    rspec-expectations (3.9.0)
+    rexml (3.2.4)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.0)
-    rubocop (0.75.1)
-      jaro_winkler (~> 1.5.1)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.0)
+    rubocop (0.93.1)
       parallel (~> 1.10)
-      parser (>= 2.6)
+      parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
+      rexml
+      rubocop-ast (>= 0.6.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
-    rubocop-performance (1.5.1)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (1.3.0)
+      parser (>= 2.7.1.5)
+    rubocop-performance (1.6.1)
       rubocop (>= 0.71.0)
     ruby-progressbar (1.10.1)
     solargraph (0.39.17)
@@ -129,7 +135,7 @@ GEM
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
-    unicode-display_width (1.6.1)
+    unicode-display_width (1.7.0)
     yard (0.9.25)
 
 PLATFORMS

data/README.md

@@ -1,10 +1,148 @@
 # attr-gather
 
-[![Actions Status](https://github.com/ianks/attr-gather/workflows/.github/workflows/ruby.yml/badge.svg)](https://github.com/ianks/attr-gather/actions)
+![Actions Status](https://github.com/ianks/attr-gather/workflows/Build%20+%20Test%20+%20Lint/badge.svg)
+[![Maintainability](https://api.codeclimate.com/v1/badges/b825a3bc37ad6a76e005/maintainability)](https://codeclimate.com/github/ianks/attr-gather/maintainability)
 
-A gem for creating simple workflows to enhance entities with extra attributes.
-At a high level, `attr-gather` provides a process to sync attributes from many
-sources (third party APIs, legacy databases, etc).
+A gem for creating workflows that "enhance" entities with extra attributes. At a high level, [attr-gather](https://github.com/ianks/attr-gather) provides a process to fetch information from many data sources (such as third party APIs, legacy databases, etc.) in a fully parallelized fashion.
+
+## Usage
+
+### Defining your workflow
+
+```ruby
+# define a workflow
+class EnhanceProfile
+  include Attr::Gather::Workflow
+
+  # contains all the task implementations
+  container TasksContainer
+
+  # filter out invalid data using a Dry::Validation::Contract
+  # anything that doesn't match this schema will be filtered out
+  filter_with_contract do
+    params do
+      required(:user_id).filled(:integer)
+
+      optional(:user).hash do
+        optional(:name).filled(:string)
+        optional(:email).filled(:string)
+        optional(:gravatar).filled(:string)
+        optional(:email_info).hash do
+          optional(:deliverable).filled(:bool?)
+          optional(:free).filled(:bool?)
+        end
+      end
+    end
+  end
+
+  # each task returns a hash of data that will be merged into the result
+  task :fetch_post do |t|
+    t.depends_on = []
+  end
+
+  # will run in parallel
+  task :fetch_user do |t|
+    t.depends_on = [:fetch_post]
+  end
+
+  # will run in parallel
+  task :fetch_email_info do |t|
+    t.depends_on = [:fetch_user]
+  end
+end
+```
+
+### Defining some tasks
+
+```ruby
+class PostFetcher
+  def call(attrs)
+    res = HTTP.get("https://jsonplaceholder.typicode.com/posts/#{attrs[:id]}")
+    post = JSON.parse(res.to_s, symbolize_names: true)
+
+    { title: post[:title], user_id: post[:userId], body: post[:body] }
+  end
+end
+```
+
+```ruby
+class UserFetcher
+  # will have access to the PostFetcher attributes here
+  def call(attrs)
+    res = HTTP.get("https://jsonplaceholder.typicode.com/users/#{attrs[:user_id]}")
+    user = JSON.parse(res.to_s, symbolize_names: true)
+
+    { user: { name: user[:name], email: user[:email] } }
+  end
+end
+```
+
+```ruby
+class EmailInfoFetcher
+  # will have access to the PostFetcher attributes here
+  def call(user:)
+    res = HTTP.timeout(3).get("https://api.trumail.io/v2/lookups/json?email=#{user[:email]}")
+    info = JSON.parse(res.to_s, symbolize_names: true)
+
+    # will deep merge with the final result
+    { user: { email_info: { deliverable: info[:deliverable], free: info[:free] } } }
+  end
+end
+```
+
+### Registering your tasks
+
+```ruby
+class MyContainer
+  extend Dry::Container::Mixin
+  
+  register :fetch_post, PostFetcher
+  register :fetch_user, UserFetcher
+  register :fetch_email_info, EmailInfoFetcher
+end
+```
+
+### Run it!
+
+```ruby
+enhancer = EnhanceUserProfile.new
+enhancer.call(id: 12).value!
+```
+
+And this is the result...
+
+```ruby
+{
+  :id => 12,
+  :user_id => 2,
+  :user => {
+    :email => "Shanna@melissa.tv",
+    :name => "Ervin Howell",
+    :email_info => { :deliverable => true, :free => true },
+    :gravatar => "https://www.gravatar.com/avatar/241af7d19a0a7438794aef21e4e19b79"
+  }
+}
+```
+
+You can even preview it as an SVG!
+
+```ruby
+enhancer.to_dot(preview: true) # requires graphviz (brew install graphviz)
+```
+
+## Features
+
+- Offers DSL for defining workflows and merging the results from each task
+- Execution engine optimally parallelizes the execution of the workflow dependency graph using [concurrent-ruby](https://github.com/ruby-concurrency/concurrent-ruby) Promises
+- Very easy to unit test
+- Ability to filter out bad/junky data using [dry-validation](https://dry-rb.org/gems/dry-validation) contracts
+
+## What are the main difference between this Ruby project and similar ones?
+
+- Operates on a single entity rather than a list, so easily adoptable in existing systems
+- Focuses on the "fetching" and filtering of data solely, and not transformation or storage
+- Focuses on having a clean PORO interface to make testing simple
+- Provides a declarative interface for merging results from many sources (APIs, legacy databases, etc.) which allows for prioritization
 
 ## Links
 

data/attr-gather.gemspec

@@ -15,6 +15,8 @@ Gem::Specification.new do |spec|
   spec.homepage = 'https://github.com/ianks/attr-gather'
   spec.license = 'MIT'
 
+  spec.required_ruby_version = '>= 2.4'
+
   # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
 
   spec.metadata['homepage_uri'] = spec.homepage

data/examples/post_enhancer.rb

@@ -28,7 +28,7 @@ class MyContainer
   end
 
   register :email_info do |user:, **_attrs|
-    res = HTTP.timeout(3).get("https://api.trumail.io/v2/lookups/json?email=#{user[:email]}")
+    res = HTTP.get("https://api.trumail.io/v2/lookups/json?email=#{user[:email]}")
     info = JSON.parse(res.to_s, symbolize_names: true)
 
     { user: { email_info: { deliverable: info[:deliverable], free: info[:free] } } }

data/lib/attr/gather/aggregators/deep_merge.rb

@@ -12,28 +12,25 @@ module Attr
         # Initialize a new DeepMerge aggregator
         #
         # @param reverse [Boolean] deep merge results in reverse order
+        # @param merge_input [Boolean] merge the result with the initial input
         #
         # @api private
-        def initialize(reverse: false, **)
+        def initialize(reverse: false, merge_input: true, **)
           @reverse = reverse
+          @merge_input = merge_input
           super
         end
 
         def call(input, execution_results)
           execution_results = execution_results.reverse_each if reverse?
-          initial = unwrap_initial_input(input)
 
-          execution_results.reduce(initial) do |memo, res|
+          execution_results.reduce(@merge_input ? input : EMPTY_HASH) do |memo, res|
             deep_merge(memo, unwrap_result(res))
           end
         end
 
         private
 
-        def unwrap_initial_input(input)
-          input
-        end
-
         def reverse?
           @reverse
         end

data/lib/attr/gather/aggregators/shallow_merge.rb

@@ -12,17 +12,19 @@ module Attr
         # Initialize a new DeepMerge aggregator
         #
         # @param reverse [Boolean] merge results in reverse order
+        # @param merge_input [Boolean] merge the result with the initial input
         #
         # @api private
-        def initialize(reverse: false, **)
+        def initialize(reverse: false, merge_input: true, **)
           @reverse = reverse
+          @merge_input = merge_input
           super
         end
 
         def call(input, execution_results)
           execution_results = execution_results.reverse_each if reverse?
 
-          execution_results.reduce(input) do |memo, res|
+          execution_results.reduce(@merge_input ? input : EMPTY_HASH) do |memo, res|
             memo.merge(unwrap_result(res))
           end
         end

data/lib/attr/gather/filters/contract.rb

@@ -14,8 +14,8 @@ module Attr
         # @param dry_contract [Dry::Contract]
         def initialize(dry_contract)
           validate_dry_contract!(dry_contract)
-
           @dry_contract = dry_contract
+          super()
         end
 
         def call(input)

data/lib/attr/gather/version.rb

@@ -2,6 +2,6 @@
 
 module Attr
   module Gather
-    VERSION = '1.2.0'
+    VERSION = '1.2.1'
   end
 end

data/lib/attr/gather/workflow/dot_serializer.rb

@@ -33,8 +33,7 @@ module Attr
 
         def serialize_row(task)
           row = all_dependants_for_task(task).map { |dt| [task, dt] }
-          lines = row.map { |item| item.map(&:name).join(' -> ') + ';' }
-          lines
+          row.map { |item| "#{item.map(&:name).join(' -> ')};" }
         end
 
         def all_dependants_for_task(input_task)

data/lib/attr/gather/workflow/dsl.rb

@@ -16,7 +16,7 @@ module Attr
         #
         # @example
         #   class EnhanceUserProfile
-        #     extend Attr::Gather::Workflow
+        #     include Attr::Gather::Workflow
         #
         #     # ...
         #
@@ -42,6 +42,56 @@ module Attr
           self
         end
 
+        # Defines a task with name and options
+        #
+        # @param task_name [Symbol] the name of the task
+        #
+        # @example
+        #   class EnhanceUserProfile
+        #     include Attr::Gather::Workflow
+        #
+        #     # ...
+        #
+        #     fetch :user_info do |t|
+        #       t.depends_on = [:fetch_gravatar_info]
+        #     end
+        #   end
+        #
+        # Calling `fetch` will yield a task object which you can configure like
+        # a PORO. Tasks will be registered for execution in the workflow.
+        #
+        # @yield [Attr::Gather::Workflow::Task] A task to configure
+        #
+        # @api public
+        def fetch(task_name, opts = EMPTY_HASH)
+          task(task_name, opts)
+        end
+
+        # Defines a task with name and options
+        #
+        # @param task_name [Symbol] the name of the task
+        #
+        # @example
+        #   class EnhanceUserProfile
+        #     include Attr::Gather::Workflow
+        #
+        #     # ...
+        #
+        #     step :fetch_user_info do |t|
+        #       t.depends_on = [:email_info]
+        #     end
+        #   end
+        #
+        # Calling `step` will yield a task object which you can configure like
+        # a PORO. Tasks will be registered for execution in the workflow.
+        #
+        # @yield [Attr::Gather::Workflow::Task] A task to configure
+        #
+        # @api public
+        def step(task_name, opts = EMPTY_HASH)
+          task(task_name, opts)
+        end
+
         # Defines a container for task dependencies
         #
         # Using a container  makes it easy to re-use workflows with different
@@ -55,7 +105,7 @@ module Attr
         #   end
         #
         #   class EnhanceUserProfile
-        #     extend Attr::Gather::Workflow
+        #     include Attr::Gather::Workflow
         #
         #     container LegacySystem
         #   end
@@ -79,7 +129,7 @@ module Attr
         #
         # @example
         #   class EnhanceUserProfile
-        #     extend Attr::Gather::Workflow
+        #     include Attr::Gather::Workflow
         #
         #     aggregator :deep_merge
         #   end
@@ -119,7 +169,7 @@ module Attr
         #   end
         #
         #   class EnhanceUserProfile
-        #     extend Attr::Gather::Workflow
+        #     include Attr::Gather::Workflow
         #
         #     # Any of the key/value pairs that had validation errors will be
         #     # filtered from the output.
@@ -151,7 +201,7 @@ module Attr
         # @example
         #
         #   class EnhanceUserProfile
-        #     extend Attr::Gather::Workflow
+        #     include Attr::Gather::Workflow
         #
         #     # Any of the key/value pairs that had validation errors will be
         #     # filtered from the output.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: attr-gather
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Ian Ker-Seymer
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-10-22 00:00:00.000000000 Z
+date: 2021-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -135,7 +135,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="