farmstead 0.0.48 → 0.0.49

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76eed38e0bcb82ac852d9c5909f5b196e158dec9
-  data.tar.gz: 02c6768169ac7b44d792b1ac728ec2df7f291d6f
+  metadata.gz: ec037c550284eabc999c7005441c22fb30c76e76
+  data.tar.gz: b1150538511bafdaf0abd042dae20a2659bd5d23
 SHA512:
-  metadata.gz: 967d13b9a83c5ddc18439538a9a0fe152d115344095669d10439d2f2023ec050b803dbb8cde264e6fe0c22e8a045b7affbee440ed8693c1cd30a0994b42d2269
-  data.tar.gz: 5263d000b55a24a1196530c5c9c407d92f1774be5859713c0ddd9eb3c45569e0cb536d4d9f28991f8a459a91bef5d0a88bb89fc0f2b764dd6e468b1f9275334e
+  metadata.gz: f10b4378ecdb678166eb64a73bd0c11cda1d9339130623b5bd79c39fdeeb508aa1ca4a9fa6e3a2b836d76a5437f3a40c193cf259a2266b897886c762bce4d26a
+  data.tar.gz: 9c0157d4d40f1dec9d98ace549b5b12d147b0d97402df89bdd903a7d4b4ec93d17b5ede53d8bce2e28b8491d08d3e7efd790c274c94f8c244278e82d5628cec0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    farmstead (0.0.48)
+    farmstead (0.0.49)
       httparty (~> 0.15.6)
       mechanize (~> 2.7)
       mysql2 (~> 0.4.10)
@@ -16,7 +16,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    backports (3.11.0)
+    backports (3.11.1)
     diff-lcs (1.3)
     domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)

data/lib/farmstead/pipeline/extract.rb

@@ -19,7 +19,7 @@ module Farmstead
           obj = JSON.parse(message.value)
           my_module = Object.const_get "<%= ENV['name'].capitalize %>::#{obj["module"]}"
           result = my_module::extract
-          Farmstead::DB.insert("test",[result])
+          Farmstead::DB.insert("test",result: result)
           @producer.produce(result, topic: "Forest")
           @producer.deliver_messages
           @consumer.mark_message_as_processed(message)

data/lib/farmstead/pipeline/load.rb

@@ -3,43 +3,26 @@
 # Load data into database
 #
 # Runs a Consumer and it will automatically pick up
-# messages from the Field Topic and do it's job
-# and then send a message as a Producer to the Forest Topic
+# messages from the Road Topic and load the message into
+# the database
 #
 # Every micro-service inherits the Service class
 module Farmstead
   module Load
-    # Picks up JSON generated by WebDriver and save it to Forest topic
-    class Producer < Farmstead::Service
+    class Service < Farmstead::Service
       def run
-        loop do
-          puts "Do something"
-          sleep 300
-        end
-      end
-    end
-
-    # Subscribed to the Field topic
-    # Works on message
-    class Consumer < Farmstead::Service
-      def run
-        @consumer.subscribe('Field')
+        @consumer.subscribe("Road")
         trap('TERM') { @consumer.stop }
         @consumer.each_message do |message|
           puts "Received: #{message.value}"
-          magic_work(message.value)
+          # Run the load method of the module referenced by the message
+          obj = JSON.parse(message.value)
+          my_module = Object.const_get "<%= ENV['name'].capitalize %>::#{obj["module"]}"
+          result = my_module::load
+          Farmstead::DB.insert("test",result: result)
           @consumer.mark_message_as_processed(message)
         end
       end
-
-      def magic_work(site)
-        hash = JSON.parse(site)
-        hash['scarecrow'] = 'true'
-        json = hash.to_json
-        puts "Writing: #{json}"
-        write_message(json, topic: 'Forest')
-      end
     end
   end
 end
-

data/lib/farmstead/pipeline/manage.rb

@@ -4,23 +4,17 @@
 # 1) A new site is added
 # 2) A scheduled site pull is configured to happen
 #
-# It then takes the config from the DB and passed it to the Wood topic
-#
-# Tinman is running as a Consumer and it will automatically pick up the message
-# and do it's job and then send a message (as a Producer) to the Field topic
-#
-# Scarecrow is running as a Consumer and it will automatically pick up the
-# message and do it's job and then send a message (as a Producer)
-# to the Forest topic
+# Field --> Forest --> Road
 #
-# CowardlyLion is running as a Consumer and it will automatically pick up the
-# message and do it's job and then send a message (as a Producer)
-# to the Road topic
-#
-# Glenda is running as a Consumer and it will automatically pick up messages
-# from the Road topic. This is the final product of scraping a site. It's stored
-# in a Hash. Glenda imports the Hash into the MySQL database where it is
-# presented by Dorothy
+# It then takes the config from the DB and passed it to the Wood topic
+# 
+# The Extract class pulls the site config from the Field Topic and then pushes
+# site data to the Forest topic.
+# 
+# The Transform class pulls the site data from teh Forest topic and then pushes 
+# to the Road topic.
+# 
+# The Load class pulls data from the Road topic and loads it into the database.
 #
 # Topics are created when Kafka comes up
 # HINT: See .env

data/lib/farmstead/pipeline/transform.rb

@@ -12,36 +12,22 @@
 # Every micro-service inherits the Service class
 module Farmstead
   module Transform
-    # Does nothing...work is handled by magic_work
-    class Producer < Farmstead::Service
+    class Service < Farmstead::Service
       def run
-        loop do
-          puts "Do nothing"
-          sleep 300
-        end
-      end
-    end
-
-    # Subscribed to the Field topic
-    # Works on message
-    class Consumer < Farmstead::Service
-      def run
-        @consumer.subscribe('Forest')
+        @consumer.subscribe("Forest")
         trap('TERM') { @consumer.stop }
         @consumer.each_message do |message|
           puts "Received: #{message.value}"
-          magic_work(message.value)
+          # Run the load method of the module referenced by the message
+          obj = JSON.parse(message.value)
+          my_module = Object.const_get "<%= ENV['name'].capitalize %>::#{obj["module"]}"
+          result = my_module::transform
+          Farmstead::DB.insert("test",result: result)
+          @producer.produce(result, topic: "Road")
+          @producer.deliver_messages
           @consumer.mark_message_as_processed(message)
         end
       end
-
-      def magic_work(site)
-        hash = JSON.parse(site)
-        hash['cowardlylion'] = 'true'
-        json = hash.to_json
-        puts "Writing: #{json}"
-        write_message(json, topic: 'Road')
-      end
     end
   end
 end

data/lib/farmstead/version.rb

@@ -1,3 +1,3 @@
 module Farmstead
-  VERSION = "0.0.48"
+  VERSION = "0.0.49"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: farmstead
 version: !ruby/object:Gem::Version
-  version: 0.0.48
+  version: 0.0.49
 platform: ruby
 authors:
 - Ken Jenney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-23 00:00:00.000000000 Z
+date: 2018-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor