farmstead 0.0.86 → 0.0.87

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f0899567bd40fc4ff0078c240c1b45601aa59150
-  data.tar.gz: c549929e975690c27cd379725a08bd751745242a
+  metadata.gz: 5e09b815a7a63b725f6e2d723f2bb09287530eda
+  data.tar.gz: 4710c794296aa0004eadf477cb219816ed418cab
 SHA512:
-  metadata.gz: eec1107db0c3a267bfc8cbc8b85afbea90c109287dc02c44a89b082bb624f0b58e4bb28d70cc502f2c999c0406047e649e8d0125331d566ea140e158c6d7ee29
-  data.tar.gz: 7971b213a236620f123284252b700da4ffa7dbd82a19e0b0f893892108ab89f283526aac636850aed2aa1579c2331b2ebaf230391427763ddc51934b9d1daaf3
+  metadata.gz: 184c8e940ea27b2dba8d729b291c1da6872b47bd92aa2d4efce8e65c8fee25af113f9122811e2630715b1e12b07fdac9da10996f822c15852a0eb177118e1b9c
+  data.tar.gz: cd8302fb0187ae2b5eb4dcd104884cea6c53981ac5b5b66ca550ca1324313ea460d9711e9d34e07d4d15c910231622f42909691735716167157207e8a918656b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    farmstead (0.0.86)
+    farmstead (0.0.87)
       httparty (~> 0.15.6)
       mechanize (~> 2.7)
       mysql2 (~> 0.4.10)
@@ -9,8 +9,8 @@ PATH
       os (~> 1.0)
       ruby-kafka (~> 0.5.1)
       sequel (~> 5.4.0)
-      sinatra (~> 2.0.0)
-      sinatra-contrib (~> 2.0.0)
+      sinatra (~> 2.0.1)
+      sinatra-contrib (~> 2.0.1)
       thor (~> 0.20.0)
 
 GEM
@@ -26,7 +26,7 @@ GEM
     ffi (1.9.18)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    httparty (0.15.6)
+    httparty (0.15.7)
       multi_xml (>= 0.5.2)
     listen (3.0.8)
       rb-fsevent (~> 0.9, >= 0.9.4)
@@ -46,7 +46,7 @@ GEM
     mini_portile2 (2.3.0)
     multi_json (1.13.1)
     multi_xml (0.6.0)
-    mustermann (1.0.1)
+    mustermann (1.0.2)
     mysql2 (0.4.10)
     net-http-digest_auth (1.4.1)
     net-http-persistent (2.9.4)
@@ -54,8 +54,8 @@ GEM
       mini_portile2 (~> 2.3.0)
     ntlm-http (0.1.1)
     os (1.0.0)
-    rack (2.0.3)
-    rack-protection (2.0.0)
+    rack (2.0.4)
+    rack-protection (2.0.1)
       rack
     rake (10.5.0)
     rb-fsevent (0.10.2)
@@ -74,25 +74,25 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.0)
-    ruby-kafka (0.5.2)
+    ruby-kafka (0.5.3)
     sequel (5.4.0)
-    sinatra (2.0.0)
+    sinatra (2.0.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.0)
+      rack-protection (= 2.0.1)
       tilt (~> 2.0)
-    sinatra-contrib (2.0.0)
+    sinatra-contrib (2.0.1)
       backports (>= 2.0)
       multi_json
       mustermann (~> 1.0)
-      rack-protection (= 2.0.0)
-      sinatra (= 2.0.0)
+      rack-protection (= 2.0.1)
+      sinatra (= 2.0.1)
       tilt (>= 1.3, < 3)
     thor (0.20.0)
     tilt (2.0.8)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.4)
+    unf_ext (0.0.7.5)
     webrobots (0.1.2)
 
 PLATFORMS

data/farmstead.gemspec

@@ -31,8 +31,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "mechanize", "~> 2.7"
   spec.add_runtime_dependency "httparty", "~> 0.15.6"
   spec.add_runtime_dependency "sequel", "~> 5.4.0"
-  spec.add_runtime_dependency "sinatra", "~> 2.0.0"
-  spec.add_runtime_dependency "sinatra-contrib", "~> 2.0.0"
+  spec.add_runtime_dependency "sinatra", "~> 2.0.1"
+  spec.add_runtime_dependency "sinatra-contrib", "~> 2.0.1"
 
   spec.add_development_dependency "dotenv", "~> 0.11.1"
   spec.add_development_dependency "listen", "~> 3.0"

data/lib/farmstead/db.rb

@@ -24,6 +24,8 @@ module Farmstead
         String(:name)
         String(:type)
         String(:module)
+        TrueClass(:pickedup)
+        TrueClass(:processed)
       end
       @@DB.create_table :test do
         primary_key(:id)
@@ -86,5 +88,15 @@ module Farmstead
       ds.insert(result: result)
     end
 
+    # Sets the value of pickedup to true
+    def mark_pickedup(source)
+      @@DB.run("UPDATE sources SET pickedup = 1 WHERE module = #{source}")
+    end
+
+    # Sets the value of processed to true
+    def mark_processed(source)
+      @@DB.run("UPDATE sources SET processed = 1 WHERE module = #{source}")
+    end
+
   end
 end

data/lib/farmstead/pipeline/manage.rb

@@ -1,17 +1,17 @@
 # Manage
 #
 # It works off of the DB when
-# 1) A new site is added
-# 2) A scheduled site pull is configured to happen
+# 1) A new source is added
+# 2) A scheduled source pull is configured to happen
 #
 # Field --> Forest --> Road
 #
 # It then takes the config from the DB and passed it to the Field topic
 # 
-# The Extract class pulls the site config from the Field Topic and then pushes
-# site data to the Forest topic.
+# The Extract class pulls the source config from the Field Topic and then pushes
+# source data to the Forest topic.
 # 
-# The Transform class pulls the site data from teh Forest topic and then pushes 
+# The Transform class pulls the source data from teh Forest topic and then pushes 
 # to the Road topic.
 # 
 # The Load class pulls data from the Road topic and loads it into the database.
@@ -32,6 +32,7 @@ module Farmstead
           puts module_name
           @producer.produce(module_name, topic: "Field")
           @producer.deliver_messages
+          Farmstead::DB.mark_pickedup(module_name)
         end
       end
 
@@ -40,25 +41,15 @@ module Farmstead
       def check_sources
         sources = @mysql.query("SELECT * FROM sources WHERE pickedup = 'false'")
         return false if sources.count.zero?
-        sources.each do |site|
-          json = site.to_json
-          siteid = get_from_json(json, 'id')
-          # import_site(json, siteid)
+        sources.each do |source|
+          json = source.to_json
+          sourceid = get_from_json(json, 'id')
+          # import_source(json, sourceid)
           write_message(json, topic: 'Wood')
-          mark_pickedup(siteid)
+          mark_pickedup(sourceid)
         end
       end
 
-      # Sets the value of pickedup to true
-      def mark_pickedup(siteid)
-        @mysql.query("UPDATE sites SET pickedup = 'true' WHERE id = #{siteid}")
-      end
-
-      # Sets the value of processed to true
-      def mark_processed(siteid)
-        @mysql.query("UPDATE sites SET processed = 'true' WHERE id = #{siteid}")
-      end
-
       # Checks for any processing tasks that need to be
       # completed at speicifc times
       def regular_tasks
@@ -72,9 +63,9 @@ module Farmstead
         end
       end
 
-      # Imports site data as a Hash into MySQL DB
-      def import_site(sitehash, siteid)
-        sitehash
+      # Imports source data as a Hash into MySQL DB
+      def import_source(sourcehash, sourceid)
+        sourcehash
       end
     end
   end

data/lib/farmstead/service.rb

@@ -39,23 +39,6 @@ module Farmstead
       end
     end
 
-    # Each Farmstead site has a Class that defines it
-    # Each service runs one or more methods on that Class
-    def find_definitions(site, service)
-      mysite = @mysql.query("SELECT * FROM sites WHERE name = '#{site}'")
-      return false if mysite.count.zero?
-      mysite.each do |sited|
-        json = sited.to_json
-        config = get_from_json(json, 'config')
-        # Convert to hash where each element is a service and then associate it
-        # with the service name passed to this method
-      end
-    end
-
-    def doit
-      find_definitions('Yahoo', 'scarecrow')
-    end
-
     def write_message(message, topic)
       @producer.produce(message, topic)
       @producer.deliver_messages

data/lib/farmstead/version.rb

@@ -1,3 +1,3 @@
 module Farmstead
-  VERSION = "0.0.86"
+  VERSION = "0.0.87"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: farmstead
 version: !ruby/object:Gem::Version
-  version: 0.0.86
+  version: 0.0.87
 platform: ruby
 authors:
 - Ken Jenney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-30 00:00:00.000000000 Z
+date: 2018-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -128,28 +128,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.1
 - !ruby/object:Gem::Dependency
   name: sinatra-contrib
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.1
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement