farmstead 0.0.15 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46565ad3447fc8f46e19db0e106bd46852d7de3e
-  data.tar.gz: 58fd7b7cb76a47f8b8a1c854ec85a5c7fe84cc87
+  metadata.gz: 6853da09a2d0ccd7d0dcbae85a830e6968fd2b5f
+  data.tar.gz: bdd81aa444d6adc620bd4535e7d65e4ba2c4ef17
 SHA512:
-  metadata.gz: 856eea79a42d3fdaadb5a9b8764d7c58f0e239bf9181bfaa3eaff675a437180fda7f3cb6e443b129546167d5eabc75ff1252a39acec3fdfb79882437053953ba
-  data.tar.gz: 3e3034737b10d7721a84cff988797a52a143a891489dc532a770a271dd11947e3489043faa7d75c0bc197ae74f62b99d14b9e4e3fe8b94dd108168cf44397190
+  metadata.gz: 878aaa7062cabba0158015b61707e1cda3e686e19c677d9fa5652a796eb5ce01be0b8533d54c0808ba1e9368bb5c98c329c03ffcedc39af2a27734137990544e
+  data.tar.gz: 1250b8c9ad1ff619e0679e818848d3e732edd3c268369735c11d48571b72bdcf1b87196e01515f58c79af01f883cd0c2831f492a518e37abf960339e0a16ba4e

data/Gemfile.lock

@@ -1,13 +1,14 @@
 PATH
   remote: .
   specs:
-    farmstead (0.0.15)
+    farmstead (0.0.16)
       httparty (~> 0.15.6)
       mechanize (~> 2.7)
       mysql2 (~> 0.4.10)
       nokogiri (~> 1.8)
       os (~> 1.0)
       ruby-kafka (~> 0.5.1)
+      sinatra (~> 2.0.0)
       thor (~> 0.20.0)
 
 GEM
@@ -41,6 +42,7 @@ GEM
     mime-types-data (3.2016.0521)
     mini_portile2 (2.3.0)
     multi_xml (0.6.0)
+    mustermann (1.0.1)
     mysql2 (0.4.10)
     net-http-digest_auth (1.4.1)
     net-http-persistent (2.9.4)
@@ -48,6 +50,9 @@ GEM
       mini_portile2 (~> 2.3.0)
     ntlm-http (0.1.1)
     os (1.0.0)
+    rack (2.0.3)
+    rack-protection (2.0.0)
+      rack
     rake (10.5.0)
     rb-fsevent (0.10.2)
     rb-inotify (0.9.10)
@@ -66,7 +71,13 @@ GEM
       rspec-support (~> 3.7.0)
     rspec-support (3.7.0)
     ruby-kafka (0.5.2)
+    sinatra (2.0.0)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.0)
+      tilt (~> 2.0)
     thor (0.20.0)
+    tilt (2.0.8)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.4)

data/farmstead.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "nokogiri", "~> 1.8"
   spec.add_runtime_dependency "mechanize", "~> 2.7"
   spec.add_runtime_dependency "httparty", "~> 0.15.6"
+  spec.add_runtime_dependency "sinatra", "~> 2.0.0"
 
   spec.add_development_dependency "dotenv", "~> 0.11.1"
   spec.add_development_dependency "listen", "~> 3.0"

data/lib/farmstead/api/defaultroutes.rb

@@ -0,0 +1,14 @@
+module Sinatra
+  module DefaultRoutes
+    def self.registered(app)
+      app.get "/" do
+        "Welcome to our API"
+      end
+      
+      app.get "/version" do
+        "Farmstead #{Farmstead::VERSION}"
+      end
+    end
+  end
+  register DefaultRoutes
+end

data/lib/farmstead/api/helpers.rb

@@ -0,0 +1,13 @@
+module Sinatra
+  module AuthHelper
+      def require_logged_in
+        redirect('/sessions/new') unless is_authenticated?
+      end
+
+      def is_authenticated?
+        return !!session[:user_id]
+      end
+  end
+
+  helpers AuthHelper
+end

data/lib/farmstead/cli.rb

@@ -28,6 +28,12 @@ module Farmstead
       Farmstead::Project.deploy
     end
 
+    desc "pipelined", "Test a pipeline API"
+    def pipeline
+      Farmstead.const_set("OPERATION", "started")
+      require_relative "farmstead/extract"
+    end
+
     desc "test COMMANDS", "Test commands"
     subcommand "test", Farmstead::CLITest
 

data/lib/farmstead/config.rb

@@ -0,0 +1,4 @@
+module Farmstead
+  config_yaml = YAML.load_file 'itinerary.yml'
+  Config = OpenStruct.new(itinerary_yaml)
+end

data/lib/farmstead/pipeline/extract.rb

@@ -8,33 +8,37 @@
 #
 # Every micro-service inherits the Service class
 module Farmstead
-  class Extract < Service
-    # Picks up JSON generated by WebDriver and save it to Forest topic
-    def producer
-      loop do
-        puts "Do something"
-        sleep 300
+  OPERATION = 1
+  module Extract
+    class Producer < Farmstead::Service
+      def doit
+        loop do
+          puts "Do something"
+          sleep 300
+        end
       end
     end
 
     # Subscribed to the Field topic
     # Works on message
-    def consumer
-      @consumer.subscribe('Field')
-      trap('TERM') { @consumer.stop }
-      @consumer.each_message do |message|
-        puts "Received: #{message.value}"
-        magic_work(message.value)
-        @consumer.mark_message_as_processed(message)
+    class Consumer < Farmstead::Service
+      def doit
+        @consumer.subscribe('Field')
+        trap('TERM') { @consumer.stop }
+        @consumer.each_message do |message|
+          puts "Received: #{message.value}"
+          magic_work(message.value)
+          @consumer.mark_message_as_processed(message)
+        end
       end
-    end
 
-    def magic_work(site)
-      hash = JSON.parse(site)
-      hash['scarecrow'] = 'true'
-      json = hash.to_json
-      puts "Writing: #{json}"
-      write_message(json, topic: 'Forest')
+      def magic_work(site)
+        hash = JSON.parse(site)
+        hash['scarecrow'] = 'true'
+        json = hash.to_json
+        puts "Writing: #{json}"
+        write_message(json, topic: 'Forest')
+      end
     end
   end
 end

data/lib/farmstead/pipeline/load.rb

@@ -8,33 +8,37 @@
 #
 # Every micro-service inherits the Service class
 module Farmstead
-  class Load < Service
+  module Load
     # Picks up JSON generated by WebDriver and save it to Forest topic
-    def producer
-      loop do
-        puts "Do something"
-        sleep 300
+    class Producer < Farmstead::Service
+      def doit
+        loop do
+          puts "Do something"
+          sleep 300
+        end
       end
     end
 
     # Subscribed to the Field topic
     # Works on message
-    def consumer
-      @consumer.subscribe('Field')
-      trap('TERM') { @consumer.stop }
-      @consumer.each_message do |message|
-        puts "Received: #{message.value}"
-        magic_work(message.value)
-        @consumer.mark_message_as_processed(message)
+    class Consumer < Farmstead::Service
+      def doit
+        @consumer.subscribe('Field')
+        trap('TERM') { @consumer.stop }
+        @consumer.each_message do |message|
+          puts "Received: #{message.value}"
+          magic_work(message.value)
+          @consumer.mark_message_as_processed(message)
+        end
       end
-    end
 
-    def magic_work(site)
-      hash = JSON.parse(site)
-      hash['scarecrow'] = 'true'
-      json = hash.to_json
-      puts "Writing: #{json}"
-      write_message(json, topic: 'Forest')
+      def magic_work(site)
+        hash = JSON.parse(site)
+        hash['scarecrow'] = 'true'
+        json = hash.to_json
+        puts "Writing: #{json}"
+        write_message(json, topic: 'Forest')
+      end
     end
   end
 end

data/lib/farmstead/pipeline/manager.rb

@@ -0,0 +1,115 @@
+# Manager
+#
+# It works off of the DB when
+# 1) A new site is added
+# 2) A scheduled site pull is configured to happen
+#
+# It then takes the config from the DB and passed it to the Wood topic
+#
+# Tinman is running as a Consumer and it will automatically pick up the message
+# and do it's job and then send a message (as a Producer) to the Field topic
+#
+# Scarecrow is running as a Consumer and it will automatically pick up the
+# message and do it's job and then send a message (as a Producer)
+# to the Forest topic
+#
+# CowardlyLion is running as a Consumer and it will automatically pick up the
+# message and do it's job and then send a message (as a Producer)
+# to the Road topic
+#
+# Glenda is running as a Consumer and it will automatically pick up messages
+# from the Road topic. This is the final product of scraping a site. It's stored
+# in a Hash. Glenda imports the Hash into the MySQL database where it is
+# presented by Dorothy
+#
+# Topics are created when Kafka comes up
+# HINT: See .env
+# Every micro-service inherits the Service class
+module Farmstead
+  module Manager
+    class Producer < Farmstead::Service
+      def doit
+        loop do
+          puts 'Checking sites'
+          check_sites
+          puts 'Checking tasks'
+          # regular_tasks
+          sleep 3
+        end
+      end
+
+      # Checks for any new sites to be processed
+      # Adds them to the message queue
+      def check_sites
+        sites = @mysql.query("SELECT * FROM sites WHERE pickedup = 'false'")
+        return false if sites.count.zero?
+        sites.each do |site|
+          json = site.to_json
+          siteid = get_from_json(json, 'id')
+          # import_site(json, siteid)
+          write_message(json, topic: 'Wood')
+          mark_pickedup(siteid)
+        end
+      end
+    end
+
+    # Subscribed to the Road topic
+    # Imports Hash into MySQL Database for each message
+    class Consumer < Farmstead::Service
+      def doit
+        @consumer.subscribe('Road')
+        trap('TERM') { @consumer.stop }
+        @consumer.each_message do |message|
+          puts "Received: #{message.value}"
+          hash = JSON.parse(message.value)
+          import_site(hash, hash[:id])
+          mark_processed(hash[:id])
+          @consumer.mark_message_as_processed(message)
+        end
+      end
+
+      # Checks for any new sites to be processed
+      # Adds them to the message queue
+      def check_sites
+        sites = @mysql.query("SELECT * FROM sites WHERE pickedup = 'false'")
+        return false if sites.count.zero?
+        sites.each do |site|
+          json = site.to_json
+          siteid = get_from_json(json, 'id')
+          # import_site(json, siteid)
+          write_message(json, topic: 'Wood')
+          mark_pickedup(siteid)
+        end
+      end
+
+      # Sets the value of pickedup to true
+      def mark_pickedup(siteid)
+        @mysql.query("UPDATE sites SET pickedup = 'true' WHERE id = #{siteid}")
+      end
+
+      # Sets the value of processed to true
+      def mark_processed(siteid)
+        @mysql.query("UPDATE sites SET processed = 'true' WHERE id = #{siteid}")
+      end
+
+      # Checks for any processing tasks that need to be
+      # completed at speicifc times
+      def regular_tasks
+        tasks = @mysql.query("SELECT * FROM tasks WHERE processed = 'false'")
+        return false if tasks.count.zero?
+        tasks.each do |task|
+          json = task.to_json
+          taskid = get_id(task)
+          write_message(json, topic: 'Wood')
+          mark_pickedup(taskid)
+        end
+      end
+
+      # Imports site data as a Hash into MySQL DB
+      def import_site(sitehash, siteid)
+        sitehash
+      end
+    end
+  end
+end
+

data/lib/farmstead/pipeline/transform.rb

@@ -11,33 +11,37 @@
 #
 # Every micro-service inherits the Service class
 module Farmstead
-  class Transform < Service
+  module Transform
     # Does nothing...work is handled by magic_work
-    def producer
-      loop do
-        puts "Do nothing"
-        sleep 300
+    class Producer < Farmstead::Service
+      def doit
+        loop do
+          puts "Do nothing"
+          sleep 300
+        end
       end
     end
 
     # Subscribed to the Field topic
     # Works on message
-    def consumer
-      @consumer.subscribe('Forest')
-      trap('TERM') { @consumer.stop }
-      @consumer.each_message do |message|
-        puts "Received: #{message.value}"
-        magic_work(message.value)
-        @consumer.mark_message_as_processed(message)
+    class Consumer < Farmstead::Service
+      def doit
+        @consumer.subscribe('Forest')
+        trap('TERM') { @consumer.stop }
+        @consumer.each_message do |message|
+          puts "Received: #{message.value}"
+          magic_work(message.value)
+          @consumer.mark_message_as_processed(message)
+        end
       end
-    end
 
-    def magic_work(site)
-      hash = JSON.parse(site)
-      hash['cowardlylion'] = 'true'
-      json = hash.to_json
-      puts "Writing: #{json}"
-      write_message(json, topic: 'Road')
+      def magic_work(site)
+        hash = JSON.parse(site)
+        hash['cowardlylion'] = 'true'
+        json = hash.to_json
+        puts "Writing: #{json}"
+        write_message(json, topic: 'Road')
+      end
     end
   end
 end

data/lib/farmstead/scaffold/Dockerfile.erb

@@ -19,6 +19,7 @@ WORKDIR /service
 ADD Gemfile* /service/
 RUN gem install bundler && bundle install --jobs 20 --retry 5
 ADD . /service
+RUN gem install farmstead-0.0.15.gem
 
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 

data/lib/farmstead/scaffold/Gemfile.erb

@@ -5,7 +5,7 @@ source "https://rubygems.org"
 git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
 gem "coffee-rails", "~> 4.2"
-gem "farmstead"
+#gem "farmstead"
 gem "httparty", "~> 0.15.6"
 gem "jbuilder", "~> 2.5"
 gem "jquery-rails", "~> 4.3.1"

data/lib/farmstead/scaffold/docker-compose.yml.erb

@@ -29,7 +29,7 @@ services:
       - 'mysql'
     build: .
     environment:
-      SERVICE: manager
+      SERVICE: Manager
     ports:
       - '3000:3000'
     env_file:
@@ -40,7 +40,7 @@ services:
       - 'manage'
     build: .
     environment:
-      SERVICE: extracter
+      SERVICE: Extracter
     env_file:
       - '.env'
 
@@ -49,7 +49,7 @@ services:
       - 'manage'
     build: .
     environment:
-      SERVICE: transformer
+      SERVICE: Transformer
     env_file:
       - '.env'
 
@@ -58,6 +58,6 @@ services:
       - 'manage'
     build: .
     environment:
-      SERVICE: loader
+      SERVICE: Loader
     env_file:
       - '.env'

data/lib/farmstead/scaffold/extract/extracter.rb.erb

@@ -1,7 +1,9 @@
 module <%= @name.capitalize %>
-  class Extracter < Farmstead::Extract
-    def hello
-      p "hello"
+  module Extracter
+    include Farmstead::Extract
+    class API < Sinatra::Base
+      register Sinatra::DefaultRoutes
+      set :port, 22340
     end
   end
 end

data/lib/farmstead/scaffold/load/loader.rb.erb

@@ -1,5 +1,6 @@
 module <%= @name.capitalize %>
-  class Loader < Farmstead::Load
+  class Loader
+    include Farmstead::Load
     def hello
       p "hello"
     end

data/lib/farmstead/scaffold/project.rb.erb

@@ -5,8 +5,8 @@
 # 1st - the class you want to call
 # 2nd - the method you want to call
 
-dynamic_class = ARGV[0].capitalize
-dynamic_method = ARGV[1]
+dynamic_module = ARGV[0].capitalize
+dynamic_class = ARGV[1]
 
 # Extend Farmstead
 require "farmstead"
@@ -20,6 +20,6 @@ module <%= @name.capitalize %>
   include Farmstead
 end
 
-klass = Object.const_get "<%= @name.capitalize %>::#{dynamic_class}"
+klass = Object.const_get "<%= @name.capitalize %>::#{dynamic_module}::#{dynamic_class}"
 service = klass.new
-service.send(dynamic_method)
+service.send("doit")

data/lib/farmstead/scaffold/supervisord.conf.erb

@@ -2,9 +2,9 @@
 nodaemon=true
 
 [program:producer]
-command=/usr/local/bin/ruby /service/project.rb %(ENV_SERVICE)s producer
+command=/usr/local/bin/ruby /service/project.rb %(ENV_SERVICE)s Producer
 redirect_stderr=true
 
 [program:consumer]
-command=/usr/local/bin/ruby /service/project.rb %(ENV_SERVICE)s consumer
+command=/usr/local/bin/ruby /service/project.rb %(ENV_SERVICE)s Consumer
 redirect_stderr=true

data/lib/farmstead/scaffold/transform/transformer.rb.erb

@@ -1,5 +1,6 @@
 module <%= @name.capitalize %>
-  class Transformer < Farmstead::Transform
+  class Transformer
+    include Farmstead::Transform
     def hello
       p "hello"
     end

data/lib/farmstead/version.rb

@@ -1,3 +1,3 @@
 module Farmstead
-  VERSION = "0.0.15"
+  VERSION = "0.0.16"
 end

data/lib/farmstead.rb

@@ -30,16 +30,21 @@ require "nokogiri"
 require "httparty"
 require "open-uri"
 require "mechanize"
+require "sinatra/base"
 
 require "farmstead/version"
 require "farmstead/project"
 require "farmstead/cli"
+require "farmstead/config"
 require "farmstead/service"
-require "farmstead/manager"
+require "farmstead/pipeline/manager"
 require "farmstead/pipeline/extract"
 require "farmstead/pipeline/transform"
 require "farmstead/pipeline/load"
 
+require "farmstead/api/defaultroutes.rb"
+require "farmstead/api/helpers.rb"
+
 module Farmstead
   # Your code goes here...
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: farmstead
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.16
 platform: ruby
 authors:
 - Ken Jenney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-14 00:00:00.000000000 Z
+date: 2018-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.15.6
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -200,18 +214,16 @@ files:
 - README.md
 - Rakefile
 - bin/farmstead
-- examples/myproject/extract/extracter.rb
-- examples/myproject/load/loader.rb
-- examples/myproject/myproject.rb
-- examples/myproject/myproject.yml
-- examples/myproject/transform/transformer.rb
 - farmstead.gemspec
 - lib/farmstead.rb
+- lib/farmstead/api/defaultroutes.rb
+- lib/farmstead/api/helpers.rb
 - lib/farmstead/cli.rb
 - lib/farmstead/cli/test.rb
-- lib/farmstead/manager.rb
+- lib/farmstead/config.rb
 - lib/farmstead/pipeline/extract.rb
 - lib/farmstead/pipeline/load.rb
+- lib/farmstead/pipeline/manager.rb
 - lib/farmstead/pipeline/transform.rb
 - lib/farmstead/project.rb
 - lib/farmstead/scaffold/.dockerignore.erb

data/examples/myproject/extract/extracter.rb

@@ -1,7 +0,0 @@
-module MyProject
-  class Extracter < Farmstead::Extract
-    def hello
-      p "hello"
-    end
-  end
-end

data/examples/myproject/load/loader.rb

@@ -1,7 +0,0 @@
-module MyProject
-  class Loader < Farmstead::Load
-    def hello
-      p "hello"
-    end
-  end
-end

data/examples/myproject/myproject.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-# Primary script to kick off services in project
-# Pass two arguments to this script
-# 1st - the class you want to call
-# 2nd - the method you want to call
-
-dynamic_class = ARGV[0]
-dynamic_method = ARGV[1]
-
-# Extend Farmstead
-require "farmstead"
-
-# Load project classes
-require_relative "extract/extracter"
-require_relative "load/loader"
-require_relative "transform/transformer"
-
-module MyProject
-  include Farmstead
-end
-
-klass = Object.const_get "MyProject::#{dynamic_class}"
-service = klass.new
-service.send(dynamic_method)

data/examples/myproject/myproject.yml

@@ -1,17 +0,0 @@
-- name: My Project
-  database:
-    - type: mysql
-    - MYSQL_ROOT_PASSWORD: Rc2$NE99p5%^
-    - MYSQL_DATABASE: farmstead
-    - MYSQL_USER: farmstead
-    - MYSQL_PASSWORD: farmstead
-    - MYSQL_HOST: mysql
-  kafka:
-    - advertise_from_local_ip: false
-    - advertised_ip: 192.168.1.2
-    - zookeeper_address: "zookeeper:2181"
-    - topics
-      - "Manage:1:1"
-      - "Extract:1:1"
-      - "Load:1:1"
-      - "Transform:1:1"

data/examples/myproject/transform/transformer.rb

@@ -1,7 +0,0 @@
-module MyProject
-  class Transformer < Farmstead::Transform
-    def hello
-      p "hello"
-    end
-  end
-end

data/lib/farmstead/manager.rb

@@ -1,99 +0,0 @@
-# Manager
-#
-# It works off of the DB when
-# 1) A new site is added
-# 2) A scheduled site pull is configured to happen
-#
-# It then takes the config from the DB and passed it to the Wood topic
-#
-# Tinman is running as a Consumer and it will automatically pick up the message
-# and do it's job and then send a message (as a Producer) to the Field topic
-#
-# Scarecrow is running as a Consumer and it will automatically pick up the
-# message and do it's job and then send a message (as a Producer)
-# to the Forest topic
-#
-# CowardlyLion is running as a Consumer and it will automatically pick up the
-# message and do it's job and then send a message (as a Producer)
-# to the Road topic
-#
-# Glenda is running as a Consumer and it will automatically pick up messages
-# from the Road topic. This is the final product of scraping a site. It's stored
-# in a Hash. Glenda imports the Hash into the MySQL database where it is
-# presented by Dorothy
-#
-# Topics are created when Kafka comes up
-# HINT: See .env
-# Every micro-service inherits the Service class
-module Farmstead
-  class Manager < Service
-    # Runs on an infinite loop processing records
-    # on MySQL DB and writing messages accordingly
-    def producer
-      loop do
-        puts 'Checking sites'
-        check_sites
-        puts 'Checking tasks'
-        # regular_tasks
-        sleep 3
-      end
-    end
-
-    # Subscribed to the Road topic
-    # Imports Hash into MySQL Database for each message
-    def consumer
-      @consumer.subscribe('Road')
-      trap('TERM') { @consumer.stop }
-      @consumer.each_message do |message|
-        puts "Received: #{message.value}"
-        hash = JSON.parse(message.value)
-        import_site(hash, hash[:id])
-        mark_processed(hash[:id])
-        @consumer.mark_message_as_processed(message)
-      end
-    end
-
-    # Checks for any new sites to be processed
-    # Adds them to the message queue
-    def check_sites
-      sites = @mysql.query("SELECT * FROM sites WHERE pickedup = 'false'")
-      return false if sites.count.zero?
-      sites.each do |site|
-        json = site.to_json
-        siteid = get_from_json(json, 'id')
-        # import_site(json, siteid)
-        write_message(json, topic: 'Wood')
-        mark_pickedup(siteid)
-      end
-    end
-
-    # Sets the value of pickedup to true
-    def mark_pickedup(siteid)
-      @mysql.query("UPDATE sites SET pickedup = 'true' WHERE id = #{siteid}")
-    end
-
-    # Sets the value of processed to true
-    def mark_processed(siteid)
-      @mysql.query("UPDATE sites SET processed = 'true' WHERE id = #{siteid}")
-    end
-
-    # Checks for any processing tasks that need to be
-    # completed at speicifc times
-    def regular_tasks
-      tasks = @mysql.query("SELECT * FROM tasks WHERE processed = 'false'")
-      return false if tasks.count.zero?
-      tasks.each do |task|
-        json = task.to_json
-        taskid = get_id(task)
-        write_message(json, topic: 'Wood')
-        mark_pickedup(taskid)
-      end
-    end
-
-    # Imports site data as a Hash into MySQL DB
-    def import_site(sitehash, siteid)
-      sitehash
-    end
-  end
-end
-