farmstead 0.0.2

data/lib/farmstead.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+#--
+# Copyright (c) 2004-2018 Ken Jenney
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require "kafka"
+require "mysql2"
+require "json"
+require "nokogiri"
+require "httparty"
+require "open-uri"
+require "mechanize"
+
+require "farmstead/version"
+require "farmstead/project"
+require "farmstead/cli"
+require "farmstead/service"
+require "farmstead/tinman"
+require "farmstead/cowardlylion"
+require "farmstead/glenda"
+require "farmstead/scarecrow"
+require "farmstead/tinman"
+
+module Farmstead
+  # Your code goes here...
+end

data/lib/farmstead/cli.rb

@@ -0,0 +1,47 @@
+require "thor"
+# require 'farmstead/cli/net'
+
+module Farmstead
+  class CLI < Thor
+    class_option :verbose, aliases: "-v", type: "boolean", desc: "Be verbose"
+    class_option :config, aliases: "-c", type: "string", desc: "Config file"
+    class_option :database, aliases: "-d", type: "string", desc: "Database"
+    class_option :deploy, aliases: "-x", type: "string", desc: "Deployment Method"
+    desc "new project_name", "Create a new project"
+    def new(project_name)
+      project = Farmstead::Project.new
+      project.name = project_name
+      project.database = options[:database] if options[:database]
+      project.config = options[:config] if options[:config]
+      project.deploy = options[:deploy] if options[:deploy]
+      project.create
+    end
+
+    desc "tinman command", "Send a command to tinman"
+    def tinman(command)
+      instance = Farmstead::Tinman.new
+      instance.send(command)
+    end
+
+    desc "scarecrow command", "Send a command to scarecrow"
+    def scarecrow(command)
+      instance = Farmstead::Scarecrow.new
+      instance.send(command)
+    end
+
+    desc "cowardlylion command", "Send a command to cowardlylion"
+    def cowardlylion(command)
+      instance = Farmstead::Cowardlylion.new
+      instance.send(command)
+    end
+
+    desc "glenda command", "Send a command to glenda"
+    def glenda(command)
+      instance = Farmstead::Glenda.new
+      instance.send(command)
+    end
+
+    # desc "net COMMANDS", "Net control Module"
+    # subcommand "net", Socialinvestigator::CLI::Net
+  end
+end

data/lib/farmstead/cowardlylion.rb

@@ -0,0 +1,41 @@
+# CowardlyLion - the Miller
+#
+# Cowardlylion is responsible for arranging data into usable blocks
+#
+#
+# CowardlyLion is running as a Consumer and it will automatically pick up
+# messages from the Forest topic and do it's job and then send
+# a message as a Producer to the Road topic
+#
+# Every micro-service inherits the Service class
+module Farmstead
+  class Cowardlylion #< Service
+    # Does nothing...work is handled by magic_work
+    def producer
+      loop do
+        puts "Do nothing"
+        sleep 300
+      end
+    end
+
+    # Subscribed to the Field topic
+    # Works on message
+    def consumer
+      @consumer.subscribe('Forest')
+      trap('TERM') { @consumer.stop }
+      @consumer.each_message do |message|
+        puts "Received: #{message.value}"
+        magic_work(message.value)
+        @consumer.mark_message_as_processed(message)
+      end
+    end
+
+    def magic_work(site)
+      hash = JSON.parse(site)
+      hash['cowardlylion'] = 'true'
+      json = hash.to_json
+      puts "Writing: #{json}"
+      write_message(json, topic: 'Road')
+    end
+  end
+end

data/lib/farmstead/glenda.rb

@@ -0,0 +1,99 @@
+# Glenda - the Scheduler
+#
+#
+# It works off of the DB when
+# 1) A new site is added
+# 2) A scheduled site pull is configured to happen
+#
+# It then takes the config from the DB and passed it to the Wood topic
+#
+# Tinman is running as a Consumer and it will automatically pick up the message
+# and do it's job and then send a message (as a Producer) to the Field topic
+#
+# Scarecrow is running as a Consumer and it will automatically pick up the
+# message and do it's job and then send a message (as a Producer)
+# to the Forest topic
+#
+# CowardlyLion is running as a Consumer and it will automatically pick up the
+# message and do it's job and then send a message (as a Producer)
+# to the Road topic
+#
+# Glenda is running as a Consumer and it will automatically pick up messages
+# from the Road topic. This is the final product of scraping a site. It's stored
+# in a Hash. Glenda imports the Hash into the MySQL database where it is
+# presented by Dorothy
+#
+# Topics are created when Kafka comes up
+# HINT: See .env
+# Every micro-service inherits the Service class
+module Farmstead
+  class Glenda < Service
+    # Runs on an infinite loop processing records
+    # on MySQL DB and writing messages accordingly
+    def producer
+      loop do
+        puts 'Checking sites'
+        check_sites
+        puts 'Checking tasks'
+        # regular_tasks
+        sleep 3
+      end
+    end
+
+    # Subscribed to the Road topic
+    # Imports Hash into MySQL Database for each message
+    def consumer
+      @consumer.subscribe('Road')
+      trap('TERM') { @consumer.stop }
+      @consumer.each_message do |message|
+        puts "Received: #{message.value}"
+        hash = JSON.parse(message.value)
+        import_site(hash, hash[:id])
+        mark_processed(hash[:id])
+        @consumer.mark_message_as_processed(message)
+      end
+    end
+
+    # Checks for any new sites to be processed
+    # Adds them to the message queue
+    def check_sites
+      sites = @mysql.query("SELECT * FROM sites WHERE pickedup = 'false'")
+      return false if sites.count.zero?
+      sites.each do |site|
+        json = site.to_json
+        siteid = get_from_json(json, 'id')
+        # import_site(json, siteid)
+        write_message(json, topic: 'Wood')
+        mark_pickedup(siteid)
+      end
+    end
+
+    # Sets the value of pickedup to true
+    def mark_pickedup(siteid)
+      @mysql.query("UPDATE sites SET pickedup = 'true' WHERE id = #{siteid}")
+    end
+
+    # Sets the value of processed to true
+    def mark_processed(siteid)
+      @mysql.query("UPDATE sites SET processed = 'true' WHERE id = #{siteid}")
+    end
+
+    # Checks for any processing tasks that need to be
+    # completed at speicifc times
+    def regular_tasks
+      tasks = @mysql.query("SELECT * FROM tasks WHERE processed = 'false'")
+      return false if tasks.count.zero?
+      tasks.each do |task|
+        json = task.to_json
+        taskid = get_id(task)
+        write_message(json, topic: 'Wood')
+        mark_pickedup(taskid)
+      end
+    end
+
+    # Imports site data as a Hash into MySQL DB
+    def import_site(sitehash, siteid)
+      sitehash
+    end
+  end
+end

data/lib/farmstead/project.rb

@@ -0,0 +1,79 @@
+require "erb"
+
+module Farmstead
+  # creates a Farmstead Project
+  class Project
+    attr_accessor :name
+    attr_accessor :config
+    attr_accessor :database
+    attr_accessor :deploy
+
+    def create
+      create_directory
+      generate_files
+      start_deploy
+    end
+
+    def start_deploy
+      Dir.chdir @name
+      deploy = system ("bash exec.sh")
+      p deploy.inspect
+    end
+
+    # Creates OR RE-Creates the Project Directory
+    def create_directory
+      remove_dir(@name) if Dir.exist?(@name)
+      Dir.mkdir(@name)
+    end
+
+    # Generate from templates in scaffold
+    def generate_files
+      erbfiles = File.join("**", "*.erb")
+      scaffold = Dir.glob(erbfiles, File::FNM_DOTMATCH)
+      scaffold.each do |file|
+        file.gsub! /\bscaffold\b/, ""
+        foldername = File.dirname(file)
+        # Create folder structure of subdirectories
+        if foldername != "/"
+          create_recursive(foldername)
+        end
+        projectpath = "#{@name}#{file}".chomp(".erb")
+        scaffoldpath = "scaffold#{file}"
+        template = File.read(scaffoldpath)
+        results = ERB.new(template).result(binding)
+        copy_to_directory(results, projectpath)
+      end
+    end
+
+    # Recursive Create
+    def create_recursive(path)
+      recursive = path.split("/")
+      directory = ""
+      recursive.each do |sub_directory|
+        directory += sub_directory + "/"
+        Dir.mkdir("#{@name}/#{directory}") unless (File.directory? directory)
+      end
+    end
+
+    # Recursive Remove
+    def remove_dir(path)
+      if File.directory?(path)
+        Dir.foreach(path) do |file|
+          if ((file.to_s != ".") && (file.to_s != ".."))
+            remove_dir("#{path}/#{file}")
+          end
+        end
+        Dir.delete(path)
+      else
+        File.delete(path)
+      end
+    end
+
+    # Copies an ERB Template as a file to the destination directory
+    def copy_to_directory(str, file_name)
+      open(file_name, "a") do |f|
+        f.puts str
+      end
+    end
+  end
+end

data/lib/farmstead/scarecrow.rb

@@ -0,0 +1,55 @@
+# Scarecrow - the Harvester
+#
+# Scarecrow is responsible for extracting data from a source
+#
+# Scarecrow is running as a Consumer and it will automatically pick up
+# messages from the Field Topic and do it's job
+# and then send a message as a Producer to the Forest Topic
+#
+# Every micro-service inherits the Service class
+module Farmstead
+  class Scarecrow < Service
+    # Picks up JSON generated by WebDriver and save it to Forest topic
+    def producer
+      loop do
+        puts 'Do something'
+        sleep 300
+      end
+    end
+
+    # Subscribed to the Field topic
+    # Works on message
+    def consumer
+      @consumer.subscribe('Field')
+      trap('TERM') { @consumer.stop }
+      @consumer.each_message do |message|
+        puts "Received: #{message.value}"
+        magic_work(message.value)
+        @consumer.mark_message_as_processed(message)
+      end
+    end
+
+    def magic_work(site)
+      hash = JSON.parse(site)
+      hash['scarecrow'] = 'true'
+      json = hash.to_json
+      puts "Writing: #{json}"
+      write_message(json, topic: 'Forest')
+    end
+
+    # Call the Site Class
+    def call_class
+      puts 'this'
+    end
+
+    def selenium
+      browser = Watir::Browser.new :chrome
+      browser.goto 'http://www.stackoverflow.com'
+      puts browser.title
+      # browser.text_field(title: 'Search').set 'Hello World!'
+      # browser.button(type: 'submit').click
+      # puts browser.title
+      browser.quit
+    end
+  end
+end

data/lib/farmstead/service.rb

@@ -0,0 +1,89 @@
+# Base Class for all micro-services
+module Farmstead
+  class Service
+    def initialize
+      read_environment
+      #mysql_init
+      @kafka = Kafka.new(
+        seed_brokers: ["#{@broker_host}:9092"],
+        client_id: @service_name
+      )
+      @producer = @kafka.producer
+      @consumer = @kafka.consumer(group_id: @service_name)
+      @logfile = "/tmp/farmlog"
+    end
+
+    def read_environment
+      @broker_host = ENV['KAFKA_ADVERTISED_HOST_NAME']
+      @mysql_host = ENV['MYSQL_HOST']
+      @mysql_pass = ENV['MYSQL_PASSWORD']
+      @mysql_user = ENV['MYSQL_USER']
+      @mysql_db = ENV['MYSQL_DATABASE']
+      @service_name = ENV['SERVICE']
+      #@selenium_hub = ENV['SELENIUM_HUB']
+    end
+
+    def mysql_init
+      @mysql = Mysql2::Client.new(
+        host: @mysql_host,
+        username: @mysql_user,
+        password: @mysql_pass,
+        database: @mysql_db
+      )
+    end
+
+    # Runs on an infinite loop processing records
+    # on MySQL DB and writing messages accordingly
+    def producer
+      loop do
+        puts 'Producing'
+      end
+    end
+
+    # Subscribes to a Topic
+    # Works on the message
+    def consumer
+      loop do
+        puts 'Consuming'
+      end
+    end
+
+    # Each Farmstead site has a Class that defines it
+    # Each service runs one or more methods on that Class
+    def find_definitions(site, service)
+      mysite = @mysql.query("SELECT * FROM sites WHERE name = '#{site}'")
+      return false if mysite.count.zero?
+      mysite.each do |sited|
+        json = sited.to_json
+        config = get_from_json(json, 'config')
+        # Convert to hash where each element is a service and then associate it
+        # with the service name passed to this method
+      end
+    end
+
+    def doit
+      find_definitions('Yahoo', 'scarecrow')
+    end
+
+    def write_message(message, topic)
+      @producer.produce(message, topic)
+      @producer.deliver_messages
+    end
+
+    # Appends to existing file
+    def write_file(filename, text)
+      File.open(filename, 'a') { |file| file.write("#{text}\n") }
+    end
+
+    # Gets the value of an element from json
+    def get_from_json(json, element)
+      hash = JSON.parse(json)
+      hash[element]
+    end
+
+    def print_time
+      time1 = Time.new
+      write_file(@logfile, "Current Time : #{time1.inspect}")
+    end
+  end
+end