farmstead 0.0.2

data/lib/farmstead/tinman.rb

@@ -0,0 +1,49 @@
+# Tinman - the Fertilizer
+#
+# Tinman is responsible for analyzing a site for changes
+# and updating site configs
+#
+# None of the logic has actually been built and
+# this may require some AI but for now it's
+# just responding that everything is OK
+#
+# Tinman is running as a Consumer to the Wood topic and it will automatically
+# pick up the message, do it's job and then send a message
+# as a Producer to the Field topic
+#
+# Every micro-service inherits the Service class
+module Farmstead
+  class Tinman < Service
+    # Doing nothing...for now
+    # thi is handled by magic_work
+    def producer
+      loop do
+        puts 'Doing nothing'
+        sleep 300
+      end
+    end
+
+    # Subscribed to the Wood topic
+    # Works on message
+    def consumer
+      @consumer.subscribe('Wood')
+      trap('TERM') { @consumer.stop }
+      @consumer.each_message do |message|
+        puts "Received: #{message.value}"
+        magic_work(message.value)
+        @consumer.mark_message_as_processed(message)
+      end
+    end
+
+    # Do the magic work to determine if the site has changed
+    # and update the config
+    # Returns JSON String
+    def magic_work(site)
+      hash = JSON.parse(site)
+      hash['tinman'] = 'true'
+      json = hash.to_json
+      puts "Writing: #{json}"
+      write_message(json, topic: 'Field')
+    end
+  end
+end

data/lib/farmstead/version.rb

@@ -0,0 +1,3 @@
+module Farmstead
+  VERSION = "0.0.2"
+end

data/scaffold/.dockerignore.erb

@@ -0,0 +1,6 @@
+vendor/
+.env
+README.md
+Dockerfile
+docker-compose.yml
+kafka

data/scaffold/.env.erb

@@ -0,0 +1,9 @@
+MYSQL_ROOT_PASSWORD=Rc2$NE99p5%^
+MYSQL_DATABASE=farmstead
+MYSQL_USER=farmstead
+MYSQL_PASSWORD=farmstead
+MYSQL_HOST=mysql
+RAILS_ENV=development
+KAFKA_ADVERTISED_HOST_NAME=192.168.1.139
+KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+KAFKA_CREATE_TOPICS=Wood:1:1,Field:1:1,Forest:1:1,Road:1:1

data/scaffold/Dockerfile-dorothy.erb

@@ -0,0 +1,24 @@
+FROM ruby:2.4.1
+
+RUN apt-get update -qq && \
+    apt-get install -y build-essential \
+    libpq-dev \
+    nodejs \
+    telnet \
+    bash \
+    supervisor \
+    vim \
+    xterm \
+    ca-certificates \
+    unzip && \
+    rm -rf /var/cache/apk/*
+
+## Setup Rails
+
+RUN gem install rails && rails new <%= @name %> <% if @database %>-d <%= @database %><% end %>
+WORKDIR /<%= @name %>
+ADD app/* /<%= @name %>/app
+
+EXPOSE 3000
+
+CMD /<%= @name %>/bin/rails s -b 0.0.0.0

data/scaffold/Dockerfile.erb

@@ -0,0 +1,25 @@
+FROM ruby:2.4.1
+
+RUN apt-get update -qq && \
+    apt-get install -y build-essential \
+    libpq-dev \
+    nodejs \
+    telnet \
+    bash \
+    supervisor \
+    vim \
+    xterm \
+    ca-certificates \
+    unzip && \
+    rm -rf /var/cache/apk/*
+
+## Setup Ruby
+RUN mkdir /myapp
+WORKDIR /myapp
+ADD Gemfile* /myapp/
+RUN gem install bundler && bundle install --jobs 20 --retry 5
+ADD . /myapp
+
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+CMD /usr/bin/supervisord

data/scaffold/Gemfile.erb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+gemspec
+
+gem "coffee-rails", "~> 4.2"
+gem "httparty", "~> 0.15.6"
+gem "jbuilder", "~> 2.5"
+gem "jquery-rails", "~> 4.3.1"
+gem "mechanize", "~> 2.7.5"
+gem "mysql2", "~> 0.4.10"
+gem "nokogiri", "~> 1.8.1"
+gem "omniauth-auth0", "~> 2.0.0"
+gem "puma", "~> 3.0"
+gem "rails", "~> 5.0.0"
+gem "ruby-kafka", "~> 0.5.1"
+gem "sass-rails", "~> 5.0"
+gem "turbolinks", "~> 5"
+gem "uglifier", ">= 1.3.0"
+gem "watir", "~> 6.10"
+
+group :development, :test do
+  gem "byebug", platform: :mri
+end
+
+group :development do
+  gem "dotenv", "~> 0.11.1"
+  gem "listen", "~> 3.0.5"
+  gem "pry"
+  gem "spring"
+  gem "spring-watcher-listen", "~> 2.0.0"
+  gem "web-console"
+end

data/scaffold/app/controllers/application_controller.rb.erb

@@ -0,0 +1,6 @@
+class ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+end
+

data/scaffold/docker-compose.yml.erb

@@ -0,0 +1,72 @@
+version: '3'
+
+services:
+  mysql:
+    image: 'mysql:5.7'
+    env_file:
+      - '.env'
+    ports:
+      - '3306:3306'
+
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - '2181:2181'
+
+  kafka:
+    depends_on:
+      - 'zookeeper'
+    image: wurstmeister/kafka:0.10.2.1
+    ports:
+      - '9092:9092'
+    env_file:
+      - '.env'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  dorothy:
+    depends_on:
+      - 'mysql'
+    build:
+      context: .
+      dockerfile: Dockerfile-dorothy
+    ports:
+      - '3000:3000'
+    env_file:
+      - '.env'
+
+  glenda:
+    depends_on:
+      - 'dorothy'
+    build: .
+    environment:
+      SERVICE: glenda
+    env_file:
+      - '.env'
+  
+  tinman:
+    depends_on:
+      - 'dorothy'
+    build: .
+    environment:
+      SERVICE: tinman
+    env_file:
+      - '.env'
+
+  scarecrow:
+    depends_on:
+      - 'dorothy'
+    build: .
+    environment:
+      SERVICE: scarecrow
+    env_file:
+      - '.env'
+
+  cowardlylion:
+    depends_on:
+      - 'dorothy'
+    build: .
+    environment:
+      SERVICE: cowardlylion
+    env_file:
+      - '.env'

data/scaffold/exec.sh.erb

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Script to start Farmstead on Docker
+
+echo "Cleaning up old environment"
+docker-compose stop > /dev/null 2>&1
+docker-compose rm -f > /dev/null 2>&1
+
+echo "Building new environment"
+docker-compose build > /dev/null 2>&1
+
+echo "Starting MySQL and Zookeeper"
+docker-compose up -d mysql zookeeper > /dev/null 2>&1
+
+echo "Sleeping"
+sleep 10
+
+echo "Starting dorothy"
+docker-compose up -d dorothy > /dev/null 2>&1
+
+echo "Create and Seed MySQL DB"
+docker-compose exec dorothy rails db:setup > /dev/null 2>&1
+
+echo "Starting everything else"
+docker-compose up -d > /dev/null 2>&1

data/scaffold/exec.sh.erb.old

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Script to start Farmstead on Docker
+
+trap mytrap ERR #0 #EXIT 0 HUP INT QUIT PIPE TERM
+
+mytrap() {
+	excode=$?
+	if [ $excode -ne 0 ]; then
+		printf "\tFailed!\n"
+    	exit 1
+    fi
+}
+
+success() {
+	printf "\tSucceeded\n"
+}
+
+# Counts down a number of seconds
+countdown() {
+	secs=$1
+	while [ $secs -gt 0 ]; do
+  	echo -ne "$secs\033[0K\r"
+   	sleep 1
+   	: $((secs--))
+	done
+}
+
+echo "Cleaning up old environment"
+docker-compose stop > /dev/null 2>&1
+docker-compose rm -f > /dev/null 2>&1
+#docker stop $(docker ps -a -q) > /dev/null 2>&1
+#docker rm $(docker ps -a -q) > /dev/null 2>&1
+success
+
+echo "Building new environment"
+docker-compose build > /dev/null 2>&1
+success
+
+echo "Starting MySQL and Zookeeper"
+docker-compose up -d mysql zookeeper > /dev/null 2>&1
+success
+
+echo "Sleeping"
+countdown 10
+
+echo "Starting dorothy"
+docker-compose up -d dorothy > /dev/null 2>&1
+success
+
+echo "Create and Seed MySQL DB"
+docker-compose exec dorothy rails db:setup > /dev/null 2>&1
+success
+
+echo "Starting everything else"
+docker-compose up -d > /dev/null 2>&1
+success

data/scaffold/supervisord.conf.erb

@@ -0,0 +1,10 @@
+[supervisord]
+nodaemon=true
+
+[program:producer]
+command=/usr/local/bin/ruby /myapp/app/lib/farmstead.rb %(ENV_SERVICE)s producer
+redirect_stderr=true
+
+[program:consumer]
+command=/usr/local/bin/ruby /myapp/app/lib/farmstead.rb %(ENV_SERVICE)s consumer
+redirect_stderr=true

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: farmstead
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Ken Jenney
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-12-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Farmstead is a modular data pipeline platform. Farmstead makes creating
+  and deploying a fully-functional data pipeline a snap. It is built on top of Rails/Ruby
+  and uses Docker. This combination allows for a super-fast deployment and prototyping
+  process.
+email:
+- mastermindg@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- ".yardopts"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- examples/myproject.yml
+- farmstead.gemspec
+- lib/farmstead.rb
+- lib/farmstead/cli.rb
+- lib/farmstead/cowardlylion.rb
+- lib/farmstead/glenda.rb
+- lib/farmstead/project.rb
+- lib/farmstead/scarecrow.rb
+- lib/farmstead/service.rb
+- lib/farmstead/tinman.rb
+- lib/farmstead/version.rb
+- scaffold/.dockerignore.erb
+- scaffold/.env.erb
+- scaffold/Dockerfile-dorothy.erb
+- scaffold/Dockerfile.erb
+- scaffold/Gemfile.erb
+- scaffold/app/controllers/application_controller.rb.erb
+- scaffold/docker-compose.yml.erb
+- scaffold/exec.sh.erb
+- scaffold/exec.sh.erb.old
+- scaffold/supervisord.conf.erb
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Farmstead is a modular data pipeline platform.
+test_files: []