farmstead 0.0.9 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3fc5cf26badb4c4d6936bf064c20e64560f6e5ed
-  data.tar.gz: 4e8778dafcee5bfe2c27aad8be836fc987dc2e55
+  metadata.gz: 032071d2a61bf9d506341f24564fe0a2ae808e3e
+  data.tar.gz: 406e17d3e1ed534c9160d5f0bda42e0f98234390
 SHA512:
-  metadata.gz: 182397b751e55d19c1a9a2fffb3f6f47c9f0ee84ef70807de1275005a165b277b3cec6360569afe1a95ef7a4c5787415fd7b0eb8e23bfde31d7885084bf0ab2c
-  data.tar.gz: 49ac4a5933d3ef9d7ca016d70cf1a21af3cbbffc25f7da34d2757bfe100a9460ddf487b1e0e65814d1f4d4b510ddbae85d7eb84227dc99300fc97868f0942233
+  metadata.gz: 6cdbf29f46a55486b4db5bd64f602a6d5a65df193e36a30b1ae8f4595e516d50aa3f68d20fdc9692ed404b19bc321b66c53472f17022bfabfbfb70d4e5b5a037
+  data.tar.gz: 8d6e13d3641de498b9473586334f927fc8133275e7a3cdff53950068ad52f173908042a97ab45ce490cba606fdf8928dc7bb071230030177afefece080cefa7c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    farmstead (0.0.9)
+    farmstead (0.0.11)
       thor (~> 0.20.0)
 
 GEM

data/README.md

@@ -3,7 +3,7 @@
 [![Gem Version](https://badge.fury.io/rb/farmstead.svg)](https://badge.fury.io/rb/farmstead)
 [![Build Status](https://api.travis-ci.org/mastermindg/farmstead.svg?branch=master)](http://travis-ci.org/mastermindg/farmstead)
 
-Farmstead is a modular data pipeline platform. Farmstead makes creating and deploying a fully-functional data pipeline a snap. It is built on top of Rails/Ruby and uses Docker. This combination allows for a super-fast deployment and prototyping process.
+Farmstead is a modular data pipeline platform. Farmstead makes creating and deploying a fully-functional data pipeline a snap. Farmstead uses containers to encapsulate the middleware which allows for a super-fast deployment and prototyping process.
 
 
 ## Table of Contents
@@ -38,8 +38,6 @@ To create a new Farmstead project:
 
 ```
 farmstead new myproject
-cd myproject
-farmstead start
 ```
 
 ### Configuration
@@ -60,17 +58,9 @@ An example configuration file is included.
 
 #### Configuration Options
 
-**Rails Authentication**
-
-There is optional Rails Authentication with AUTH0.
-
-**Rails Environment**
-
-The default environment is development but can be set to production (if you're ready).
-
 **Database**
 
-The default database is MySQL but can be set to anything that Rails can handle. 
+The default database is MySQL but can be set to either MySQL, Postgres, or SQLLite. Extensions will be available 
 
 **Kafka**
 
@@ -103,29 +93,33 @@ farmstead new myproject -x kubernetes
 
 ## Architecture
 
-Kafka and Database ETL
+Kafka and Database 
+
+ETL
 
-Presently only Dorothy is running Rails. The rest of the services are only running a Kafka consumer and producer. Eventually that will change.
+* Extract
+* Transform
+* Load
 
-### Scheduler - Glenda
+All of the services are only running a Kafka consumer and producer. There is a Manager service that 
 
-Glenda
+**Classes:**
 
-### Fertilize - Tinman
+### Farmstead::Manager
 
-Tinman
+Task scheduling, batch processing, and general flow control. Exposes a very simple web service where you can pull logs and see the data in real-time.
 
-### Harvest - Scarecrow
+### Farmstead::Extract
 
-Scarecrow
+Extracts the data from the source.
 
-### Mill - CowardlyLion
+### Farmstead::Transform
 
-Cowardlylion
+Transforms one or more datasets.
 
-### Serve - Dorothy
+### Farmstead::Load
 
-Dorothy
+Loads the data into a database.
 
 ## License
 
@@ -134,7 +128,6 @@ MIT
 
 ## TODO
 
-1. Find out how to create roles for Dorothy
+1. Database extensions
 2. Get Micro-services working
-3. Figure out how to get Selenium working or if it's necessary (Selenium)
 

data/examples/myproject/extract/extracter.rb

@@ -0,0 +1,7 @@
+module MyProject
+  class Extracter < Farmstead::Extract
+    def hello
+      p "hello"
+    end
+  end
+end

data/examples/myproject/load/loader.rb

@@ -0,0 +1,7 @@
+module MyProject
+  class Loader < Farmstead::Load
+    def hello
+      p "hello"
+    end
+  end
+end

data/examples/myproject/myproject.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# Primary script to kick off services in project
+# Pass two arguments to this script
+# 1st - the class you want to call
+# 2nd - the method you want to call
+
+dynamic_class = ARGV[0]
+dynamic_method = ARGV[1]
+
+# Extend Farmstead
+require "farmstead"
+
+# Load project classes
+require_relative "extract/extracter"
+require_relative "load/loader"
+require_relative "transform/transformer"
+
+module MyProject
+  include Farmstead
+end
+
+klass = Object.const_get "MyProject::#{dynamic_class}"
+service = klass.new
+service.send(dynamic_method)

data/examples/myproject/myproject.yml

@@ -0,0 +1,17 @@
+- name: My Project
+  database:
+    - type: mysql
+    - MYSQL_ROOT_PASSWORD: Rc2$NE99p5%^
+    - MYSQL_DATABASE: farmstead
+    - MYSQL_USER: farmstead
+    - MYSQL_PASSWORD: farmstead
+    - MYSQL_HOST: mysql
+  kafka:
+    - advertise_from_local_ip: false
+    - advertised_ip: 192.168.1.2
+    - zookeeper_address: "zookeeper:2181"
+    - topics
+      - "Manage:1:1"
+      - "Extract:1:1"
+      - "Load:1:1"
+      - "Transform:1:1"

data/examples/myproject/transform/transformer.rb

@@ -0,0 +1,7 @@
+module MyProject
+  class Transformer < Farmstead::Transform
+    def hello
+      p "hello"
+    end
+  end
+end

data/farmstead.gemspec

@@ -20,7 +20,6 @@ Gem::Specification.new do |spec|
     f.match(%r{^(test|spec|features)/})
   end
 
-  #spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.executables   = ['farmstead']
   spec.require_paths = ["lib"]
 

data/lib/farmstead/cli.rb

@@ -3,10 +3,16 @@ require "thor"
 
 module Farmstead
   class CLI < Thor
-    class_option :verbose, aliases: "-v", type: "boolean", desc: "Be verbose"
+    class_option :verbose, aliases: "--v", type: "boolean", desc: "Be verbose"
     class_option :config, aliases: "-c", type: "string", desc: "Config file"
     class_option :database, aliases: "-d", type: "string", desc: "Database"
     class_option :deploy, aliases: "-x", type: "string", desc: "Deployment Method"
+
+    desc "version", "Get the gem version"
+    def version
+      puts "Farmstead #{Farmstead::VERSION}"
+    end
+
     desc "new project_name", "Create a new project"
     def new(project_name)
       project = Farmstead::Project.new
@@ -53,5 +59,7 @@ module Farmstead
 
     # desc "net COMMANDS", "Net control Module"
     # subcommand "net", Socialinvestigator::CLI::Net
+
+    map "-v" => "version"
   end
 end

data/lib/farmstead/{glenda.rb → manager.rb}

@@ -1,5 +1,4 @@
-# Glenda - the Scheduler
-#
+# Manager
 #
 # It works off of the DB when
 # 1) A new site is added
@@ -27,7 +26,7 @@
 # HINT: See .env
 # Every micro-service inherits the Service class
 module Farmstead
-  class Glenda < Service
+  class Manager < Service
     # Runs on an infinite loop processing records
     # on MySQL DB and writing messages accordingly
     def producer
@@ -97,3 +96,4 @@ module Farmstead
     end
   end
 end
+

data/lib/farmstead/pipeline/extract.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# Extract data from the source
+#
+# Runs a Consumer and it will automatically pick up
+# messages from the Field Topic and do it's job
+# and then send a message as a Producer to the Forest Topic
+#
+# Every micro-service inherits the Service class
+module Farmstead
+  class Extract < Service
+    # Picks up JSON generated by WebDriver and save it to Forest topic
+    def producer
+      loop do
+        puts "Do something"
+        sleep 300
+      end
+    end
+
+    # Subscribed to the Field topic
+    # Works on message
+    def consumer
+      @consumer.subscribe('Field')
+      trap('TERM') { @consumer.stop }
+      @consumer.each_message do |message|
+        puts "Received: #{message.value}"
+        magic_work(message.value)
+        @consumer.mark_message_as_processed(message)
+      end
+    end
+
+    def magic_work(site)
+      hash = JSON.parse(site)
+      hash['scarecrow'] = 'true'
+      json = hash.to_json
+      puts "Writing: #{json}"
+      write_message(json, topic: 'Forest')
+    end
+  end
+end

data/lib/farmstead/{scarecrow.rb → pipeline/load.rb}

@@ -1,18 +1,18 @@
-# Scarecrow - the Harvester
-#
-# Scarecrow is responsible for extracting data from a source
+# frozen_string_literal: true
+
+# Load data into database
 #
-# Scarecrow is running as a Consumer and it will automatically pick up
+# Runs a Consumer and it will automatically pick up
 # messages from the Field Topic and do it's job
 # and then send a message as a Producer to the Forest Topic
 #
 # Every micro-service inherits the Service class
 module Farmstead
-  class Scarecrow < Service
+  class Load < Service
     # Picks up JSON generated by WebDriver and save it to Forest topic
     def producer
       loop do
-        puts 'Do something'
+        puts "Do something"
         sleep 300
       end
     end
@@ -36,20 +36,6 @@ module Farmstead
       puts "Writing: #{json}"
       write_message(json, topic: 'Forest')
     end
-
-    # Call the Site Class
-    def call_class
-      puts 'this'
-    end
-
-    def selenium
-      browser = Watir::Browser.new :chrome
-      browser.goto 'http://www.stackoverflow.com'
-      puts browser.title
-      # browser.text_field(title: 'Search').set 'Hello World!'
-      # browser.button(type: 'submit').click
-      # puts browser.title
-      browser.quit
-    end
   end
 end
+

data/lib/farmstead/{cowardlylion.rb → pipeline/transform.rb}

@@ -1,15 +1,17 @@
-# CowardlyLion - the Miller
+# frozen_string_literal: true
+
+# Transform data
 #
-# Cowardlylion is responsible for arranging data into usable blocks
+# Arrange data into usable blocks
 #
 #
-# CowardlyLion is running as a Consumer and it will automatically pick up
+# Running as a Consumer and it will automatically pick up
 # messages from the Forest topic and do it's job and then send
 # a message as a Producer to the Road topic
 #
 # Every micro-service inherits the Service class
 module Farmstead
-  class Cowardlylion #< Service
+  class Transform < Service
     # Does nothing...work is handled by magic_work
     def producer
       loop do

data/lib/farmstead/scaffold/extract/extracter.rb.erb

@@ -0,0 +1,7 @@
+module MyProject
+  class Extracter < Farmstead::Extract
+    def hello
+      p "hello"
+    end
+  end
+end

data/lib/farmstead/scaffold/load/loader.rb

@@ -0,0 +1,7 @@
+module MyProject
+  class Loader < Farmstead::Load
+    def hello
+      p "hello"
+    end
+  end
+end

data/lib/farmstead/scaffold/project.rb.erb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# Primary script to kick off services in project
+# Pass two arguments to this script
+# 1st - the class you want to call
+# 2nd - the method you want to call
+
+dynamic_class = ARGV[0]
+dynamic_method = ARGV[1]
+
+# Extend Farmstead
+require "farmstead"
+
+# Load project classes
+require_relative "extract/extracter"
+require_relative "load/loader"
+require_relative "transform/transformer"
+
+module MyProject
+  include Farmstead
+end
+
+klass = Object.const_get "MyProject::#{dynamic_class}"
+service = klass.new
+service.send(dynamic_method)

data/lib/farmstead/scaffold/project.yml.erb

@@ -0,0 +1,17 @@
+- name: My Project
+  database:
+    - type: mysql
+    - MYSQL_ROOT_PASSWORD: Rc2$NE99p5%^
+    - MYSQL_DATABASE: farmstead
+    - MYSQL_USER: farmstead
+    - MYSQL_PASSWORD: farmstead
+    - MYSQL_HOST: mysql
+  kafka:
+    - advertise_from_local_ip: false
+    - advertised_ip: 192.168.1.2
+    - zookeeper_address: "zookeeper:2181"
+    - topics
+      - "Manage:1:1"
+      - "Extract:1:1"
+      - "Load:1:1"
+      - "Transform:1:1"

data/lib/farmstead/scaffold/transform/transformer.rb

@@ -0,0 +1,7 @@
+module MyProject
+  class Transformer < Farmstead::Transform
+    def hello
+      p "hello"
+    end
+  end
+end

data/lib/farmstead/version.rb

@@ -1,3 +1,3 @@
 module Farmstead
-  VERSION = "0.0.9"
+  VERSION = "0.0.11"
 end

data/lib/farmstead.rb

@@ -35,11 +35,10 @@ require "farmstead/version"
 require "farmstead/project"
 require "farmstead/cli"
 require "farmstead/service"
-require "farmstead/tinman"
-require "farmstead/cowardlylion"
-require "farmstead/glenda"
-require "farmstead/scarecrow"
-require "farmstead/tinman"
+require "farmstead/manager"
+require "farmstead/pipeline/extract"
+require "farmstead/pipeline/transform"
+require "farmstead/pipeline/load"
 
 module Farmstead
   # Your code goes here...

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: farmstead
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.11
 platform: ruby
 authors:
 - Ken Jenney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-31 00:00:00.000000000 Z
+date: 2018-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -87,28 +87,33 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/console
 - bin/farmstead
-- bin/setup
-- examples/myproject.yml
+- examples/myproject/extract/extracter.rb
+- examples/myproject/load/loader.rb
+- examples/myproject/myproject.rb
+- examples/myproject/myproject.yml
+- examples/myproject/transform/transformer.rb
 - farmstead.gemspec
 - lib/farmstead.rb
 - lib/farmstead/cli.rb
-- lib/farmstead/cowardlylion.rb
-- lib/farmstead/glenda.rb
+- lib/farmstead/manager.rb
+- lib/farmstead/pipeline/extract.rb
+- lib/farmstead/pipeline/load.rb
+- lib/farmstead/pipeline/transform.rb
 - lib/farmstead/project.rb
 - lib/farmstead/scaffold/.dockerignore.erb
 - lib/farmstead/scaffold/.env.erb
-- lib/farmstead/scaffold/Dockerfile-dorothy.erb
 - lib/farmstead/scaffold/Dockerfile.erb
 - lib/farmstead/scaffold/Gemfile.erb
-- lib/farmstead/scaffold/app/controllers/application_controller.rb.erb
 - lib/farmstead/scaffold/docker-compose.yml.erb
 - lib/farmstead/scaffold/exec.sh.erb
+- lib/farmstead/scaffold/extract/extracter.rb.erb
+- lib/farmstead/scaffold/load/loader.rb
+- lib/farmstead/scaffold/project.rb.erb
+- lib/farmstead/scaffold/project.yml.erb
 - lib/farmstead/scaffold/supervisord.conf.erb
-- lib/farmstead/scarecrow.rb
+- lib/farmstead/scaffold/transform/transformer.rb
 - lib/farmstead/service.rb
-- lib/farmstead/tinman.rb
 - lib/farmstead/version.rb
 homepage: https://github.com/mastermindg/farmstead
 licenses:

data/bin/console

@@ -1,18 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "farmstead"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-# require "irb"
-# IRB.start(__FILE__)
-
-require "farmstead"
-
-Farmstead::CLI.start(ARGV)

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here

data/examples/myproject.yml

@@ -1,26 +0,0 @@
-- name: My Project
-  rails:
-    - authentication:
-      - enable: true
-      - AUTH0_CLIENT_ID: myclientid
-      - AUTH0_CLIENT_SECRET: myclientsecret
-      - AUTH0_DOMAIN: myauth0domain
-      - AUTH0_DOMAIN: http://localhost:3000/auth/auth0/callback
-      - AUTH0_AUDIENCE: 
-    - environment: development
-  database:
-    - type: mysql
-    - MYSQL_ROOT_PASSWORD: Rc2$NE99p5%^
-    - MYSQL_DATABASE: farmstead
-    - MYSQL_USER: farmstead
-    - MYSQL_PASSWORD: farmstead
-    - MYSQL_HOST: mysql
-  kafka:
-    - advertise_from_local_ip: false
-    - advertised_ip: 192.168.1.2
-    - zookeeper_address: "zookeeper:2181"
-    - topics
-      - "Wood:1:1"
-      - "Field:1:1"
-      - "Forest:1:1"
-      - "Road:1:1"

data/lib/farmstead/scaffold/Dockerfile-dorothy.erb

@@ -1,24 +0,0 @@
-FROM ruby:2.4.1
-
-RUN apt-get update -qq && \
-    apt-get install -y build-essential \
-    libpq-dev \
-    nodejs \
-    telnet \
-    bash \
-    supervisor \
-    vim \
-    xterm \
-    ca-certificates \
-    unzip && \
-    rm -rf /var/cache/apk/*
-
-## Setup Rails
-
-RUN gem install rails && rails new <%= @name %> <% if @database %>-d <%= @database %><% end %>
-WORKDIR /<%= @name %>
-ADD app/* /<%= @name %>/app
-
-EXPOSE 3000
-
-CMD /<%= @name %>/bin/rails s -b 0.0.0.0

data/lib/farmstead/scaffold/app/controllers/application_controller.rb.erb

@@ -1,6 +0,0 @@
-class ApplicationController < ActionController::Base
-  # Prevent CSRF attacks by raising an exception.
-  # For APIs, you may want to use :null_session instead.
-  protect_from_forgery with: :exception
-end
-

data/lib/farmstead/tinman.rb

@@ -1,49 +0,0 @@
-# Tinman - the Fertilizer
-#
-# Tinman is responsible for analyzing a site for changes
-# and updating site configs
-#
-# None of the logic has actually been built and
-# this may require some AI but for now it's
-# just responding that everything is OK
-#
-# Tinman is running as a Consumer to the Wood topic and it will automatically
-# pick up the message, do it's job and then send a message
-# as a Producer to the Field topic
-#
-# Every micro-service inherits the Service class
-module Farmstead
-  class Tinman < Service
-    # Doing nothing...for now
-    # thi is handled by magic_work
-    def producer
-      loop do
-        puts 'Doing nothing'
-        sleep 300
-      end
-    end
-
-    # Subscribed to the Wood topic
-    # Works on message
-    def consumer
-      @consumer.subscribe('Wood')
-      trap('TERM') { @consumer.stop }
-      @consumer.each_message do |message|
-        puts "Received: #{message.value}"
-        magic_work(message.value)
-        @consumer.mark_message_as_processed(message)
-      end
-    end
-
-    # Do the magic work to determine if the site has changed
-    # and update the config
-    # Returns JSON String
-    def magic_work(site)
-      hash = JSON.parse(site)
-      hash['tinman'] = 'true'
-      json = hash.to_json
-      puts "Writing: #{json}"
-      write_message(json, topic: 'Field')
-    end
-  end
-end