farmstead 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3e3c1869967a48ec7d9a3e457b8581b0dd94c84e
+  data.tar.gz: 60e3c66863f512c334ba875494cca177dfd422a8
+SHA512:
+  metadata.gz: 14685151776dbc81a792391fe98fc28a11581d9cc04d9c5c5820e2e33dcf4259efb9af322b4b097b4d972add0e39eb6c6b8c1168f7280e656aa36db7f655acf0
+  data.tar.gz: 423fee21c397e561b8d958560cf3306b5533564eee60519ef37323b8d2d0a3f025eab092d0babed4dc58daf1197bed021b03f95fdfed21ff13a42c4a4c47d9af

data/.gitignore

@@ -0,0 +1,26 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+tmp/
+log/
+.Gemfile
+.ruby-version
+.byebug_history
+debug.log
+pkg
+/.bundle
+/dist
+/doc/rdoc
+/*/doc
+/*/test/tmp
+rebuild.sh
+*.gem

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,148 @@
+AllCops:
+  TargetRubyVersion: 2.4
+  # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
+  # to ignore them, so only the ones explicitly set in this file are enabled.
+  DisabledByDefault: true
+  Exclude:
+    - '**/templates/**/*'
+    - '**/vendor/**/*'
+
+# Prefer &&/|| over and/or.
+Style/AndOr:
+  Enabled: true
+
+# Do not use braces for hash literals when they are the last argument of a
+# method call.
+Style/BracesAroundHashParameters:
+  Enabled: true
+  EnforcedStyle: context_dependent
+
+# Align `when` with `case`.
+Layout/CaseIndentation:
+  Enabled: true
+
+# Align comments with method definitions.
+Layout/CommentIndentation:
+  Enabled: true
+
+Layout/EmptyLineAfterMagicComment:
+  Enabled: true
+
+# In a regular class definition, no empty lines around the body.
+Layout/EmptyLinesAroundClassBody:
+  Enabled: true
+
+# In a regular method definition, no empty lines around the body.
+Layout/EmptyLinesAroundMethodBody:
+  Enabled: true
+
+# In a regular module definition, no empty lines around the body.
+Layout/EmptyLinesAroundModuleBody:
+  Enabled: true
+
+Layout/FirstParameterIndentation:
+  Enabled: true
+
+# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
+Style/HashSyntax:
+  Enabled: true
+
+# Method definitions after `private` or `protected` isolated calls need one
+# extra level of indentation.
+Layout/IndentationConsistency:
+  Enabled: true
+  EnforcedStyle: rails
+
+# Two spaces, no tabs (for indentation).
+Layout/IndentationWidth:
+  Enabled: true
+
+Layout/LeadingCommentSpace:
+  Enabled: true
+
+Layout/SpaceAfterColon:
+  Enabled: true
+
+Layout/SpaceAfterComma:
+  Enabled: true
+
+Layout/SpaceAroundEqualsInParameterDefault:
+  Enabled: true
+
+Layout/SpaceAroundKeyword:
+  Enabled: true
+
+Layout/SpaceAroundOperators:
+  Enabled: true
+
+Layout/SpaceBeforeComma:
+    Enabled: true
+
+Layout/SpaceBeforeFirstArg:
+    Enabled: true
+
+Style/DefWithParentheses:
+  Enabled: true
+
+# Defining a method with parameters needs parentheses.
+Style/MethodDefParentheses:
+  Enabled: true
+
+Style/FrozenStringLiteralComment:
+  Enabled: true
+  EnforcedStyle: always
+  #Exclude:
+
+# Use `foo {}` not `foo{}`.
+Layout/SpaceBeforeBlockBraces:
+  Enabled: true
+
+# Use `foo { bar }` not `foo {bar}`.
+Layout/SpaceInsideBlockBraces:
+  Enabled: true
+
+# Use `{ a: 1 }` not `{a:1}`.
+Layout/SpaceInsideHashLiteralBraces:
+  Enabled: true
+
+Layout/SpaceInsideParens:
+  Enabled: true
+
+# Check quotes usage according to lint rule below.
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+# Detect hard tabs, no hard tabs.
+Layout/Tab:
+  Enabled: true
+
+# Blank lines should not have any spaces.
+Layout/TrailingBlankLines:
+  Enabled: true
+
+# No trailing whitespace.
+Layout/TrailingWhitespace:
+  Enabled: true
+
+# Use quotes for string literals when they are enough.
+Style/UnneededPercentQ:
+  Enabled: true
+
+# Align `end` with the matching keyword or starting expression except for
+# assignments, where it should be aligned with the LHS.
+Lint/EndAlignment:
+  Enabled: true
+  EnforcedStyleAlignWith: variable
+
+# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
+Lint/RequireParentheses:
+  Enabled: true
+
+Style/RedundantReturn:
+  Enabled: true
+  AllowMultipleReturnValues: true
+
+Style/Semicolon:
+  Enabled: true
+  AllowAsExpressionSeparator: true

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.16.1

data/.yardopts

@@ -0,0 +1,2 @@
+--quiet
+/lib/**/*.rb

data/Gemfile

@@ -0,0 +1,35 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in farmstead.gemspec
+gemspec
+
+gem "coffee-rails", "~> 4.2"
+gem "httparty", "~> 0.15.6"
+gem "jbuilder", "~> 2.5"
+gem "jquery-rails", "~> 4.3.1"
+gem "mechanize", "~> 2.7.5"
+gem "mysql2", "~> 0.4.10"
+gem "nokogiri", "~> 1.8.1"
+gem "omniauth-auth0", "~> 2.0.0"
+gem "puma", "~> 3.0"
+gem "rails", "~> 5.0.0"
+gem "ruby-kafka", "~> 0.5.1"
+gem "sass-rails", "~> 5.0"
+gem "turbolinks", "~> 5"
+gem "uglifier", ">= 1.3.0"
+gem "watir", "~> 6.10"
+
+group :development, :test do
+  gem "byebug", platform: :mri
+end
+
+group :development do
+  gem "dotenv", "~> 0.11.1"
+  gem "listen", "~> 3.0.5"
+  gem "pry"
+  gem "spring"
+  gem "spring-watcher-listen", "~> 2.0.0"
+  gem "web-console"
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Ken Jenney
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,110 @@
+# farmstead
+
+[![Build Status](https://api.travis-ci.org/mastermindg/farmstead.svg?branch=master)](http://travis-ci.org/mastermindg/farmstead)
+
+Farmstead is a modular data pipeline platform. Farmstead makes creating and deploying a fully-functional data pipeline a snap. It is built on top of Rails/Ruby and uses Docker. This combination allows for a super-fast deployment and prototyping process.
+
+
+## Table of Contents
+
+<!-- TOC depthFrom:1 depthTo:6 withLinks:1 orderedList:0 -->
+
+- [Getting started](#getting-started)
+    - [Configuration](#configuration)
+    - [Environment Variables](#env)
+    - [Deployment Methods](#deployment)
+- [Architecture](#architecture)
+    - [Scheduler - Glenda](#glenda)
+    - [Fertilize - Tinman](#tinman)
+    - [Harvest - Scarecrow](#scarecrow)
+    - [Mill - CowardlyLion](#cowardlylion)
+    - [Serve - Dorothy](#dorothy)
+- [License](#license)
+
+<!-- /TOC -->
+
+
+## Getting Started
+
+To get started you'll need to install the Farmstead gem:
+
+```ruby
+gem install farmstead
+```
+
+To create a new Farmstead project:
+
+```
+farmstead new myproject
+cd myproject
+farmstead start
+```
+
+### Configuration
+
+Farmstead tries to follow Rails conventions. Some of the configuration options are available from the command-line. For instance, to chose a different database techhnology use the -d flag:
+
+```
+farmstead new myproject -d postgres
+```
+
+There are default values for database users and passwords. For more advanced usages you can use a configuration file. A configuration file must be written in YAML and is passed to Farmstead via command-line:
+
+```
+farmstead new myproject -c myproject.yml
+```
+
+An example configuration file is included.
+
+### Environment Variables
+
+Farmstead builds projects on Docker. In order to keep secrets out of the Rails code we use environment varibles. These are read from the .env file at the root of the project. This is IGNORED in Git, so be sure to keep these safe in a deployment/build system (i.e. Jenkins).
+
+Keep in mind that you can always customize the Rails application using further using more environment variables as well.
+
+### Deployment Methods
+
+The default method is Docker. It is also possible to deploy using Rancher and Kubernetes (on top of Docker). To deploy using another method use the -x flag:
+
+
+```
+farmstead new myproject -x kubernetes
+```
+
+## Architecture
+
+Kafka and Database ETL
+
+Presently only Dorothy is running Rails. The rest of the services are only running a Kafka consumer and producer. Eventually that will change.
+
+### Scheduler - Glenda
+
+Glenda
+
+### Fertilize - Tinman
+
+Tinman
+
+### Harvest - Scarecrow
+
+Scarecrow
+
+### Mill - CowardlyLion
+
+Cowardlylion
+
+### Serve - Dorothy
+
+Dorothy
+
+## License
+
+MIT
+
+
+## TODO
+
+1. Find out how to create roles for Dorothy
+2. Get Micro-services working
+3. Figure out how to get Selenium working or if it's necessary (Selenium)
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "farmstead"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+# require "irb"
+# IRB.start(__FILE__)
+
+require "farmstead"
+
+Farmstead::CLI.start(ARGV)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/myproject.yml

@@ -0,0 +1,23 @@
+- name: My Project
+  hosts: 127.0.0.1
+  connection: local
+  gather_facts: False
+  remote_user: root
+  roles:
+    - create
+
+AUTH0_CLIENT_ID={CLIENT_ID}
+AUTH0_CLIENT_SECRET={CLIENT_SECRET}
+AUTH0_DOMAIN={DOMAIN}
+AUTH0_CALLBACK_URL=http://localhost:3000/auth/auth0/callback
+AUTH0_AUDIENCE=
+MYSQL_ROOT_PASSWORD=Rc2$NE99p5%^
+MYSQL_DATABASE=farmstead
+MYSQL_USER=farmstead
+MYSQL_PASSWORD=farmstead
+MYSQL_HOST=mysql
+RAILS_ENV=development
+KAFKA_ADVERTISED_HOST_NAME=192.168.1.13
+KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+KAFKA_CREATE_TOPICS=Wood:1:1,Field:1:1,Forest:1:1,Road:1:1
+

data/farmstead.gemspec

@@ -0,0 +1,26 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "farmstead/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "farmstead"
+  spec.version       = Farmstead::VERSION
+  spec.authors       = ["Ken Jenney"]
+  spec.email         = ["mastermindg@gmail.com"]
+
+  spec.summary       = "Farmstead is a modular data pipeline platform."
+  spec.description   = "Farmstead is a modular data pipeline platform. Farmstead makes creating and deploying a fully-functional data pipeline a snap. It is built on top of Rails/Ruby and uses Docker. This combination allows for a super-fast deployment and prototyping process."
+
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end