apollon 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed479ad53dbcbc3b7a91c23301481b4439fc8252
-  data.tar.gz: 91d003050980e277c730f19d68aab330342c316e
+  metadata.gz: bade97999dd2e42347ce97c159457533b9a19042
+  data.tar.gz: c02d3789acdfca7e208245d1a29acc52f444fcb8
 SHA512:
-  metadata.gz: 23129cd00f7f2f8310e8dbaa0186f839536913da864f52b5bdce3b2c1189bf4bef5e05d9c10fdc282d57b84b47db6363e7969771577269b26b5c61dfc4e2af51
-  data.tar.gz: cb7bc15eb700ffcc1feb0c68065757abf8b0997459209a9e1229bf1c95c56af7a3d08072de71cbf5706931655cadeca57feb653d5ad821f51a7320756f288fe0
+  metadata.gz: 7452280fd78247841fc5424570c538127ab6ea58274949d8f811e644aca3d59d21e137045a156a3fba4475f2ece72bf68f8a4ad78697c22a9cf87d29789d9df5
+  data.tar.gz: a82c64b792a4d759a6f1d22c62d2cfd291016ffc39fc9e7c6abb84f278f1bffccd4dca9ea86f8548c501ae41c4143a99bd1562588bfa52894aa3c4e0f93180c5

data/.gemspec

@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+require 'rubygems'
+
+$:.push File.expand_path("../lib", __FILE__)
+
+require 'apollon/version.rb'
+
+Gem::Specification.new do |s|
+  s.name = 'apollon'
+  s.version = Apollon::VERSION
+  # s.platform = Gem::Platform::RUBY
+  s.summary = 'Apollo Fat Box'
+  s.description = 'CLI for Apollo Fat Box'
+  s.authors = [
+    'Tomas Korcak'
+  ]
+  s.email = 'korczis@gmail.com'
+
+  s.homepage = 'https://github.com/korczis/apollon'
+  s.license = 'MIT'
+  s.require_paths = ['lib']
+
+  s.files = `git ls-files`.split("\n")
+  s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+
+  s.add_dependency 'aws-sdk-core', '~> 2.0.0.rc7'
+  s.add_dependency 'digitalocean', '~> 1.1', '>= 1.1.0'
+  s.add_dependency 'json_pure', '~> 1.8', '>= 1.8.1'
+  s.add_dependency 'librarian-puppet', '~> 1.1', '>= 1.1.2'
+  s.add_dependency 'multi_json', '~> 1.10', '>= 1.10.1'
+  s.add_dependency 'prettyprint', '~> 0.0', '>= 0.0.1'
+  s.add_dependency 'puppet', '~> 3.6', '>= 3.6.2'
+  s.add_dependency 'rake', '~> 10.3', '>= 10.3.2'
+end

data/.gitignore

@@ -0,0 +1,5 @@
+*.box
+.idea/
+.vagrant/
+packer_cache/
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in gooddata.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,87 @@
+PATH
+  remote: .
+  specs:
+    apollon (0.0.2)
+      aws-sdk-core (~> 2.0.0.rc7)
+      digitalocean (~> 1.1, >= 1.1.0)
+      json_pure (~> 1.8, >= 1.8.1)
+      librarian-puppet (~> 1.1, >= 1.1.2)
+      multi_json (~> 1.10, >= 1.10.1)
+      prettyprint (~> 0.0, >= 0.0.1)
+      puppet (~> 3.6, >= 3.6.2)
+      rake (~> 10.3, >= 10.3.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    CFPropertyList (2.2.8)
+    activemodel (4.1.2)
+      activesupport (= 4.1.2)
+      builder (~> 3.1)
+    activesupport (4.1.2)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    aws-sdk-core (2.0.0.rc9)
+      builder (~> 3.0)
+      jamespath
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+    builder (3.2.2)
+    digitalocean (1.1.0)
+      faraday (~> 0.8.9)
+      faraday_middleware (~> 0.9.0)
+      recursive-open-struct (~> 0.4.5)
+    facter (2.1.0)
+      CFPropertyList (~> 2.2.6)
+    faraday (0.8.9)
+      multipart-post (~> 1.2.0)
+    faraday_middleware (0.9.1)
+      faraday (>= 0.7.4, < 0.10)
+    her (0.7.2)
+      activemodel (>= 3.0.0, < 4.2)
+      activesupport (>= 3.0.0, < 4.2)
+      faraday (>= 0.8, < 1.0)
+      multi_json (~> 1.7)
+    hiera (1.3.4)
+      json_pure
+    highline (1.6.21)
+    i18n (0.6.9)
+    jamespath (0.5.0)
+    json (1.8.1)
+    json_pure (1.8.1)
+    librarian (0.1.2)
+      highline
+      thor (~> 0.15)
+    librarian-puppet (1.1.2)
+      json
+      librarian (>= 0.1.2)
+      puppet_forge
+    minitest (5.3.5)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    multipart-post (1.2.0)
+    prettyprint (0.0.1)
+      thor
+    puppet (3.6.2)
+      facter (> 1.6, < 3)
+      hiera (~> 1.0)
+      json_pure
+      rgen (~> 0.6.5)
+    puppet_forge (1.0.3)
+      her (~> 0.6)
+    rake (10.3.2)
+    recursive-open-struct (0.4.5)
+    rgen (0.6.6)
+    thor (0.19.1)
+    thread_safe (0.3.4)
+    tzinfo (1.2.1)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  apollon!

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Tomas Korcak
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,100 @@
+# apollo-fat-box
+
+Vagrant box containing lot of tools and libs for data science
+
+## Status
+
+[![Code Climate](https://codeclimate.com/github/korczis/apollo-fat-box.png)](https://codeclimate.com/github/korczis/apollo-fat-box)
+
+## About
+
+What is this heavy-weight box for? For super-quick prototyping. 
+
+Yes. You know what.
+
+1. You got an idea. You want to immediately start working on it. 
+2. You realize you need to setup development environment with all the tools and stuff.
+
+That is what this fat boy is here for.
+It can be made up and running with [few commands](https://github.com/korczis/apollo-fat-box#getting-started).
+
+## What is included
+
+Lot of useful stuff, for full list see [WHATS_IN.md](https://github.com/korczis/apollo-fat-box/blob/master/WHATS_IN.md)
+
+- Databases - couchdb, memcached, mongodb, postgresql, redis, ...
+- Programming Languages - c, c++, clojure, node.js, ruby/rvm, scala, ...
+- Tools - mocha, ...
+
+Again, for full list see [WHATS_IN.md](https://github.com/korczis/apollo-fat-box/blob/master/WHATS_IN.md)
+
+## Automagically forwarded services
+
+This lists services which are automatically forwarded from running fat box to localhost.
+
+That means that you can access them using [http://localhost:{HostPortNumer}](http://localhost:{HostPortNumer}) from table below.
+
+| Service       | Guest  | Host                            | Username | Password |
+| --------------|:------:|:-------------------------------:|:--------:|:--------:|
+| SSH           | 22     | [2222](ssh://localhost:2222)    | vagrant  | vagrant  |
+| Nginx         | 80     | [81](http://localhost:81)       | n/a      | n/a      |
+| Zookeeper     | 2181   | 2182                            | n/a      | n/a      |
+| PostgreSQL    | 5432   | 5433                            | vagrant  | vagrant  |
+| CouchDB       | 5984   | [5985](http://localhost:5985)   | n/a      | n/a      |
+| Redis         | 6379   | 6380                            | n/a      | n/a      |
+| Tomcat7       | 8080   | [8081](http://localhost:8081)   | vagrant  | vagrant  |
+| ElasticSearch | 9200   | [9201](http://localhost:9201)   | n/a      | n/a      |
+| Memcached     | 11211  | 11212                           | n/a      | n/a      |
+| RabbitMQ      | 15672  | [15673](http://localhost:15673) | guest    | guest    |
+| MongoDB       | 27017  | [27018](http://localhost:27018) | n/a      | n/a      |
+| MongoDB Rest  | 28017  | [28018](http://localhost:28018) | n/a      | n/a      |
+
+## Getting Started
+
+### Prerequisites
+
+- [Ruby](https://www.ruby-lang.org/en/)
+- [Ruby gems](https://rubygems.org/pages/download)
+- [Bundler](http://bundler.io/)
+- [Vagrant](https://www.vagrantup.com/downloads.html)
+- [Virtualbox](https://www.virtualbox.org/wiki/Downloads)
+
+### Installation
+
+```
+# Clone this repo containging Vagrantfile with vagrant settings
+git clone https://github.com/korczis/apollo-fat-box.git
+
+# Enter the folder with Vagrantfile, be aware of trailing box folder in path!
+cd apollo-fat-box
+
+# Kick off the vagrant up process
+vagrant up
+```
+
+Example output of succesful launch can be found [here](https://gist.github.com/korczis/4789d566361b095f2e73).
+
+## Logging in
+
+For loggining-in use vagrant ssh.
+
+```
+vagrant ssh
+```
+
+## Destroying instance 
+
+Destroy instance using vagrant destroy.
+
+```
+vagrant destroy
+```
+
+## TODO
+
+For list of TODO stuff see [TODO.md](https://github.com/korczis/apollo-fat-box/blob/master/TODO.md)
+
+## Credits
+
+- [@pavelbinar](https://github.com/pavelbinar) for QA and not just that.
+- [@thnkr](https://github.com/thnkr/) for ideas, encouragement and other.

data/Rakefile

@@ -0,0 +1,20 @@
+desc 'Get all tasks'
+task :tasklist do
+	puts Rake.application.tasks
+end
+
+task :usage do
+  puts "No rake task specified so listing them ..."
+  Rake.application['tasklist'].invoke
+end
+
+desc 'Prepare required vagrant dependencies on local machine'
+task :prepare do
+  exec 'vagrant plugin install vagrant-digitalocean'
+end
+
+# task :default => [:usage]
+
+if $0 == __FILE__
+	Rake.application['usage'].invoke
+end

data/TODO.md

@@ -0,0 +1,22 @@
+# To Do
+
+List of stuff which needs to be be done.
+
+## High Priority 
+
+- Install Apache Tika
+- Install Apache Hadoop + Pig/Hive
+- Install Apache HBase
+- Install Apache Storm as Service (Apache Storm already installed in /opt/apache-storm-0.9.1)
+- Install Kafka
+- Install TitanDB
+- Optionally Install Emscripten
+
+## Medium Priority
+
+- Change password for all services to vagrant/vagrant
+
+## Low Priority
+
+- Better message of the day with some stats
+

data/Vagrantfile

@@ -0,0 +1,92 @@
+# encoding: utf-8
+
+require 'pp'
+require 'multi_json'
+
+require_relative 'lib/lib'
+
+# Make sure that the box we need is locally available
+BOX_PATH = File.join(File.dirname(__FILE__), 'artifacts', 'vagrant.box')
+CORE_PACKER_FOLDER = File.join(File.dirname(__FILE__), 'packs', 'core', 'pack.sh')
+
+if File.exists?(BOX_PATH) == false
+  puts "Locally built vagrant.box does not exists. Creating ..."
+
+  Dir.chdir(File.dirname(CORE_PACKER_FOLDER)) {
+    system CORE_PACKER_FOLDER
+  }
+
+  puts "Done."
+
+end
+
+# Construct path to default apollo-fat-box profile
+BASE_DIR = File.expand_path(File.dirname(__FILE__))
+
+VAGRANT_DIR = File.join(BASE_DIR, 'vagrant')
+
+PROFILES_DIR = File.join(VAGRANT_DIR, 'profiles')
+
+DEFAULT_PROFILE_FILE = 'default.json'
+
+DEFAULT_PROFILE_PATH = File.join(PROFILES_DIR, DEFAULT_PROFILE_FILE)
+
+# Load default profile
+nodes = MultiJson::load(File.new(DEFAULT_PROFILE_PATH))
+
+# Iterate all configurations form loaded file
+Vagrant.configure("2") do |config|
+  nodes.each do |node_def|
+    node_name = node_def['hostname'] || 'apollo-fat-node'
+
+    config.vm.define node_name do |node|
+      node.vm.box = node_def['box']
+      node.vm.box_url = node_def['box_url']
+
+      node.vm.boot_timeout = 1800
+
+      node.ssh.username = 'vagrant'
+      node.ssh.password = 'vagrant'
+
+      ports = node_def['network']['ports'] || []
+      ports.each do |port|
+        guest_port = port['guest']
+        host_port = port['host']
+        node.vm.network 'forwarded_port', guest: guest_port, host: host_port, auto_correct: true
+      end
+
+      # Specify and customize virtualization provider
+      node.vm.provider 'virtualbox' do |provider|
+        provider.customize ['modifyvm', :id, '--memory', 2048]
+        provider.customize ['modifyvm', :id, '--cpuexecutioncap', '85']
+      end
+
+      # config.vm.provider :digital_ocean do |provider, override|
+      #   override.ssh.private_key_path = '~/.ssh/id_rsa'
+      #   override.vm.box = 'digital_ocean'
+      #
+      #   provider.client_id = ''
+      #   provider.api_key = ''
+      # end
+
+      provisions = node_def['provisions'] || []
+      provisions.each do |provision|
+        if provision['name'] === 'puppet'
+          node.vm.provision :puppet do |puppet|
+            # Path to manifests folder
+            val = provision['manifests_path']
+            puppet.manifests_path = val if val
+
+            # Name of manifest used
+            val = provision['manifest_file']
+            puppet.manifest_file = val if val
+
+            # Folder where are puppet modules stored
+            val = provision['module_path']
+            puppet.module_path = val if val
+          end
+        end
+      end
+    end
+  end
+end

data/WHATS_IN.md

@@ -0,0 +1,55 @@
+# Installed
+
+## Databases
+
+- MongoDB - from 10gen dist - http://docs.mongodb.org/manual/tutorial/install-mongodb-on-ubuntu/
+- CouchdDB 1.5.0 - from official source
+- PostgreSQL 9.3 - from official source
+
+## Libraries
+
+- build-essantials, linux-headers-$(uname -r) - from official source
+- DKMS - from official source 
+- libzmq3, libzmq3-dev - from official source
+
+## Programming Languages
+
+- clang 3.2 - from official source
+- clojure 1.2 - from official source
+- clojure 1.4 - from official source
+- haskell - using official source
+- jdk-7u55-linux-x64 - http://www.enqlu.com/2014/03/how-to-install-oracle-java-78-jdk-and.html
+- jdk-8u5-linux-x64 - http://www.enqlu.com/2014/03/how-to-install-oracle-java-78-jdk-and.html
+- llvm 3.2 - from official source
+- nodejs 0.10.25 - from official source
+- python 3.4 - from official source
+- rvm - from http://rvm.io/
+- ruby 1.9.3 - using rvm
+- scala 2.9.2 - from official source
+
+## Servers
+
+- Apache Solr 4.8.0 - http://lucene.apache.org/solr/
+- Apache Zookeper 3.4.5 - official source
+- ElasticSearch 1.1.1 - http://www.elasticsearch.org/overview/elkdownloads/
+- Memcached 1.4.14 - from official source
+- Nginx - from official source
+- Redis 2.6.13 - from official source
+- RabbitMQ 3.1.3 - from official source
+- Tomcat7 - from official source
+
+
+## Tools
+
+- htop - from official source
+- leiningen 1.7.1 - from official source
+- links - from official source
+- lynx - from official source
+- maven 3.0.5 - from official source
+- mc 4.8.11 - from official sourc
+- mosh - from official source
+- puppet - from official source
+- puppet-master - from official source
+- Vagrant - from official source
+- VirtualBox Guest Additions - from host
+

data/artifacts/.gitkeepme

@@ -0,0 +1 @@
+# Keep this file to keep me this folder

data/clean.sh

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+
+rm -rvf ~/VirtualBox\ VMs/apollo-fat-box_apollo-fat-box

data/lib/apollon/app/app.rb

@@ -0,0 +1 @@
+# encoding: utf-8

data/lib/apollon/core/core.rb

@@ -0,0 +1,8 @@
+# encoding: UTF-8
+
+require 'pathname'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + '*.rb').each do |file|
+  require file
+end

data/lib/apollon/helpers/helpers.rb

@@ -0,0 +1,28 @@
+require 'net/http'
+require 'json'
+
+uri = URI("https://api.github.com/gists")
+
+payload = {
+  'description' => "My test gist",
+  'public' => true,
+  'files' => {
+    'test.txt' => {
+      'content' => "This is a test!\n\nI am making a public gist."
+    }
+  }
+}
+
+req = Net::HTTP::Post.new(uri.path)
+req.body = payload.to_json
+
+puts req.inspect
+puts req.body.inspect
+
+# GitHub API is strictly via HTTPS, so SSL is mandatory
+res = Net::HTTP.start(uri.hostname, uri.port, :use_ssl => true) do |http|
+  http.request(req)
+end
+
+puts res.inspect
+puts res.body.inspect

data/lib/apollon/provisions/provisions.rb

@@ -0,0 +1,8 @@
+# encoding: UTF-8
+
+require 'pathname'
+
+base = Pathname(__FILE__).dirname.expand_path
+Dir.glob(base + '*.rb').each do |file|
+  require file
+end

data/lib/apollon/provisions/puppet.rb

@@ -0,0 +1 @@
+# encoding: UTF-8

data/lib/apollon/version.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+
+module Apollon
+  VERSION = '0.0.3'
+end

data/lib/apollon.rb

@@ -0,0 +1 @@
+require 'apollon/lib'

data/packs/README.md

@@ -0,0 +1,15 @@
+# Packs
+
+packer.io
+
+## Packs list
+
+List of referential packer.io packs.
+
+### core
+
+Core packer.io pack
+
+### examples
+
+Various packer.io example packs

data/packs/core/README.md

@@ -0,0 +1,3 @@
+# core
+
+Core referential packer.io package

data/packs/core/pack-digitalocean.sh

@@ -0,0 +1,8 @@
+#! /usr/bin/env bash
+
+packer build \
+    -only=digitalocean \
+    -var "digitalocean_id=$DIGITALOCEAN_ID" \
+    -var "digitalocean_api_key=$DIGITALOCEAN_API_KEY" \
+    -var "puppet_manifest_file=../../puppet/manifests/sites/apollocrawler.com.pp"
+    packer.json

data/packs/core/pack-vagrant.sh

@@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+
+ARCH='i386'
+ISO_CHECKSUM='08d25bf879e353686a974b7b14ae7d81'
+
+MACHINE_TYPE=`uname -m`
+if [ ${MACHINE_TYPE} == 'x86_64' ]; then
+    ARCH="amd64"
+    ISO_CHECKSUM="01545fa976c8367b4f0d59169ac4866c"
+fi
+
+packer build \
+    -only=virtualbox-iso \
+    -var "arch=$ARCH" \
+    -var "iso_checksum=$ISO_CHECKSUM" \
+    packer.json

data/packs/core/packer.json

@@ -0,0 +1,82 @@
+{
+    "variables": {
+        "arch": "amd64",
+        "iso_checksum": "01545fa976c8367b4f0d59169ac4866c",
+
+        "ssh_username": "vagrant",
+        "ssh_password": "vagrant",
+
+        "digitalocean_id": "",
+        "digitalocean_api_key": "",
+
+        "puppet_manifest_dir": "../../puppet/manifests",
+        "puppet_manifest_file": "../../puppet/manifests/web.pp",
+        "puppet_module_paths": "../../puppet/modules",
+
+        "vagrant_output": "../../artifacts/vagrant.box"
+    },
+
+    "builders": [
+        {
+            "type": "virtualbox-iso",
+            "guest_os_type": "Ubuntu_64",
+            "iso_url": "http://releases.ubuntu.com/14.04/ubuntu-14.04-server-{{user `arch`}}.iso",
+            "iso_checksum": "{{user `iso_checksum`}}",
+            "iso_checksum_type": "md5",
+            "ssh_username": "{{user `ssh_username`}}",
+            "ssh_password": "{{user `ssh_password`}}",
+            "ssh_wait_timeout": "60m",
+            "boot_command": [
+                "<esc><esc><enter><wait>",
+                "/install/vmlinuz noapic preseed/url=http://dl.dropboxusercontent.com/s/6teatka2gwz26n6/preseed.pkg?dl=1&token_hash=AAG33VL21u-_yHt5qTpQHjGvig1l7CLNlMY5PsHlwrfP1Q&expiry=1400794765 <wait>",
+                "debian-installer=en_US auto locale=en_US kbd-chooser/method=us <wait>",
+                "hostname={{ .Name }} <wait>",
+                "fb=false debconf/frontend=noninteractive <wait>",
+                "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA keyboard-configuration/variant=USA console-setup/ask_detect=false <wait>",
+                "initrd=/install/initrd.gz -- <enter><wait>"
+            ],
+            "shutdown_command": "echo 'packer' | sudo -S shutdown -P now"
+        },
+        {
+            "type": "digitalocean",
+            "client_id": "{{user `digitalocean_id`}}",
+            "api_key": "{{user `digitalocean_api_key`}}",
+            "region": "sfo1",
+            "image": "ubuntu-14-04-x64",
+            "size": "2gb",
+            "droplet_name": "apollo-fat-box-{{timestamp}}",
+            "snapshot_name": "apollo-fat-box-{{timestamp}}",
+            "ssh_port": 22,
+            "ssh_timeout": "5m",
+            "ssh_username": "root",
+            "private_networking": false
+        }
+    ],
+
+    "provisioners": [
+        {
+            "type": "shell",
+            "script": "provision.sh"
+        },
+        {
+            "type": "shell",
+            "script": "scripts/provision-digitalocean.sh",
+            "only": ["digitalocean"]
+        },
+        {
+            "type": "puppet-masterless",
+            "only": ["digitalocean"],
+            "manifest_dir": "{{user `puppet_manifest_dir`}}",
+            "manifest_file": "{{user `puppet_manifest_file`}}",
+            "module_paths": "{{user `puppet_module_paths`}}"
+        }
+    ],
+
+    "post-processors": [
+        {
+            "type": "vagrant",
+            "only": ["virtualbox-iso"],
+            "output": "{{user `vagrant_output`}}"
+        }
+    ]
+}