outrider 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 41830dc2f20f2dca04892fa4a93db30a9a12286e
+  data.tar.gz: f8daadaa7e1bd18aecc62cc61b1650a18db96012
+SHA512:
+  metadata.gz: 2058dddd555c90772157e5ee5c37f13d3cc93677aba9c2376a8d1526ddba06e6af58f5a031b807235ea9183edee53cb9da8df97e403ae769831e2a2c65d59a65
+  data.tar.gz: 08439e7eb5656870da32361b22372979be7a540528f13187a14826611da593ecf863073ad673885ab9bdbc5915b9ad6b3056890c28beafa8e57bae97075e99f0

data/.gitignore

@@ -0,0 +1,17 @@
+/.bundle/
+/.yardoc
+#/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/config/*
+!/config/schema.sql
+!/config/messages.yml
+logfile.log
+#/lib/projects/*
+#!/lib/projects/
+*.log
+.DS_Store
+/log/*

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-version

@@ -0,0 +1 @@
+2.2.1

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.0.0

data/Capfile

@@ -0,0 +1,30 @@
+# Load DSL and set up stages
+require 'capistrano/setup'
+
+# Include default deployment tasks
+require 'capistrano/deploy'
+
+# Include tasks from other gems included in your Gemfile
+#
+# For documentation on these, see for example:
+#
+#   https://github.com/capistrano/rvm
+#   https://github.com/capistrano/rbenv
+#   https://github.com/capistrano/chruby
+#   https://github.com/capistrano/bundler
+#   https://github.com/capistrano/rails
+#   https://github.com/capistrano/passenger
+#
+# require 'capistrano/rvm'
+# require 'capistrano/rbenv'
+# require 'capistrano/chruby'
+# require 'capistrano/bundler'
+# require 'capistrano/rails/assets'
+# require 'capistrano/rails/migrations'
+# require 'capistrano/passenger'
+# require 'capistrano/ssh_doctor'
+require 'rvm1/capistrano3'
+require 'capistrano/bundler'
+
+# Load custom tasks from `lib/capistrano/tasks` if you have any defined
+Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r }

data/Gemfile

@@ -0,0 +1,20 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in outrider_ruby.gemspec
+gemspec
+
+gem 'trollop',                 '~> 2.1.2'
+gem 'nokogiri',                '~> 1.6.6.2'
+gem 'mechanize',               '~> 2.7.3'
+gem 'activesupport',           '~> 4.2.1'
+gem 'i18n',                    '~> 0.7.0'
+gem 'activerecord',            '~> 4.2.1'
+gem 'mysql2',                  '~> 0.3.18'
+gem 'rspec',                   '~> 3.2.0'
+gem 'capistrano',              '~> 3.4.0'
+gem 'rvm1-capistrano3',        '~> 1.3.2.2'
+gem 'capistrano-bundler',      '~> 1.1.4'
+gem 'rack',                    '~> 1.6.0'
+gem 'sinatra',                 '~> 1.4.6'
+gem 'facets',                  '~> 3.0.0'
+

data/Gemfile.lock

@@ -0,0 +1,119 @@
+PATH
+  remote: .
+  specs:
+    outrider (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.2.1)
+      activesupport (= 4.2.1)
+      builder (~> 3.1)
+    activerecord (4.2.1)
+      activemodel (= 4.2.1)
+      activesupport (= 4.2.1)
+      arel (~> 6.0)
+    activesupport (4.2.1)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    arel (6.0.0)
+    builder (3.2.2)
+    capistrano (3.4.0)
+      i18n
+      rake (>= 10.0.0)
+      sshkit (~> 1.3)
+    capistrano-bundler (1.1.4)
+      capistrano (~> 3.1)
+      sshkit (~> 1.2)
+    colorize (0.7.5)
+    diff-lcs (1.2.5)
+    domain_name (0.5.24)
+      unf (>= 0.0.5, < 1.0.0)
+    facets (3.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    i18n (0.7.0)
+    json (1.8.2)
+    mechanize (2.7.3)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (~> 2.0)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    mime-types (2.4.3)
+    mini_portile (0.6.2)
+    minitest (5.6.0)
+    mysql2 (0.3.18)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.2)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    ntlm-http (0.1.1)
+    rack (1.6.0)
+    rack-protection (1.5.3)
+      rack
+    rake (10.4.2)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.2)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    rvm1-capistrano3 (1.3.2.2)
+      capistrano (~> 3.0)
+      sshkit (>= 1.2)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    sshkit (1.7.1)
+      colorize (>= 0.7.0)
+      net-scp (>= 1.1.2)
+      net-ssh (>= 2.8.0)
+    thread_safe (0.3.5)
+    tilt (2.0.1)
+    trollop (2.1.2)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.6)
+    webrobots (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (~> 4.2.1)
+  activesupport (~> 4.2.1)
+  bundler (~> 1.9)
+  capistrano (~> 3.4.0)
+  capistrano-bundler (~> 1.1.4)
+  facets (~> 3.0.0)
+  i18n (~> 0.7.0)
+  mechanize (~> 2.7.3)
+  mysql2 (~> 0.3.18)
+  nokogiri (~> 1.6.6.2)
+  outrider!
+  rack (~> 1.6.0)
+  rake (~> 10.0)
+  rspec (~> 3.2.0)
+  rvm1-capistrano3 (~> 1.3.2.2)
+  sinatra (~> 1.4.6)
+  trollop (~> 2.1.2)

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Jaap Badlands
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,261 @@
+# OUTRIDER
+
+### Introduction
+
+**Outrider Data Framework provides structure and tools for collecting, cleaning, storing and analysing data from around the web** 
+
+Built using Ruby and Python, Outrider's purpose is to provide an easy-to-use interface and set of tools to help create and run tasks that can programmatically visit, process, scrape, clean, store, analyse, access and display data from online sources. 
+
+Outrider projects are easily created using a rake command and your new project file automatically has access to the OutriderTools API and database. For examples 
+
+```ruby  
+# creates a new project file
+rake project:build['nz_herald','http://nzherald.com']
+
+# /projects/nz_herald/auxiliary.rb
+# Returns scraped data from each page to be stored in the database
+# using the Outrider storage format.
+def crawl options
+OutriderTools::Crawl::site( @config, ->(page, uri){
+  unless(  page.css('.articleTitle').text.strip.empty? )
+    clean_date = DateTime.strptime(page.css('.storyDate').text.strip, '%a %b %d %H:%M:%S %Z %Y').to_s #Tue Mar 03 08:27:23 UTC 2015
+    return {
+      :title_raw                 => page.css('.articleTitle').text.strip,
+      :author                    => page.css('.authorName a').text.strip,
+      :content_raw               => page.css('#articleBody p').map{ |paragraph| paragraph.text.strip }.to_json,
+      :date_published_raw        => page.css('.storyDate').text.strip,
+      :date_published_timestamp  => clean_date,
+      :status                    => 'scraped'
+    }
+  else
+    return {
+      :status              => 'rejected'
+    }
+  end
+})
+end
+```
+
+
+### Features
+
+| Feature | Purpose |
+| ------- | ------- |
+Data Mining | Outrider provides tools for **collecting**, **cleaning** and **storing data** from the web. 
+Statistical Analysis | Outrider provides libraries for running **statistical algorithms** over datasets.
+	
+
+
+## How it works
+#### Command Line Interface
+At it's basic level, Outrider provides a command line interface, whose commands give us the ability to call and pass arguments to our API. The Command line is used by running `./lib/ignite.rb`. When you call this file through your shell, you must pass it a command to run and any arguments to pass through to the command. Such as:
+
+```shell
+./lib/ignite.rb crawl -p project_name
+```
+
+#### API
+The commands that can be recognised and run through the CLI form the *Outrider API*. The API sits behind and is accessed through the the command line. Actions that are common to the purposes of Outriders main goals - such as crawling and scraping, are made available publicly by the API and can be called by passing it as the first argument to `./lib/ignite.rb` when using the CLI. In the above example, the word *crawl* is a method of the Outrider API and `-p project_name` tells the API which project to look for the *crawl* implementation.
+
+##### API Extension
+The api can be extended by creating the functionality in your project's *auxiliary.rb* and modifying commandify.rb
+
+For example:
+
+```ruby
+# in ./lib/project/:project_name/auxiliary.rb
+def my_own_command( options )
+	# Here you have access to the OutriderTools module
+	# Also, options is a hash of the commands passed in through CLI
+end
+```
+
+Commands are handled by a gem called Trollop 
+
+*IMPORTANT!* 
+* Do not modify the existing Trollop configuration 
+* Read the Trollop documentation at http://manageiq.github.io/trollop/
+* Put your own configuration in the specified places - see below.
+* Always run tests `rspec spec` after modifying this
+* TODO - move this functionality into an setup where they don't have to touch commandify.rb
+
+```ruby
+# in ./lib/outrider/commandify.rb
+module Commandify
+  def self.process
+	# Place custom command options here. See instructions at http://manageiq.github.io/trollop/
+	sub_commands << %w()
+	# Set these to accept arguments through the command line and pass them to your auxiliary methods
+	command_opts = Trollop::options do
+	  # REQUIRED. Do not mess with the default options. 
+	  # Do not duplicate arguments or their short form. 
+	  # Run tests after modifying
+	  # opt :domain, "The domain", :short => "-d", :type => String, :default => ''
+
+
+	  # CUSTOM. Place custom command options here
+		
+		
+	end
+  end
+end
+```
+
+
+##### In your command line
+```shell
+ ./lib/ignite.rb my_own_command -p project_name -your_argument_key value
+```
+
+###### Options
+Once set up in commandify.rb, calling your new method in auxiliary.rb will pass in a hash of the options specified in the command line call. This means your auxiliary methods need to accept the options hash.
+```ruby 
+def auxiliary_method( options )
+  # options contains a hash such as { :project => 'project_name' }
+end
+```
+
+#### Projects
+When working with Outrider, you first create and then work within a *project*. You can create as many of these as you want. These let us create custom functionality to handle different jobs uniquely.
+
+
+#### Creating Projects
+##### CLI
+Projects can be created using the command line
+``` shell 
+# Creates ./lib/projects/:project_name/auxiliary.rb 
+#and adds a record to the projects table in the database, 
+# including a seed entry to the raw_data table
+./lib/ignite.rb create_project -p project_name
+
+# Just adds a record to the projects table of the database 
+#and the seed entry in the raw_data table (doesn't create an auxialiary file)
+./lib/ignite.rb create_project_db_row -p project_name -d http://domain.com
+
+# Deletes the project folder and row in the project table of the database
+./lib/ignite.rb delete_project -p project_name
+```
+
+##### RAKE
+Outrider is assumed to run on two machines - a dev and a production server. When you create a new project you do so on the dev server and it gets added to git, however in order to make a project runnable in production, there must be some entries in the database for that. Outrider has a rake command that is run from the dev server which creates the files and db entry on the dev server and also creates the necessary database entries on the production server. 
+
+```shell
+rake project:build['project_name','project_domain']
+```
+
+#### Customizing Projects
+Once created, a project consist of a file `./lib/projects/:project_name/auxiliary.rb` which contains a class whose public methods correspond to the CLI commands. 
+
+```ruby
+# lib/projects/test_project/auxiliary.rb
+
+class TestProject < Project
+
+	def initialize
+		project_name :test_project
+	end
+
+	def crawl
+		# See OutriderTools documentation http://github.com/deadlysyntax/outrider
+		# You have full access to the OutriderTools module 
+		# You inheret all the methods and instance variables defined in the global Project class
+		# Which also gives you access to @config which is a hash containing :id, :title and :domain of the project
+	
+		OutriderTools::Crawl::site( @config, ->(page, uri){
+      		# Use the Nokogiri page object here to do what you want to each page
+    	})
+	end
+end
+```
+
+
+
+## The process
+A call to `./ignite.rb crawl -p test_project` will 
+1. Check the validity of the command against the API definition in **./lib/outrider/commandify.rb**, 
+2. In this case **crawl** is a legitimate API method, and since that passes it will then
+3. Look in `./lib/project/test_project/auxiliary.rb` for a public method called **crawl**. 
+4. If it doesn't find it there, it will look in the global `Project` object
+5. It will call **crawl** and pass in all the options specified in the command line (as long as they're set up in commandify)
+
+
+
+# The OutriderTools module
+Outrider Tools is module that provides an API for the core functionality at the the heart of the framework. It is loaded globally, so that you can call these functions from your projects' `auxiliary.rb` files.
+ 
+### OutriderTools API
+#### Crawl
+##### site
+```ruby 
+OutriderTools::Crawl::site( project, each_page_callback )
+```
+| Argument | Expected Value | Description |
+| -------- | -------------- | ----------- |
+**project** | { :id => 0, :title => '', :domain => '' } | A hash of project config values
+**each_page_callback** | ->( page, uri ){} | A callback function to run, which gets passed the Nokogiri object and URI for each page
+
+Recursively looks to the ProjectData in the database for the first `status: 'unscraped'` data record for the specified project. While at each page, it will run the callback and pass it the Nokogiri::HTML object and the current URI. It builds a list of sanitized urls and adds them as ProjectData rows in the database. Thus, recursively filtering through an entire domain and acting on each page
+
+http://www.rubydoc.info/github/sparklemotion/nokogiri
+
+________________________________
+
+#### Scrape
+```ruby 
+OutriderTools::Scrape::page( url, operate )
+```
+| Argument | Expected Value | Description |
+| -------- | -------------- | ----------- |
+**url** | "http://domain.com" | A url to scrape
+**operate** | ->( page, uri ){} | A callback function to run, which gets passed the Nokogiri object and URI for each page
+
+Will go to the URL and run the callback and pass it the Nokogiri::HTML object and the current URI. 
+
+http://www.rubydoc.info/github/sparklemotion/nokogiri
+
+________________________________
+
+# Installation
+#### Git clone
+
+Move to the directory you'd like to put the Outrider app.
+
+> git clone git@github.com:deadlysyntax/outrider.git
+
+
+#### Configuration
+The following configuration files are required to be created are. Assuming ./ is app root
+```ruby
+# - config
+# - - \ - database.yml
+# - - - - hosts.yml
+# - - \ - deploy.rb
+# - - \ - deploy \
+# - - \ - deploy \ - production.rb
+```
+
+
+#### Database
+*TODO add migrations*
+The mysql-based schema for the Outrider database is found at https://github.com/deadlysyntax/outrider/blob/master/config/schema.sql This is only a guide, and will in future be handled by Active Record.
+
+Set up a database, import the schema (if using mysql) and create the following file **./config/database.yml**. This file is expected by the system and will not run without these steps being complete properly.
+
+```yaml
+host:     localhost
+username: root
+password: root
+database: outrider
+adapter:  mysql2
+```
+
+
+#### System Information
+Requires Ruby 2.2.1. Outrider is run on two machines. The development machine and the remote server and deployed using Capistrano. 
+
+#### Tests
+```shell
+# in project root ./
+rspec spec
+```
+