outrider 0.0.1

data/Rakefile

@@ -0,0 +1,52 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task" 
+require "net/ssh"
+require "yaml"
+
+task :default => :spec
+
+
+
+class TaskHelper
+  
+  attr_reader :data
+  
+  def initialize env
+    #load environment variables
+    @data  = YAML.load_file(File.join( Dir.home, "/.outrider/config/hosts.yml") )[env]
+  end
+end
+
+
+
+
+
+namespace :project  do
+  
+  @prod   = TaskHelper.new('prod').data
+  @dev    = TaskHelper.new('dev').data
+  
+  
+  
+  desc "Builds new project on development machine and production"
+  task :build, [:title, :domain] do |t, args|
+    puts ":: Building project #{args}"
+    # Build remote command
+    command = "#{@prod['ruby']}; #{@prod['ignite_path']} create_project_db_row -p #{args[:title]} -d #{args[:domain]}"
+    # add project to local system
+    sh "#{@dev['ignite_path']} create_project -p #{args[:title]} -d #{args[:domain]}"
+    # ssh and run on production server
+    begin
+      ssh = Net::SSH.start(@prod['host'], @prod['user'], :port => @prod['port'], :password => @prod['password'] )
+      res = ssh.exec!(command)
+      ssh.close
+      puts res
+    rescue Exception => e
+      puts "Unable to connect to #{@prod['host']} using #{@prod['user']} :: #{e}"
+    end
+  end
+  
+  
+  
+  
+end

data/app/run.rb

@@ -0,0 +1,15 @@
+require 'sinatra'
+
+
+
+get '/' do
+  "Outrider reporting for duty"
+end
+
+get '/test/1' do
+  "<h1 class='test_class'>Test 1</h1><a href='http://outriderapp.com/test/2'>Link</a><br /><p class='content'>This page is no use to you</p>"
+end
+
+get '/test/2' do
+  "<h1 class='test_class'>Test 2</h1><a href='http://outriderapp.com/test/1'>Link</a><a href='http://google.com'>External Link</a><a href='http://outriderapp.com/test/test.pdf'>Internal File Link</a><a href='http://google.com/test.pdf'>External File Link</a><br /><p class='content'>This page is no use to you</p>"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "outrider"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/outrider

@@ -0,0 +1,8 @@
+#!/usr/bin/ruby
+
+# Outrider loads in everything we need
+require 'outrider'
+# The engine is initialized - this sets everything up for us
+engine   = Engine.new
+# The process of interprating commands is started
+engine.run

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config.ru

@@ -0,0 +1,2 @@
+require ::File.expand_path('../app/run',  __FILE__)
+run Sinatra::Application

data/config/messages.yml

@@ -0,0 +1 @@
+no_method:     "Method doesn't exist"

data/config/schema.sql

@@ -0,0 +1,40 @@
+# ************************************************************
+# Database: outrider
+# ************************************************************
+
+
+
+# Dump of table projects
+# ------------------------------------------------------------
+
+CREATE TABLE `projects` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `title` varchar(255) DEFAULT NULL,
+  `domain` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+
+
+# Dump of table raw_data
+# ------------------------------------------------------------
+
+CREATE TABLE `raw_data` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `date_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  `url` varchar(255) DEFAULT NULL,
+  `title_raw` varchar(255) DEFAULT NULL,
+  `content_raw` text,
+  `status` varchar(255) DEFAULT 'unscraped',
+  `author` varchar(255) DEFAULT NULL,
+  `date_published_raw` varchar(255) DEFAULT NULL,
+  `project_id` int(11) DEFAULT NULL,
+  `date_published_timestamp` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+
+# Our seed data
+# -------------------------------------------------------------
+INSERT INTO `projects` (`title`, `domain`)
+VALUES ('test_project','http://outriderapp.com/test/1'),

data/lib/ignite.rb

@@ -0,0 +1,8 @@
+#!/usr/bin/ruby
+
+# Outrider loads in everything we need
+require_relative './outrider.rb'
+# The engine is initialized - this sets everything up for us
+engine   = Engine.new
+# The process of interprating commands is started
+engine.run

data/lib/outrider.rb

@@ -0,0 +1,94 @@
+#require_relative "outrider/version"
+require_relative "outrider/engine"
+require_relative "outrider/tools"
+require_relative "outrider/commandify"
+require_relative "outrider/project"
+
+
+# Provides an interface as to what commands can be run
+class Outrider
+  
+  
+  attr_reader :command_list, :project, :db, :config
+  
+  
+  
+  # When initialized, do so with our base project facade,
+  # and change it later once everything else is initialized based on the specified project
+  def initialize
+    @project  = Project.new
+    
+    @config   = {
+      :database => load_global_config( 'database',  "Couldn't load database configuration" ),
+      :messages => load_yaml( __FILE__, "../config/messages.yml",   "Couldn't load messages config file")
+    }
+    @db       = load_database 
+  end
+  
+  
+
+
+
+  def set_project_object project
+    project_path = OutriderTools::Store::get_filepath __FILE__, "../projects/#{project}/auxiliary.rb"
+    if File.exist? project_path
+      require_relative "../projects/#{project}/auxiliary"
+      # Initialze object for the project we're working on
+      @project = project.classify.constantize.new
+    else
+      return false
+    end
+  end
+
+
+
+
+  
+  def operate command, options = {}
+    
+      return @config[:messages]["no_method"] if command.nil?
+      
+      if @project.respond_to?(command)
+        return @project.send( command, options ) 
+      else
+        return @config[:messages]["no_method"]
+      end
+  end
+
+
+
+
+  private
+
+
+
+
+  def load_yaml file_object, filename, error_message
+    file = OutriderTools::Store::get_filepath file_object,  filename 
+    if File.exist? file
+       return YAML::load( File.open( file ))
+    else
+       return error_message
+    end
+  end
+  
+  
+  
+  def load_global_config filename, error_message
+    file = File.join( Dir.home, "/.outrider/config/#{filename}.yml" )
+    if File.exist? file
+       return YAML::load( File.open( file ))
+    else
+       return error_message
+    end
+  end
+
+
+
+
+  def load_database
+    ActiveRecord::Base.establish_connection(@config[:database])
+  end
+  
+  
+end

data/lib/outrider/commandify.rb

@@ -0,0 +1,49 @@
+module Commandify
+
+  
+  def self.process
+    # SYSTEM API METHODS Do not modify this line
+    sub_commands = %w(create_project create_project_db_row delete_project crawl intel test_super)
+    
+    
+    
+    # Place custom command options here. See instructions at http://manageiq.github.io/trollop/
+    sub_commands << %w()
+    
+    
+    global_opts  = Trollop::options do
+      banner "CLI for Outrider data processing tools"
+      opt :dry_run, "Don't actually do anything", :short => "-n"
+      stop_on sub_commands
+    end
+
+    command      = ARGV.shift
+
+    # Do not modify this
+    command_opts = Trollop::options do
+       # REQUIRED. Do not mess with these. Do not duplicate arguments or their short form. Run tests after modifying
+        opt :domain,        "The domain",                            :short   => "-d",  :type    => String, :default => ''
+        opt :limit,         "Limit",                                 :short   => "-l",  :type    => Integer,:default => 1000
+        opt :project,       "The name of the project",               :short   => "-p",  :type    => String, :default => ''
+        opt :filename,      "Write data to a filename",              :short   => "-f",  :type    => String, :default => ''
+        opt :restrict,      "Can only be crawled within the domain", :short   => "-r",  :default => true
+        opt :set_project,   "If we need to set project",             :short   => "-s",  :default => true
+        opt :intel_command, "What's the secondary command",          :short   => "-i",  :type    => String, :default => ''
+        
+        # CUSTOM. Place custom command options here
+        
+        
+      end
+      
+      puts "Invalid Command" unless sub_commands.include? command
+
+
+    return {
+      :action  => command,
+      :options => command_opts
+    }
+  end
+  
+  
+  
+end

data/lib/outrider/engine.rb

@@ -0,0 +1,20 @@
+class Engine
+  
+  attr_accessor :commands
+
+  def initialize commands = Commandify::process
+    @commands = commands
+  end
+  
+  
+  def run
+    p "No Method Given" if @commands[:action].nil?
+    outrider = Outrider.new
+    outrider.set_project_object( @commands[:options][:project] ) if @commands[:options][:set_project] == true
+    outrider.operate( @commands[:action], @commands[:options] )
+  end
+  
+  
+  
+  
+end

data/lib/outrider/intel.rb

@@ -0,0 +1,14 @@
+##  Intel is outrider's interface for running 
+##  statistical analysis libraries against it's data 
+##
+
+module OutriderIntel
+  
+  
+  def self.word_frequency words
+    words.inject(Hash.new(0)){|p,v| p[v]+=1; p}
+  end
+  
+  
+  
+end

data/lib/outrider/project.rb

@@ -0,0 +1,146 @@
+class Project
+  
+  attr_reader :config, :logger
+  
+  @@log    = Logger.new(STDOUT)
+  
+  def initialize
+    @config = {}
+  end
+  
+  
+  
+  def set_config name
+    project_meta = Projects.find_by( title: name )
+    @config = {
+      :id     => project_meta.id,
+      :title  => project_meta.title,
+      :domain => project_meta.domain
+    } unless project_meta.nil?
+  end
+  
+  
+  
+  
+  def self.create_folder options
+    
+    class_name   = options[:project].classify
+    project_name = options[:project].parameterize.underscore
+    file_path    = OutriderTools::Store::get_filepath( __FILE__, "../../projects/#{options[:project]}/auxiliary.rb" )
+
+    #create project files by making a copy of test_project
+    require 'fileutils'
+    
+    #create directories if they dont exist
+    dirname = File.dirname(file_path)
+    unless File.directory?(dirname)
+      FileUtils.mkdir_p(dirname)
+      @@log.info "Making directory: #{dirname}"
+    end
+    
+    #generate our default project class
+    File.open( file_path, 'w') { |file| 
+      file.write(%Q{class #{class_name} < Project\n\tdef initialize\n\t\tproject_name :#{project_name}\n\tend\nend})
+      @@log.info "Auxiliary File Created in: #{file.path}"
+    }
+  end
+  
+  
+  
+  
+  
+  def self.create_db_row options
+    #create project in database
+    project = Projects.create({ :title => options[:project], :domain => options[:domain] })
+    # TODO it might get stuck here if this domain already is in the raw data table
+    entry   = ProjectData.create({ :url => options[:domain], :status => 'unscraped', :project_id => project.id })
+    @@log.info "Project created in database: #{project.id}"
+  end
+  
+
+
+
+  
+  
+  
+  def self.delete options
+    #delete folder
+    folder_path  = OutriderTools::Store::get_filepath( __FILE__, "../../projects/#{options[:project]}" )
+    FileUtils.rm_rf( folder_path )
+    @@log.info "Deleting: #{folder_path}"
+    
+    #delete from database
+    project = Projects.find_by( title: options[:project] )
+    project.destroy unless project.nil?
+    @@log.info "Deleting: project from database: #{options[:project]}"
+  end
+  
+  
+  
+  
+  
+  
+  #
+  # These methods are here simply to help run our unit tests
+  def test_super options
+    p "Super Test Called"
+    return "Super Test Called"
+  end
+  
+  def project_name name
+    set_config name.to_s
+  end
+  
+  
+
+  
+  
+  
+  # A command line tool that lets us build a project
+  # /lib/ignite.rb create_project -p project -d domain.com
+  #
+  def create_project options
+    Project::create_folder options
+    Project::create_db_row options
+  end  
+  
+  
+  
+  
+  def create_project_db_row options
+    Project::create_db_row options
+  end
+    
+    
+    
+   def delete_project options 
+    return Project::delete options
+  end
+
+
+
+    
+end
+
+
+
+
+
+# Access to active record classes
+# This makes them usable throughout our code, as project.rb  
+# is autoloaded
+
+
+class Projects < ActiveRecord::Base
+  self.table_name = 'projects'
+  #has_many :project_data
+end
+
+
+
+class ProjectData < ActiveRecord::Base
+  self.table_name = 'raw_data'
+  #belongs_to :projects  
+end
+
+