outrider 0.0.1

data/lib/outrider/tools.rb

@@ -0,0 +1,224 @@
+require 'nokogiri'
+require 'active_record'
+require 'active_support/core_ext/module/attribute_accessors'
+require 'active_support/inflector'
+require 'open-uri'
+require 'uri'
+require 'set'
+require 'json'
+require 'yaml'
+require 'logger'
+require 'time'
+require 'trollop'
+require_relative 'intel'
+
+
+module OutriderTools
+  
+  
+  module Crawl 
+    
+
+    
+    
+    def self.site project, operate
+      
+      @log = Logger.new(STDOUT)
+      
+      recurse = ->() do
+        #
+        # Pick a from the database to crawl
+        unless ProjectData.where( status: 'unscraped', project_id: project[:id] ).exists?
+          @log.info "No pages to scrape"
+          return false
+        end  
+
+        working_page = ProjectData.where( status: 'unscraped', project_id: project[:id]).first
+        working_page.status = 'processing'
+        working_page.save
+        
+        @log.info "Scraping #{working_page.url}"
+        # Scape it
+        data, links = OutriderTools::Scrape::page( working_page.url, operate)
+
+        # save links
+        OutriderTools::Link::save_many(links, project, @log )
+        
+        @log.info "Saving page data for url #{working_page.url}"
+        @log.info data[:status]
+        working_page.update( data ) unless data.nil?
+        
+        return true
+      end
+
+      crawl = true
+      while crawl
+        crawl = recurse.call
+      end
+      
+    end
+    
+    
+    
+
+    
+    
+    
+  end
+  
+  module Scrape
+     
+     
+     
+     
+     
+     def self.page( url, operate )
+       @log      = Logger.new('log/logfile.log', 'daily')
+       files     = OutriderTools::Clean::file_types
+       begin
+         page_uri = URI.parse( url )
+         doc      = Nokogiri.HTML( open(page_uri) ) 
+         # Yield page and URI to the block passed in 
+         data  = operate.( doc, page_uri )        
+
+         # Find all the links on the page
+         hrefs = doc.css('a[href]').map{ |a| a['href'] }
+
+         clean_uris  = OutriderTools::Clean::tidy_urls( hrefs, page_uri, page_uri, files )
+         return data, clean_uris
+         
+       rescue OpenURI::HTTPError # Guard against 404s
+         @log.error "Skipping invalid link #{page_uri}"
+       rescue ArgumentError => e
+         @log.error "Skipping page that causes argument error: #{e}"
+       rescue RuntimeError => e
+         @log.error "Invalid Redirection: #{e}"
+       rescue Exception => e
+         @log.error "Error #{e}"
+         raise e
+       end
+       
+       return { :status => 'rejected' }
+       
+     end
+     
+  end
+  
+  
+  
+  module Link
+    
+      def self.save_many( links, project, log )
+        
+        unless links.nil? 
+          links.each  do |link|
+            # Check if link already exists
+            #if ProjectData.find_by(url: link.to_s).nil?
+            unless ProjectData.where( url: link.to_s, project_id: project[:id] ).exists?  
+              ProjectData.create({
+                :url        => link.to_s,
+                :status     => 'unscraped',
+                :project_id => project[:id]
+              })
+              log.info "Adding new url to database: #{link.to_s}"
+            else
+              log.info "URL already exists in database: #{link.to_s}"
+            end
+          end
+        end
+        
+        
+      end
+    
+  end
+  
+  
+  
+  
+  module Clean
+    
+    
+    def self.tidy_urls hrefs, page_uri, domain, files 
+      # Make these URIs, throwing out problem ones like mailto:
+      uris = hrefs.map{ |href| URI.join( page_uri, href ) rescue nil }.compact
+
+      # Pare it down to only those pages that are on the same site
+      uris.select!{ |uri| uri.host == domain.host }
+
+      # Throw out links to files (this could be more efficient with regex)
+      uris.reject!{ |uri| files.any?{ |ext| uri.path.end_with?(".#{ext}") } }
+      
+      # Throw out duplicates
+      uris.reject!{ |uri| ProjectData.exists?( url: uri.to_s) }
+
+      # Remove #foo fragments so that sub-page links aren't differentiated
+      uris.each{ |uri| uri.fragment = nil }
+
+      return uris
+      
+    end
+    
+    
+    
+    
+    
+    
+    
+    def self.file_types sub = :all 
+      case sub
+      when :all 
+        return %w[png jpeg jpg gif svg txt js css zip gz pdf]
+      when :images
+        return %w[png jpeg jpg gif svg]
+      when :pdfs
+        return %w[pdf]
+      else
+        return %w[png jpeg jpg gif svg txt js css zip gz pdf]
+      end
+    end
+    
+    
+    
+    
+    # takes string of words, sorts out duds and returns array
+    def self.process_words_to_array words = ""
+      clean_words = words.split.each do |word|
+        word.downcase!
+      end
+    end
+    
+    
+    
+    # takes array of strings and combines them
+    def self.word_array_to_string strings
+      the_string = ''
+      strings.each do |string|
+        the_string += string.gsub(/[^a-z0-9\s]/i, '')
+      end
+      return the_string
+    end
+     
+  end
+  
+  
+  
+  
+  module Store
+    
+    def self.get_filepath base, filename
+      File.expand_path(File.join(File.dirname(base), filename ))
+    end
+    
+  end
+  
+  
+  
+  
+  
+  
+  
+end
+
+
+
+

data/lib/outrider/version.rb

@@ -0,0 +1,3 @@
+module Outrider
+  VERSION = "0.0.1"
+end

data/outrider.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "outrider"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Jaap Badlands"]
+  spec.email         = ["jaap@deadlysyntax.com"]
+
+  spec.summary       = %q{Outrider Web Automation Framework provides structure and tools for writing web-automation tasks}
+  spec.description   = %q{Outrider's purpose is to provide an easy-to-use programming interface and organisational structure, to create and run tasks that can automatically visit, interact with and test websites and also that process, clean and store data, and tools for statistical analysis. }
+  spec.homepage      = "https://github.com/deadlysyntax/outrider"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = ['outrider']
+  spec.require_paths = ["lib"]
+
+  if spec.respond_to?(:metadata)
+    #spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com' to prevent pushes to rubygems.org, or delete to allow pushes to any server."
+  end
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/projects/nz_herald/auxiliary.rb

@@ -0,0 +1,56 @@
+class NzHerald < Project
+  
+
+	def initialize
+		project_name :nz_herald
+		@log = Logger.new(STDOUT)
+	end
+	
+  
+	
+	
+  def crawl options
+    OutriderTools::Crawl::site( @config, ->(page, uri){
+      unless(  page.css('.articleTitle').text.strip.empty? )
+        clean_date = DateTime.strptime(page.css('.storyDate').text.strip, '%a %b %d %H:%M:%S %Z %Y').to_s #Tue Mar 03 08:27:23 UTC 2015
+        return {
+          :title_raw                 => page.css('.articleTitle').text.strip,
+          :author                    => page.css('.authorName a').text.strip,
+          :content_raw               => page.css('#articleBody p').map{ |paragraph| paragraph.text.strip }.to_json,
+          :date_published_raw        => page.css('.storyDate').text.strip,
+          :date_published_timestamp  => clean_date,
+          :status                    => 'scraped'
+        }
+      else
+        return {
+          :status              => 'rejected'
+        }
+      end
+    })
+  end
+  
+  
+  
+  def intel options
+    
+    raw_articles = ProjectData.where( "project_id = ? AND content_raw IS NOT NULL", @config[:id] ).limit(10000)
+    
+    paragraphs   = ''
+    words        = []
+    
+    raw_articles.each do |article|
+      paragraphs   += OutriderTools::Clean::word_array_to_string(  JSON.parse( article.content_raw ) ) + ' '
+    end
+    
+    words        = OutriderIntel::word_frequency( OutriderTools::Clean::process_words_to_array( paragraphs ) )
+    words        = words.sort_by { |word, frequency| frequency }
+      
+    p words
+    
+  end
+  
+  
+  
+end
+
+

data/projects/stuff/auxiliary.rb

@@ -0,0 +1,71 @@
+class Stuff < Project
+  
+  
+  
+  def initialize
+    
+    project_name :stuff
+
+  end
+  
+  
+  
+  def crawl options
+    OutriderTools::Crawl::site( @config, ->(page, uri){
+      unless( page.css('.story_landing').text.strip.empty? )
+        #p page.css('.story_landing .story__dateline span').empty?
+        unless page.css('.story_landing .story__dateline span').empty?
+          clean_date = DateTime.strptime( page.css('.story_landing .story__dateline span')[0]["content"], '%a %b %d %H:%M:%S %Z %Y').to_s 
+        else
+          return { :status  => 'rejected' }
+        end
+        
+        return {
+          :url                      => uri.to_s,
+          :title_raw                => page.css('.story_content_top h1').text.strip,
+          :author                   => page.css('.story__byline span[itemprop="name"]').text.strip,
+          :content_raw              => page.css('.story_landing > p').map{ |paragraph| paragraph.text.strip }.to_json,
+          :date_published_raw       => page.css('.story_landing .story__dateline span')[0]["content"],
+          :date_published_timestamp => clean_date,
+          :status                   => 'scraped'
+        }
+      else
+        return { :status  => 'rejected' }
+      end
+    })
+  end
+  
+  
+  
+  def intel options
+    
+    raw_articles = ProjectData.where( "project_id = ? AND content_raw IS NOT NULL", @config[:id] ).limit(10000)
+    
+    paragraphs   = ''
+    words        = []
+    
+    raw_articles.each do |article|
+      paragraphs   += OutriderTools::Clean::word_array_to_string(  JSON.parse( article.content_raw ) ) + ' '
+    end
+    
+    words        = OutriderIntel::word_frequency( OutriderTools::Clean::process_words_to_array( paragraphs ) )
+    words        = words.sort_by { |word, frequency| frequency }
+      
+    p words
+    
+  end
+  
+  
+  
+  
+
+
+    
+  
+  
+  
+  
+  
+  
+  
+end

data/projects/test_project/auxiliary.rb

@@ -0,0 +1,63 @@
+class TestProject < Project
+  
+  
+  
+  def initialize
+    
+    project_name :test_project
+
+  end
+  
+  
+  
+  
+  def test_method_true options
+    return options
+  end
+  
+  def test_method_false options
+    return false
+  end
+  
+  
+  
+
+  #
+  # options are passed through from the command line
+  #
+  def crawl options
+
+    OutriderTools::Crawl::site( @config, ->(page, uri){
+      unless( page.css('.story_landing').text.strip.empty? )
+        #p page.css('.story_landing .story__dateline span').empty?
+        unless page.css('.story_landing .story__dateline span').empty?
+          clean_date = DateTime.strptime( page.css('.story_landing .story__dateline span')[0]["content"], '%a %b %d %H:%M:%S %Z %Y').to_s 
+        else
+          return { :status  => 'rejected' }
+        end
+        
+        return {
+          :url                      => uri.to_s,
+          :title_raw                => page.css('.story_content_top h1').text.strip,
+          :author                   => page.css('.story__byline span[itemprop="name"]').text.strip,
+          :content_raw              => page.css('.story_landing > p').map{ |paragraph| paragraph.text.strip }.to_json,
+          :date_published_raw       => page.css('.story_landing .story__dateline span')[0]["content"],
+          :date_published_timestamp => clean_date,
+          :status                   => 'scraped'
+        }
+      else
+        return { :status  => 'rejected' }
+      end
+    })
+    
+  end
+
+    
+  
+  
+  
+  
+  
+  
+  
+end