confed_scraper 0.0.2

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.2@confed_scraper --create

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,33 @@
+PATH
+  remote: .
+  specs:
+    confed_scraper (0.0.1)
+      nokogiri
+      rest-client
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    coderay (0.9.8)
+    method_source (0.6.7)
+      ruby_parser (>= 2.3.1)
+    mime-types (1.17.2)
+    nokogiri (1.5.0)
+    pry (0.9.7.4)
+      coderay (~> 0.9.8)
+      method_source (~> 0.6.7)
+      ruby_parser (>= 2.3.1)
+      slop (~> 2.1.0)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+    ruby_parser (2.3.1)
+      sexp_processor (~> 3.0)
+    sexp_processor (3.0.9)
+    slop (2.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  confed_scraper!
+  pry

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/confed_scraper.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "confed_scraper/version"
+
+Gem::Specification.new do |s|
+  s.name        = "confed_scraper"
+  s.version     = ConfedScraper::VERSION
+  s.authors     = ["Matt Polito"]
+  s.email       = ["matt.polito@gmail.com"]
+  s.homepage    = "http://github.com/mattpolito/confed_scraper"
+  s.summary     = %q{Site scrapers to ease data import into Confed}
+  s.description = %q{Site scrapers to ease data import into Confed}
+
+  s.rubyforge_project = "confed_scraper"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "pry"
+  s.add_runtime_dependency "rest-client"
+  s.add_runtime_dependency "nokogiri"
+end

data/lib/confed_scraper/confreaks_scraper.rb

@@ -0,0 +1,29 @@
+module ConfedScraper
+  class ConfreaksScraper < Scraper
+    SITE_URI = "http://confreaks.net"
+
+    def process
+      content = get_content_from(url)
+      videos_doc= Nokogiri::HTML.parse(content)
+      links = videos_doc.xpath('//*[@class="video"]//*[@class="title"]/a')
+      video_data = []
+      links.each_with_index do |link|
+        vid = {}
+        show_page_url = SITE_URI + link.attr('href')
+        show_page_content = RestClient.get(show_page_url)
+        show_page_doc = Nokogiri::HTML.parse(show_page_content)
+
+        scrape_message(show_page_url)
+
+        title = show_page_doc.xpath('//*[@class="video-title"]').text.strip
+        vid[:title] = title
+        vid[:uri] = show_page_url
+        vid[:presenters] = show_page_doc.xpath('//*[@class="video-presenters"]/a').map(&:text)
+        vid[:description] = show_page_doc.xpath('//*[@class="video-abstract"]/p').text.strip
+        video_data << vid
+      end
+
+      video_data
+    end
+  end
+end

data/lib/confed_scraper/scraper.rb

@@ -0,0 +1,20 @@
+module ConfedScraper
+  class Scraper
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+    end
+
+    def process(url)
+    end
+
+    def get_content_from(url)
+      RestClient.get(url)
+    end
+
+    def scrape_message(url)
+      p "Scraping: #{url}"
+    end
+  end
+end

data/lib/confed_scraper/version.rb

@@ -0,0 +1,3 @@
+module ConfedScraper
+  VERSION = "0.0.2"
+end

data/lib/confed_scraper.rb

@@ -0,0 +1,3 @@
+require "confed_scraper/version"
+require 'confed_scraper/scraper'
+require 'confed_scraper/confreaks_scraper'

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: confed_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Matt Polito
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-17 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: &70212084710360 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70212084710360
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: &70212084709940 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70212084709940
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70212084709520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70212084709520
+description: Site scrapers to ease data import into Confed
+email:
+- matt.polito@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- Rakefile
+- confed_scraper.gemspec
+- lib/confed_scraper.rb
+- lib/confed_scraper/confreaks_scraper.rb
+- lib/confed_scraper/scraper.rb
+- lib/confed_scraper/version.rb
+homepage: http://github.com/mattpolito/confed_scraper
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: confed_scraper
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Site scrapers to ease data import into Confed
+test_files: []