scrapers 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: db591801f04aaf0af906b5b93790c8ee7521a6d1
-  data.tar.gz: a72274f8326f3f1ad4412b15dec1fd801d821bad
+  metadata.gz: 9714ca4691bf79afd82dd1bac8588919703faf66
+  data.tar.gz: 2de713c77701348724b080af2d039739c2e33b67
 SHA512:
-  metadata.gz: 824c3b131c1bdcd1f4cfecba7b691eb1c07110075ca884452c2476614ec8d9466214c8c42e201e8516f04d67cdb96444840fbbe4015f4fdb5752501768d4f4c5
-  data.tar.gz: b7c9dcb97048074027e88d75f3fcedac78066b04a1cddf9b27dea075321523c31f4224842e14c770e37bf797c3064d3995321a4baa53b3e3574d64473611d100
+  metadata.gz: 1efecf60008126c24dee448e948ce49e9f187cac161732d5bb61ca171219e927116a05d799c676dbb789d0ed4e175837defe5889855076d3bad8090462249b25
+  data.tar.gz: e8d1d51f9dae5a2072b3f3282704982f1ab3483f2727c9ebcbc3a0c70aa3b58d1f1a7588bbd439fe762978ad69acd3093f7237c3ab5e1a9b04eef7228951b28c

data/Gemfile

@@ -2,3 +2,11 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in scrapers.gemspec
 gemspec
+
+group :development, :test do
+  gem "pry"
+  gem "pry-nav"
+  gem "pry-rescue"
+  gem "pry-stack_explorer"
+  gem "awesome_print"
+end

data/Rakefile

@@ -1 +1,47 @@
 require "bundler/gem_tasks"
+require "highline/import"
+require "active_support/core_ext/string/inflections"
+
+desc "Create the basis for a new scraper"
+task :new, [:module_name] do |t, args|
+  if args.module_name
+    module_name = args.module_name
+  else
+    module_name = ask("<%= color('What is the name of your new scraper module?', YELLOW) %>") {|q| q.default = "NewScraper" }
+  end
+
+  file_name = module_name.underscore + ".rb"
+  dir_name = File.join(File.dirname(__FILE__),'lib','scrapers')
+
+  new_scraper_path = File.join(dir_name, file_name)
+
+  template = <<-EOT
+module Scrapers
+  module #{module_name}
+
+    def self.scrape(url)
+      results = Hash.new
+
+      Mechanize.start(url) do |m|
+
+      end
+
+      results
+    end
+
+  end
+end
+EOT
+  
+  if File.exist?(new_scraper_path)
+    if agree("<%= color('#{file_name}', BLUE); color('already exists. Do you want to overwrite it?', YELLOW) %>", true)
+      File.unlink(new_scraper_path)
+    else
+      exit 0
+    end
+  end
+  
+  File.write(new_scraper_path, template)
+  say("<%= color('New scraper in', YELLOW) %> <%= color('#{new_scraper_path}', BLUE) %>")
+  
+end

data/bin/rubytapas

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'bundler/setup'
+require 'thor'
+require 'netrc'
+require 'scrapers/rubytapas'
+
+################################################################################
+#
+# Scraper for RubyTapas episodes.
+#
+# (Note: you need to subscribe to RupyTapas to download episodes.)
+#
+################################################################################
+
+class RubyTapasDownload < Thor
+
+  RUBYTAPAS="rubytapas.dpdcart.com"
+  RUBYTAPAS_EPISODE_URL = "https://#{RUBYTAPAS}/subscriber/post?id=\#{episode}"
+
+  desc "download", "Downloads the listed episode's files into a new directory with the episode tag in the given directory"
+  method_option :destination, :aliases => %w{-d --dest}, :desc => "Destination to store the downloads", :default => "."
+  method_option :url, :desc => "url to episode downloads (overrides episode)"
+  method_option :episode, :aliases => %w{-e --ep}, :desc => "Episode number"
+  method_option :user, :aliases => %w{-u -U}, :desc => "dpdcart user. Default is read from $HOME/.netrc"
+  method_option :password, :aliases => %w{-p -pw}, :desc => "dpdcart password. Default is read from $HOME/.netrc"
+
+  def download
+    netrc = Netrc.read
+    user, pw = netrc[RUBYTAPAS]
+    user = options.fetch("user", user)
+    pw = options.fetch("password", pw)
+    url = options.fetch("url", nil)
+    episode = options.fetch("episode", nil)
+    destination = options.fetch("destination", nil)
+    STDERR.puts "destination: #{destination}, episode: #{episode}, url: #{url}, user: #{user}, pw: #{pw.length}"
+    unless url
+      raise "Must give episode or full url" unless episode
+      url = RUBYTAPAS_EPISODE_URL.sub(%r[\#{episode}], episode)
+    end
+    Scrapers::RubyTapas.scrape url, user, pw, destination
+  end
+
+end
+
+RubyTapasDownload.start

data/lib/scrapers/manning_dashboard.rb

@@ -0,0 +1,15 @@
+module Scrapers
+  module ManningDashboard
+
+    def self.scrape(url)
+      results = Hash.new
+
+      Mechanize.start(url) do |m|
+
+      end
+
+      results
+    end
+
+  end
+end

data/lib/scrapers/rubytapas.rb

@@ -1,5 +1,6 @@
 require 'fileutils'
 require 'ostruct'
+require 'mechanize'
 
 module Scrapers
   

data/lib/scrapers/version.rb

@@ -2,7 +2,7 @@ module Scrapers
   module Version
 
     MAJOR = 1
-    MINOR = 2
+    MINOR = 3
     BUILD = 0
 
   end

data/scrapers.gemspec

@@ -18,12 +18,17 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "mechanize"
+  spec.add_dependency "netrc"
+
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
-  spec.add_dependency "mechanize"
   spec.add_development_dependency "guard"
   spec.add_development_dependency "guard-rspec"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "vcr"
+  
+  spec.add_development_dependency "active_support"
+  spec.add_development_dependency "highline"
 end

metadata

@@ -1,23 +1,23 @@
 --- !ruby/object:Gem::Specification
 name: scrapers
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Tamara Temple
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-18 00:00:00.000000000 Z
+date: 2014-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -25,7 +25,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: netrc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -39,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -53,13 +67,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mechanize
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -122,11 +136,40 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: active_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A library of web site scrapers utilizing mechanize and other goodies.
   Helpful in gathering images, moving things, saving things, etc.
 email:
 - tamouse@gmail.com
-executables: []
+executables:
+- rubytapas
 extensions: []
 extra_rdoc_files: []
 files:
@@ -137,6 +180,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/rubytapas
 - lib/scrapers.rb
 - lib/scrapers/allrecipes.rb
 - lib/scrapers/discoverynews.rb
@@ -144,6 +188,7 @@ files:
 - lib/scrapers/esod.rb
 - lib/scrapers/gocomics.rb
 - lib/scrapers/imgur.rb
+- lib/scrapers/manning_dashboard.rb
 - lib/scrapers/nasa_apod.rb
 - lib/scrapers/rubytapas.rb
 - lib/scrapers/sinfest.rb