ngpod_scraper 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +9 -0
- data/Gemfile +4 -0
- data/LICENSE +20 -0
- data/README.markdown +46 -0
- data/Rakefile +1 -0
- data/VERSION +1 -0
- data/bin/ngpod_scraper +6 -0
- data/lib/ngpod_scraper/client.rb +42 -0
- data/lib/ngpod_scraper/ngpod_scraper.rb +9 -0
- data/lib/ngpod_scraper/photo.rb +87 -0
- data/lib/ngpod_scraper/photo_page.rb +25 -0
- data/lib/ngpod_scraper/version.rb +3 -0
- data/lib/ngpod_scraper.rb +12 -0
- data/ngpod_scraper.gemspec +29 -0
- data/spec/fixtures/config.yml +8 -0
- data/spec/fixtures/test.html +2698 -0
- data/spec/fixtures/test.jpg +0 -0
- data/spec/ngpod_scraper/client_spec.rb +81 -0
- data/spec/ngpod_scraper/ngpod_scraper_spec.rb +17 -0
- data/spec/ngpod_scraper/photo_page_spec.rb +39 -0
- data/spec/ngpod_scraper/photo_spec.rb +129 -0
- data/spec/spec.opts +1 -0
- data/spec/spec_helper.rb +16 -0
- metadata +183 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 George Mendoza
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.markdown
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
README
|
2
|
+
========================================================================
|
3
|
+
|
4
|
+
Download the photo of the day from the National Geographic website
|
5
|
+
and save it as a desktop wallpaper.
|
6
|
+
|
7
|
+
Installation
|
8
|
+
------------------------------------------------------------------------
|
9
|
+
|
10
|
+
gem install ngpod_scraper
|
11
|
+
|
12
|
+
Requirements
|
13
|
+
------------------------------------------------------------------------
|
14
|
+
|
15
|
+
* nokogiri
|
16
|
+
* pow
|
17
|
+
* rmagick
|
18
|
+
* valuable
|
19
|
+
|
20
|
+
Usage
|
21
|
+
------------------------------------------------------------------------
|
22
|
+
|
23
|
+
ngpod_scaper path/to/config.yml
|
24
|
+
|
25
|
+
Sample config.yml
|
26
|
+
------------------------------------------------------------------------
|
27
|
+
|
28
|
+
url: http://photography.nationalgeographic.com/photography/photo-of-the-day
|
29
|
+
show_logs: true
|
30
|
+
photo:
|
31
|
+
path_format: '/home/gsmendoza/Pictures/national-geographic/#{year}-#{month}/#{name}'
|
32
|
+
wallpaper_path_format: '/home/gsmendoza/Pictures/national-geographic/photo-of-the-day.jpg'
|
33
|
+
wallpaper_width: 1280
|
34
|
+
wallpaper_height: 800
|
35
|
+
wallpaper_background_color: black
|
36
|
+
|
37
|
+
`path_format` and `wallpaper_path_format` are evaluated as strings.
|
38
|
+
For convenience, the following values are provided:
|
39
|
+
|
40
|
+
* time, year, month, day, hour - based on `Time.now`
|
41
|
+
* name - the filename of the photo
|
42
|
+
|
43
|
+
Displaying the photos as your screensaver
|
44
|
+
------------------------------------------------------------------------
|
45
|
+
|
46
|
+
In Ubuntu, you'll need to do [this](http://cro.alienpants.com/index.php/2008/01/04/customising-the-pictures-folder-screensaver-in-ubuntu-gutsy/).
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/bin/ngpod_scraper
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
module NgpodScraper
|
2
|
+
class Client
|
3
|
+
attr_reader :config
|
4
|
+
|
5
|
+
def initialize(config)
|
6
|
+
@config = symbolize_keys(config)
|
7
|
+
end
|
8
|
+
|
9
|
+
def get_photo
|
10
|
+
PhotoPage.new(config[:url], config).photo
|
11
|
+
end
|
12
|
+
|
13
|
+
def log(message)
|
14
|
+
if config[:show_logs]
|
15
|
+
puts message
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def run
|
20
|
+
photo = get_photo
|
21
|
+
if photo.exists?
|
22
|
+
log "Wasn't able to download the photo of the day. Maybe the photo has already been downloaded."
|
23
|
+
else
|
24
|
+
photo.save
|
25
|
+
photo.save_wallpaper
|
26
|
+
log "Downloaded #{photo.path}"
|
27
|
+
end
|
28
|
+
photo
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def symbolize_keys(hash)
|
34
|
+
result = {}
|
35
|
+
hash.each do |key, value|
|
36
|
+
value = symbolize_keys(value) if value.is_a?(Hash)
|
37
|
+
result[key.to_sym] = value
|
38
|
+
end
|
39
|
+
result
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
module NgpodScraper
|
2
|
+
class Photo < Valuable
|
3
|
+
has_value :file
|
4
|
+
has_value :path_format
|
5
|
+
has_value :url
|
6
|
+
has_value :wallpaper_background_color
|
7
|
+
has_value :wallpaper_height
|
8
|
+
has_value :wallpaper_width
|
9
|
+
has_value :wallpaper_path_format
|
10
|
+
|
11
|
+
def exists?
|
12
|
+
Pow(path).exists?
|
13
|
+
end
|
14
|
+
|
15
|
+
def image
|
16
|
+
@image ||= Magick::ImageList.new(path)
|
17
|
+
end
|
18
|
+
|
19
|
+
def path
|
20
|
+
return @path if defined?(@path)
|
21
|
+
path_format_in_a_string = %Q{"#{path_format}"}
|
22
|
+
@path = Pow(eval(path_format_in_a_string)).to_s
|
23
|
+
end
|
24
|
+
|
25
|
+
def save
|
26
|
+
Pow(path).create "w" do |f|
|
27
|
+
f.write file.read
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def save_wallpaper
|
32
|
+
Pow(wallpaper_path).parent.create
|
33
|
+
wallpaper.write(wallpaper_path)
|
34
|
+
end
|
35
|
+
|
36
|
+
def wallpaper
|
37
|
+
scale = wallpaper_scale
|
38
|
+
resized_image = image.resize(scale * image.columns, scale * image.rows)
|
39
|
+
|
40
|
+
background_color = wallpaper_background_color
|
41
|
+
background = Magick::Image.new(wallpaper_width, wallpaper_height){
|
42
|
+
self.background_color = background_color
|
43
|
+
}
|
44
|
+
|
45
|
+
background.composite(resized_image,
|
46
|
+
(wallpaper_width - resized_image.columns)/2,
|
47
|
+
(wallpaper_height - resized_image.rows)/2,
|
48
|
+
Magick::OverCompositeOp)
|
49
|
+
end
|
50
|
+
|
51
|
+
def wallpaper_path
|
52
|
+
return @wallpaper_path if defined?(@wallpaper_path)
|
53
|
+
wallpaper_path_format_in_a_string = %Q{"#{wallpaper_path_format}"}
|
54
|
+
@wallpaper_path = Pow(eval(wallpaper_path_format_in_a_string)).to_s
|
55
|
+
end
|
56
|
+
|
57
|
+
def wallpaper_scale
|
58
|
+
[wallpaper_height.to_f/image.rows, wallpaper_width.to_f/image.columns, 1].min
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def day
|
64
|
+
time.strftime('%d')
|
65
|
+
end
|
66
|
+
|
67
|
+
def hour
|
68
|
+
time.strftime('%H')
|
69
|
+
end
|
70
|
+
|
71
|
+
def month
|
72
|
+
time.strftime('%m')
|
73
|
+
end
|
74
|
+
|
75
|
+
def name
|
76
|
+
Pow(url).name
|
77
|
+
end
|
78
|
+
|
79
|
+
def time
|
80
|
+
Time.now
|
81
|
+
end
|
82
|
+
|
83
|
+
def year
|
84
|
+
time.strftime('%Y')
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module NgpodScraper
|
2
|
+
class PhotoPage
|
3
|
+
attr_reader :config, :url
|
4
|
+
|
5
|
+
def initialize(url, config = {})
|
6
|
+
@url = url
|
7
|
+
@config = config
|
8
|
+
end
|
9
|
+
|
10
|
+
def photo
|
11
|
+
url = photo_url
|
12
|
+
attributes = (config[:photo] || {}).merge(:url => url, :file => open(photo_url))
|
13
|
+
return Photo.new(attributes)
|
14
|
+
end
|
15
|
+
|
16
|
+
def photo_url
|
17
|
+
page = Nokogiri::HTML(open(url))
|
18
|
+
begin
|
19
|
+
page.search(".primary_photo img").attr('src').value
|
20
|
+
rescue Exception => e
|
21
|
+
raise MissingPhotoUrl, "Cannot find the photo url: #{e.message}"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'nokogiri'
|
2
|
+
require 'open-uri'
|
3
|
+
require 'pow'
|
4
|
+
require 'RMagick'
|
5
|
+
require 'valuable'
|
6
|
+
require 'yaml'
|
7
|
+
|
8
|
+
require "ngpod_scraper/version"
|
9
|
+
require 'ngpod_scraper/ngpod_scraper'
|
10
|
+
require 'ngpod_scraper/client'
|
11
|
+
require 'ngpod_scraper/photo'
|
12
|
+
require 'ngpod_scraper/photo_page'
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "ngpod_scraper/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "ngpod_scraper"
|
7
|
+
s.version = NgpodScraper::VERSION
|
8
|
+
s.authors = ["George Mendoza"]
|
9
|
+
s.email = ["gsmendoza@gmail.com"]
|
10
|
+
s.homepage = "http://github.com/gsmendoza/ngpod_scraper"
|
11
|
+
s.summary = %q{National Geographic Photo of the Day Screen Scraper}
|
12
|
+
s.description = %q{A tool for scraping National Geographic's Photo of the Day}
|
13
|
+
|
14
|
+
s.rubyforge_project = "ngpod_scraper"
|
15
|
+
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
|
21
|
+
# specify any dependencies here; for example:
|
22
|
+
s.add_development_dependency "fakeweb", "~> 1.2.8"
|
23
|
+
s.add_development_dependency "rspec", "~> 1.3.0"
|
24
|
+
|
25
|
+
s.add_runtime_dependency "nokogiri", ">= 1.4.2"
|
26
|
+
s.add_runtime_dependency "pow", ">= 0.2.3"
|
27
|
+
s.add_runtime_dependency "rmagick", ">= 2.13.1"
|
28
|
+
s.add_runtime_dependency "valuable", ">= 0.8.2"
|
29
|
+
end
|
@@ -0,0 +1,8 @@
|
|
1
|
+
url: http://photography.nationalgeographic.com/photography/photo-of-the-day
|
2
|
+
show_logs: false
|
3
|
+
photo:
|
4
|
+
path_format: 'tmp/#{year}-#{month}/#{name}'
|
5
|
+
wallpaper_path_format: 'tmp/photo-of-the-day.jpg'
|
6
|
+
wallpaper_width: 1280
|
7
|
+
wallpaper_height: 800
|
8
|
+
wallpaper_background_color: black
|