right_scraper 1.0.8 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/right_scraper/scraper.rb +3 -2
- data/lib/right_scraper/scraper_base.rb +2 -2
- data/right_scraper.gemspec +1 -1
- metadata +1 -1
|
@@ -57,6 +57,7 @@ module RightScale
|
|
|
57
57
|
# repo(Hash|RightScale::Repository):: Repository to be scraped
|
|
58
58
|
# Note: repo can either be a Hash or a RightScale::Repo instance.
|
|
59
59
|
# See the RightScale::Repo class for valid Hash keys.
|
|
60
|
+
# incremental(FalseClass|TrueClass):: Whether scrape should be incremental if possible (true by default)
|
|
60
61
|
#
|
|
61
62
|
# === Block
|
|
62
63
|
# If a block is given, it will be called back with progress information
|
|
@@ -71,12 +72,12 @@ module RightScale
|
|
|
71
72
|
#
|
|
72
73
|
# === Raise
|
|
73
74
|
# 'Invalid repository type':: If repository type is not known
|
|
74
|
-
def scrape(repo, &callback)
|
|
75
|
+
def scrape(repo, incremental=true, &callback)
|
|
75
76
|
repo = RightScale::Repository.from_hash(repo) if repo.is_a?(Hash)
|
|
76
77
|
repo.repo_type = repo.repo_type.to_s # In case it's a symbol
|
|
77
78
|
raise "Invalid repository type" unless SCRAPERS.include?(repo.repo_type)
|
|
78
79
|
@scraper = @scrapers[repo.repo_type] ||= SCRAPERS[repo.repo_type].new(@scrape_dir, @max_bytes, @max_seconds)
|
|
79
|
-
@scraper.scrape(repo, &callback)
|
|
80
|
+
@scraper.scrape(repo, incremental, &callback)
|
|
80
81
|
@last_repo_dir = @scraper.current_repo_dir
|
|
81
82
|
@scraper.succeeded?
|
|
82
83
|
end
|
|
@@ -56,12 +56,12 @@ module RightScale
|
|
|
56
56
|
# Each scraper implementation should override scrape_imp which is called
|
|
57
57
|
# after this method initializes all the scraper attributes properly.
|
|
58
58
|
# See RightScale::Scraper#scrape
|
|
59
|
-
def scrape(repo, &callback)
|
|
59
|
+
def scrape(repo, incremental, &callback)
|
|
60
60
|
@repo = repo
|
|
61
61
|
@callback = callback
|
|
62
62
|
@current_repo_dir = ScraperBase.repo_dir(root_dir, repo)
|
|
63
63
|
@scrape_dir_path = File.expand_path(File.join(@current_repo_dir, '..'))
|
|
64
|
-
@incremental = incremental_update?
|
|
64
|
+
@incremental = incremental && incremental_update?
|
|
65
65
|
@errors = []
|
|
66
66
|
FileUtils.rm_rf(@current_repo_dir) unless @incremental
|
|
67
67
|
scrape_imp
|
data/right_scraper.gemspec
CHANGED
|
@@ -23,7 +23,7 @@ require 'rubygems'
|
|
|
23
23
|
|
|
24
24
|
spec = Gem::Specification.new do |spec|
|
|
25
25
|
spec.name = 'right_scraper'
|
|
26
|
-
spec.version = '1.0.
|
|
26
|
+
spec.version = '1.0.9'
|
|
27
27
|
spec.authors = ['Raphael Simon']
|
|
28
28
|
spec.email = 'raphael@rightscale.com'
|
|
29
29
|
spec.homepage = 'https://github.com/rightscale/right_scraper'
|