socializer-scraper 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/socializer/scraper/extractor.rb +4 -2
- data/lib/socializer/scraper/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bc110e79790596f048e8745e0d346317c0ef1c0a
|
|
4
|
+
data.tar.gz: 4d084ed27c23b9218f808a8084ed6accae5462be
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: df9d0eb4c18b6c1e4f0bff4abd2c93b0ebb02af8567cda2d14550734d466fdafd6b47e3848def697d73bd6b7a9b64eff306b8db2583f1fcd6251d55d3f6fe57a
|
|
7
|
+
data.tar.gz: 258073993bb483525cbe1a2c39f521bac12f8d3a4be58f7d1d48d1b299696c1c5ae19a5982308b985bbc5358c990b5b9ad56ca97aa42d4c2ad5c47a74be2e3f9
|
|
@@ -29,7 +29,8 @@ module Socializer
|
|
|
29
29
|
|
|
30
30
|
def run *patterns, &block
|
|
31
31
|
data, options = {}, patterns.extract_options!
|
|
32
|
-
page_wise = options.
|
|
32
|
+
page_wise = options.delete(:page_wise)
|
|
33
|
+
patterns = patterns.push(options)
|
|
33
34
|
|
|
34
35
|
perform(*patterns) do |page|
|
|
35
36
|
collectors.each do |collector|
|
|
@@ -71,12 +72,13 @@ module Socializer
|
|
|
71
72
|
private
|
|
72
73
|
|
|
73
74
|
def perform *patterns, &block
|
|
75
|
+
options = patterns.extract_options!
|
|
74
76
|
message = "Please, provide a URL that starts with HTTP or HTTPS"
|
|
75
77
|
raise URI::InvalidURIError, message unless @url.url?
|
|
76
78
|
|
|
77
79
|
patterns.push(/.*/) if patterns.empty?
|
|
78
80
|
|
|
79
|
-
Anemone.crawl(@url) do |anemone|
|
|
81
|
+
Anemone.crawl(@url, options) do |anemone|
|
|
80
82
|
anemone.storage = Anemone::Storage.MongoDB
|
|
81
83
|
anemone.on_pages_like(*patterns) do |page|
|
|
82
84
|
@page, @html, @current_url = page, nil, page.url
|