snapcrawl 0.5.0.rc1 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/snapcrawl/cli.rb +2 -2
- data/lib/snapcrawl/config.rb +6 -1
- data/lib/snapcrawl/templates/docopt.txt +5 -5
- data/lib/snapcrawl/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b30cb829ebe341a727a3d1b5e9f26144a1063cef6fd241b953c17f53710deca7
|
4
|
+
data.tar.gz: 483678c0d5bce6c9ef63cf635619bd91c2a4d1c9441450e55390a9dc3293b496
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0f75cad05799ddeea7ac2926c021b40b46b10f53725a2a58d41a3cbb80da17110033f18ef5c9060e94bc3c0e0235ff279a665d38204695cfeba709c0364aa99a
|
7
|
+
data.tar.gz: 6977cda01914e9864d20eec2b8ad6e6d51a0b088123ada31c9ad8a5b725c1c2301fa1f8073e625bd20d7db1be2ecc857351584819b1c5fa41be4263ab818fa62
|
data/lib/snapcrawl/cli.rb
CHANGED
@@ -19,8 +19,8 @@ module Snapcrawl
|
|
19
19
|
private
|
20
20
|
|
21
21
|
def execute(args)
|
22
|
-
|
23
|
-
|
22
|
+
config_file = args['--config']
|
23
|
+
Config.load config_file if config_file
|
24
24
|
|
25
25
|
tweaks = args['SETTINGS'].pair_split
|
26
26
|
apply_tweaks tweaks if tweaks
|
data/lib/snapcrawl/config.rb
CHANGED
@@ -12,9 +12,14 @@ module Snapcrawl
|
|
12
12
|
|
13
13
|
file = "#{file}.yml" unless file =~ /\.ya?ml$/
|
14
14
|
|
15
|
+
# FIXME: Cannot use logger here due to the "chicken and egg" with
|
16
|
+
# Config. The $logger is available, but it was not yet fully
|
17
|
+
# configured with log_level etc.
|
15
18
|
if File.exist? file
|
19
|
+
# $logger.debug "loading config file %{green}#{file}%{reset}"
|
16
20
|
push file
|
17
21
|
else
|
22
|
+
# $logger.debug "creating config file %{green}#{file}%{reset}"
|
18
23
|
create_config file
|
19
24
|
end
|
20
25
|
end
|
@@ -31,6 +36,7 @@ module Snapcrawl
|
|
31
36
|
snaps_dir: 'snaps',
|
32
37
|
name_template: '%{url}',
|
33
38
|
url_whitelist: nil,
|
39
|
+
url_blacklist: nil,
|
34
40
|
css_selector: nil,
|
35
41
|
log_level: 1,
|
36
42
|
log_color: 'auto',
|
@@ -38,7 +44,6 @@ module Snapcrawl
|
|
38
44
|
end
|
39
45
|
|
40
46
|
def create_config(file)
|
41
|
-
$logger.debug "creating config file %{green}#{file}%{reset}"
|
42
47
|
content = File.read config_template
|
43
48
|
dir = File.dirname file
|
44
49
|
FileUtils.mkdir_p dir
|
@@ -7,9 +7,9 @@ Usage:
|
|
7
7
|
|
8
8
|
Options:
|
9
9
|
-c, --config FILE
|
10
|
-
Path to config file, with or without the .yml extension
|
11
|
-
A sample file will be created if not found
|
12
|
-
|
10
|
+
Path to config file, with or without the .yml extension.
|
11
|
+
A sample file will be created if not found.
|
12
|
+
The default filename is 'snapcrawl.yml'.
|
13
13
|
|
14
14
|
-h, --help
|
15
15
|
Show this screen
|
@@ -18,9 +18,9 @@ Options:
|
|
18
18
|
Show version number
|
19
19
|
|
20
20
|
Settings:
|
21
|
-
|
21
|
+
Provide any of the options available in the config as 'key=value'.
|
22
22
|
|
23
23
|
Examples:
|
24
24
|
snapcrawl example.com
|
25
25
|
snapcrawl example.com --config simple
|
26
|
-
snapcrawl example.com depth=1 log_level=2
|
26
|
+
snapcrawl example.com depth=1 log_level=2 width=768
|
data/lib/snapcrawl/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: snapcrawl
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.0
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Danny Ben Shitrit
|
@@ -162,9 +162,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
162
162
|
version: '2.3'
|
163
163
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
164
164
|
requirements:
|
165
|
-
- - "
|
165
|
+
- - ">="
|
166
166
|
- !ruby/object:Gem::Version
|
167
|
-
version:
|
167
|
+
version: '0'
|
168
168
|
requirements: []
|
169
169
|
rubygems_version: 3.0.3
|
170
170
|
signing_key:
|