fetcher 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Manifest.txt +3 -1
- data/Rakefile +1 -1
- data/bin/fetch +1 -1
- data/lib/fetcher.rb +14 -7
- data/lib/fetcher/cli/opts.rb +20 -0
- data/lib/fetcher/{runner.rb → cli/runner.rb} +17 -23
- data/lib/fetcher/version.rb +7 -0
- data/lib/fetcher/worker.rb +3 -1
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ef1b964e3b4a79b13b008147f894244610851bb9
|
4
|
+
data.tar.gz: 84815891771fe636ecf09891451dca80bb4fd83f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 01df88051ef004d00c25e9dcbdb1fb363737f1862106af02c4f74ad6712894960eb47c152853be0a29689c41aa47db7ebb57b00ef9154ff8564e92dafb4d213e
|
7
|
+
data.tar.gz: 13a9c6a495483b07a0fc4b693cec8cc205aa7de48333f95884535d4ac2ab61cd3e4d5e85b4b24c41f477fcee3a971d125381bae36d872b319d12a18dfb240d3c
|
data/Manifest.txt
CHANGED
data/Rakefile
CHANGED
data/bin/fetch
CHANGED
data/lib/fetcher.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
1
3
|
###
|
2
4
|
# NB: for local testing run like:
|
3
5
|
#
|
4
|
-
# 1.8.x: ruby -Ilib -rrubygems lib/fetcher.rb
|
5
6
|
# 1.9.x: ruby -Ilib lib/fetcher.rb
|
6
7
|
|
7
8
|
# core and stlibs
|
@@ -9,7 +10,6 @@
|
|
9
10
|
require 'yaml'
|
10
11
|
require 'pp'
|
11
12
|
require 'logger'
|
12
|
-
require 'optparse'
|
13
13
|
require 'fileutils'
|
14
14
|
require 'uri'
|
15
15
|
require 'net/http'
|
@@ -25,20 +25,22 @@ require 'logutils'
|
|
25
25
|
|
26
26
|
# our own code
|
27
27
|
|
28
|
-
require 'fetcher/
|
28
|
+
require 'fetcher/version'
|
29
29
|
require 'fetcher/worker'
|
30
30
|
|
31
31
|
|
32
32
|
module Fetcher
|
33
33
|
|
34
|
-
VERSION = '0.2.0'
|
35
|
-
|
36
34
|
# version string for generator meta tag (includes ruby version)
|
37
35
|
def self.banner
|
38
|
-
"
|
36
|
+
"fetcher #{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
|
39
37
|
end
|
40
38
|
|
41
39
|
def self.main
|
40
|
+
|
41
|
+
## NB: only load (require) cli code if called
|
42
|
+
|
43
|
+
require 'fetcher/cli/runner'
|
42
44
|
|
43
45
|
# allow env variable to set RUBYOPT-style default command line options
|
44
46
|
# e.g. -o site
|
@@ -64,4 +66,9 @@ module Fetcher
|
|
64
66
|
end # module Fetcher
|
65
67
|
|
66
68
|
|
67
|
-
|
69
|
+
if __FILE__ == $0
|
70
|
+
Fetcher.main
|
71
|
+
else
|
72
|
+
# say hello
|
73
|
+
puts Fetcher.banner
|
74
|
+
end
|
@@ -1,42 +1,36 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
|
2
|
-
module Fetcher
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
@hash = {}
|
8
|
-
end
|
9
|
-
|
10
|
-
def put( key, value )
|
11
|
-
@hash[ key.to_s ] = value
|
12
|
-
end
|
13
|
-
|
14
|
-
def output_path
|
15
|
-
@hash.fetch( 'output', '.' )
|
16
|
-
end
|
4
|
+
# more core and stlibs
|
5
|
+
|
6
|
+
require 'optparse'
|
17
7
|
|
18
|
-
|
8
|
+
# our own code (for command line interface/cli)
|
19
9
|
|
10
|
+
require 'fetcher/cli/opts'
|
11
|
+
|
12
|
+
|
13
|
+
module Fetcher
|
20
14
|
|
21
15
|
class Runner
|
22
16
|
|
23
17
|
include LogUtils::Logging
|
24
|
-
|
18
|
+
|
25
19
|
attr_reader :opts
|
26
|
-
|
20
|
+
|
27
21
|
def initialize
|
28
|
-
@opts
|
22
|
+
@opts = Opts.new
|
29
23
|
end
|
30
24
|
|
31
25
|
def run( args )
|
32
26
|
opt=OptionParser.new do |cmd|
|
33
|
-
|
27
|
+
|
34
28
|
cmd.banner = "Usage: fetch [options] uri"
|
35
|
-
|
36
|
-
cmd.on( '-o', '--output PATH',
|
29
|
+
|
30
|
+
cmd.on( '-o', '--output PATH', "Output Path (default is '#{opts.output_path}')" ) { |s| opts.output_path = s }
|
37
31
|
|
38
32
|
# todo: find different letter for debug trace switch (use v for version?)
|
39
|
-
cmd.on(
|
33
|
+
cmd.on( '-v', '--verbose', 'Show debug trace' ) do
|
40
34
|
# todo/fix: use/change to logutils settings for level
|
41
35
|
# logger.datetime_format = "%H:%H:%S"
|
42
36
|
# logger.level = Logger::DEBUG
|
@@ -57,7 +51,7 @@ Further information:
|
|
57
51
|
|
58
52
|
EOS
|
59
53
|
|
60
|
-
cmd.on_tail(
|
54
|
+
cmd.on_tail( '-h', '--help', 'Show this message' ) do
|
61
55
|
puts usage
|
62
56
|
exit
|
63
57
|
end
|
data/lib/fetcher/worker.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
1
3
|
module Fetcher
|
2
4
|
|
3
5
|
class Worker
|
@@ -85,7 +87,7 @@ module Fetcher
|
|
85
87
|
|
86
88
|
logger.debug "GET #{uri.request_uri} uri=#{uri}, redirect_limit=#{redirect_limit}"
|
87
89
|
|
88
|
-
request = Net::HTTP::Get.new( uri.request_uri, { 'User-Agent'=>
|
90
|
+
request = Net::HTTP::Get.new( uri.request_uri, { 'User-Agent' => "fetcher gem v#{VERSION}" } )
|
89
91
|
if uri.instance_of? URI::HTTPS
|
90
92
|
http.use_ssl = true
|
91
93
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fetcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Gerald Bauer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-08-
|
11
|
+
date: 2013-08-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logutils
|
@@ -66,7 +66,9 @@ files:
|
|
66
66
|
- Rakefile
|
67
67
|
- bin/fetch
|
68
68
|
- lib/fetcher.rb
|
69
|
-
- lib/fetcher/
|
69
|
+
- lib/fetcher/cli/opts.rb
|
70
|
+
- lib/fetcher/cli/runner.rb
|
71
|
+
- lib/fetcher/version.rb
|
70
72
|
- lib/fetcher/worker.rb
|
71
73
|
homepage: https://github.com/geraldb/fetcher
|
72
74
|
licenses:
|