fetcher 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Manifest.txt +3 -1
- data/Rakefile +1 -1
- data/bin/fetch +1 -1
- data/lib/fetcher.rb +14 -7
- data/lib/fetcher/cli/opts.rb +20 -0
- data/lib/fetcher/{runner.rb → cli/runner.rb} +17 -23
- data/lib/fetcher/version.rb +7 -0
- data/lib/fetcher/worker.rb +3 -1
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ef1b964e3b4a79b13b008147f894244610851bb9
|
4
|
+
data.tar.gz: 84815891771fe636ecf09891451dca80bb4fd83f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 01df88051ef004d00c25e9dcbdb1fb363737f1862106af02c4f74ad6712894960eb47c152853be0a29689c41aa47db7ebb57b00ef9154ff8564e92dafb4d213e
|
7
|
+
data.tar.gz: 13a9c6a495483b07a0fc4b693cec8cc205aa7de48333f95884535d4ac2ab61cd3e4d5e85b4b24c41f477fcee3a971d125381bae36d872b319d12a18dfb240d3c
|
data/Manifest.txt
CHANGED
data/Rakefile
CHANGED
data/bin/fetch
CHANGED
data/lib/fetcher.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
1
3
|
###
|
2
4
|
# NB: for local testing run like:
|
3
5
|
#
|
4
|
-
# 1.8.x: ruby -Ilib -rrubygems lib/fetcher.rb
|
5
6
|
# 1.9.x: ruby -Ilib lib/fetcher.rb
|
6
7
|
|
7
8
|
# core and stlibs
|
@@ -9,7 +10,6 @@
|
|
9
10
|
require 'yaml'
|
10
11
|
require 'pp'
|
11
12
|
require 'logger'
|
12
|
-
require 'optparse'
|
13
13
|
require 'fileutils'
|
14
14
|
require 'uri'
|
15
15
|
require 'net/http'
|
@@ -25,20 +25,22 @@ require 'logutils'
|
|
25
25
|
|
26
26
|
# our own code
|
27
27
|
|
28
|
-
require 'fetcher/
|
28
|
+
require 'fetcher/version'
|
29
29
|
require 'fetcher/worker'
|
30
30
|
|
31
31
|
|
32
32
|
module Fetcher
|
33
33
|
|
34
|
-
VERSION = '0.2.0'
|
35
|
-
|
36
34
|
# version string for generator meta tag (includes ruby version)
|
37
35
|
def self.banner
|
38
|
-
"
|
36
|
+
"fetcher #{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
|
39
37
|
end
|
40
38
|
|
41
39
|
def self.main
|
40
|
+
|
41
|
+
## NB: only load (require) cli code if called
|
42
|
+
|
43
|
+
require 'fetcher/cli/runner'
|
42
44
|
|
43
45
|
# allow env variable to set RUBYOPT-style default command line options
|
44
46
|
# e.g. -o site
|
@@ -64,4 +66,9 @@ module Fetcher
|
|
64
66
|
end # module Fetcher
|
65
67
|
|
66
68
|
|
67
|
-
|
69
|
+
if __FILE__ == $0
|
70
|
+
Fetcher.main
|
71
|
+
else
|
72
|
+
# say hello
|
73
|
+
puts Fetcher.banner
|
74
|
+
end
|
@@ -1,42 +1,36 @@
|
|
1
|
+
# encoding: utf-8
|
1
2
|
|
2
|
-
module Fetcher
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
@hash = {}
|
8
|
-
end
|
9
|
-
|
10
|
-
def put( key, value )
|
11
|
-
@hash[ key.to_s ] = value
|
12
|
-
end
|
13
|
-
|
14
|
-
def output_path
|
15
|
-
@hash.fetch( 'output', '.' )
|
16
|
-
end
|
4
|
+
# more core and stlibs
|
5
|
+
|
6
|
+
require 'optparse'
|
17
7
|
|
18
|
-
|
8
|
+
# our own code (for command line interface/cli)
|
19
9
|
|
10
|
+
require 'fetcher/cli/opts'
|
11
|
+
|
12
|
+
|
13
|
+
module Fetcher
|
20
14
|
|
21
15
|
class Runner
|
22
16
|
|
23
17
|
include LogUtils::Logging
|
24
|
-
|
18
|
+
|
25
19
|
attr_reader :opts
|
26
|
-
|
20
|
+
|
27
21
|
def initialize
|
28
|
-
@opts
|
22
|
+
@opts = Opts.new
|
29
23
|
end
|
30
24
|
|
31
25
|
def run( args )
|
32
26
|
opt=OptionParser.new do |cmd|
|
33
|
-
|
27
|
+
|
34
28
|
cmd.banner = "Usage: fetch [options] uri"
|
35
|
-
|
36
|
-
cmd.on( '-o', '--output PATH',
|
29
|
+
|
30
|
+
cmd.on( '-o', '--output PATH', "Output Path (default is '#{opts.output_path}')" ) { |s| opts.output_path = s }
|
37
31
|
|
38
32
|
# todo: find different letter for debug trace switch (use v for version?)
|
39
|
-
cmd.on(
|
33
|
+
cmd.on( '-v', '--verbose', 'Show debug trace' ) do
|
40
34
|
# todo/fix: use/change to logutils settings for level
|
41
35
|
# logger.datetime_format = "%H:%H:%S"
|
42
36
|
# logger.level = Logger::DEBUG
|
@@ -57,7 +51,7 @@ Further information:
|
|
57
51
|
|
58
52
|
EOS
|
59
53
|
|
60
|
-
cmd.on_tail(
|
54
|
+
cmd.on_tail( '-h', '--help', 'Show this message' ) do
|
61
55
|
puts usage
|
62
56
|
exit
|
63
57
|
end
|
data/lib/fetcher/worker.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
1
3
|
module Fetcher
|
2
4
|
|
3
5
|
class Worker
|
@@ -85,7 +87,7 @@ module Fetcher
|
|
85
87
|
|
86
88
|
logger.debug "GET #{uri.request_uri} uri=#{uri}, redirect_limit=#{redirect_limit}"
|
87
89
|
|
88
|
-
request = Net::HTTP::Get.new( uri.request_uri, { 'User-Agent'=>
|
90
|
+
request = Net::HTTP::Get.new( uri.request_uri, { 'User-Agent' => "fetcher gem v#{VERSION}" } )
|
89
91
|
if uri.instance_of? URI::HTTPS
|
90
92
|
http.use_ssl = true
|
91
93
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fetcher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Gerald Bauer
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-08-
|
11
|
+
date: 2013-08-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logutils
|
@@ -66,7 +66,9 @@ files:
|
|
66
66
|
- Rakefile
|
67
67
|
- bin/fetch
|
68
68
|
- lib/fetcher.rb
|
69
|
-
- lib/fetcher/
|
69
|
+
- lib/fetcher/cli/opts.rb
|
70
|
+
- lib/fetcher/cli/runner.rb
|
71
|
+
- lib/fetcher/version.rb
|
70
72
|
- lib/fetcher/worker.rb
|
71
73
|
homepage: https://github.com/geraldb/fetcher
|
72
74
|
licenses:
|