fetcher 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.markdown +27 -9
- data/Rakefile +12 -2
- data/bin/fetch +0 -0
- data/lib/fetcher.rb +15 -1
- data/lib/fetcher/runner.rb +11 -17
- data/lib/fetcher/worker.rb +48 -19
- metadata +63 -68
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 930c43d21e2b74f6d413e59aef2cbdff621d9c54
|
4
|
+
data.tar.gz: abeb5fc3593a9f13beaba961d1fcc8c30a0cd0d3
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6ceebb4318870c02805cdf52c53ae8b9ff35c0adb004638adc0082a954662ea05201390888aaaeac3e8b60efb05cb0f1de08308087a87606f5a93fce1f5f8652
|
7
|
+
data.tar.gz: e3100ec244e84a23798c38b895ce5017d99582462d064f07b198e18b88cd695c8ff6752834e02275be4a6ad6404ae00e198112411557107eddc9202f2c9dc35a
|
data/README.markdown
CHANGED
@@ -1,22 +1,35 @@
|
|
1
1
|
# fetcher - Fetch Text Documents or Binary Blobs via HTTP, HTTPS
|
2
2
|
|
3
|
-
* [
|
3
|
+
* home :: [github.com/geraldb/fetcher](https://github.com/geraldb/fetcher)
|
4
|
+
* bugs :: [github.com/geraldb/fetcher/issues](https://github.com/geraldb/fetcher/issues)
|
5
|
+
* gem :: [rubygems.org/gems/fetcher](https://rubygems.org/gems/fetcher)
|
6
|
+
* rdoc :: [rubydoc.info/gems/fetcher](http://rubydoc.info/gems/fetcher)
|
7
|
+
* forum :: [groups.google.com/group/webslideshow](https://groups.google.com/group/webslideshow)
|
8
|
+
|
4
9
|
|
5
10
|
## Description
|
6
11
|
|
7
|
-
TBD
|
8
12
|
|
9
13
|
## Usage
|
10
14
|
|
11
|
-
|
12
|
-
|
13
|
-
Fetcher.copy( '
|
15
|
+
### Copy (to File)
|
16
|
+
|
17
|
+
Fetcher.copy( 'https://raw.github.com/openfootball/at-austria/master/2013_14/bl.txt', '/tmp/bl.txt' )
|
14
18
|
|
15
19
|
or
|
16
20
|
|
17
|
-
|
18
|
-
worker
|
19
|
-
|
21
|
+
worker = Fetcher::Worker.new
|
22
|
+
worker.copy( 'https://raw.github.com/openfootball/at-austria/master/2013_14/bl.txt', '/tmp/bl.txt' )
|
23
|
+
|
24
|
+
|
25
|
+
### Read (into String)
|
26
|
+
|
27
|
+
txt = Fetcher.read( 'https://raw.github.com/openfootball/at-austria/master/2013_14/bl.txt' )
|
28
|
+
|
29
|
+
or
|
30
|
+
|
31
|
+
worker = Fetcher::Worker.new
|
32
|
+
txt = worker.read( 'https://raw.github.com/openfootball/at-austria/master/2013_14/bl.txt' )
|
20
33
|
|
21
34
|
|
22
35
|
|
@@ -29,11 +42,16 @@ Just install the gem:
|
|
29
42
|
|
30
43
|
## Real World Usage
|
31
44
|
|
32
|
-
The [`slideshow`](http://slideshow.
|
45
|
+
The [`slideshow`](http://slideshow-s9.github.io) (also known as Slide Show (S9)) gem
|
33
46
|
that lets you create slide shows
|
34
47
|
and author slides in plain text using a wiki-style markup language that's easy-to-write and easy-to-read
|
35
48
|
ships with the `fetcher` gem.
|
36
49
|
|
50
|
+
The [`sportdb`](https://github.com/geraldb/sport.db.ruby) gem that lets you read football (soccer) fixtures
|
51
|
+
and more in plain text
|
52
|
+
ships with the `fetcher` gem.
|
53
|
+
|
54
|
+
|
37
55
|
|
38
56
|
## License
|
39
57
|
|
data/Rakefile
CHANGED
@@ -8,7 +8,7 @@ Hoe.spec 'fetcher' do
|
|
8
8
|
self.summary = 'fetcher - Fetch Text Documents or Binary Blobs via HTTP, HTTPS'
|
9
9
|
self.description = summary
|
10
10
|
|
11
|
-
self.urls = ['
|
11
|
+
self.urls = ['https://github.com/geraldb/fetcher']
|
12
12
|
|
13
13
|
self.author = 'Gerald Bauer'
|
14
14
|
self.email = 'webslideshow@googlegroups.com'
|
@@ -16,5 +16,15 @@ Hoe.spec 'fetcher' do
|
|
16
16
|
# switch extension to .markdown for gihub formatting
|
17
17
|
self.readme_file = 'README.markdown'
|
18
18
|
self.history_file = 'History.markdown'
|
19
|
+
|
20
|
+
self.extra_deps = [
|
21
|
+
['logutils', '>= 0.6']
|
22
|
+
]
|
23
|
+
|
24
|
+
self.licenses = ['Public Domain']
|
25
|
+
|
26
|
+
self.spec_extras = {
|
27
|
+
:required_ruby_version => '>= 1.9.2'
|
28
|
+
}
|
19
29
|
|
20
|
-
end
|
30
|
+
end
|
data/bin/fetch
CHANGED
File without changes
|
data/lib/fetcher.rb
CHANGED
@@ -19,6 +19,10 @@ require 'date'
|
|
19
19
|
require 'cgi'
|
20
20
|
|
21
21
|
|
22
|
+
# 3rd party gems
|
23
|
+
|
24
|
+
require 'logutils'
|
25
|
+
|
22
26
|
# our own code
|
23
27
|
|
24
28
|
require 'fetcher/runner'
|
@@ -27,7 +31,7 @@ require 'fetcher/worker'
|
|
27
31
|
|
28
32
|
module Fetcher
|
29
33
|
|
30
|
-
VERSION = '0.
|
34
|
+
VERSION = '0.2.0'
|
31
35
|
|
32
36
|
# version string for generator meta tag (includes ruby version)
|
33
37
|
def self.banner
|
@@ -47,6 +51,16 @@ module Fetcher
|
|
47
51
|
Runner.new.run(args)
|
48
52
|
end
|
49
53
|
|
54
|
+
# convenience shortcuts
|
55
|
+
|
56
|
+
def self.copy( src, dest )
|
57
|
+
Worker.new.copy( src, dest )
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.read( src )
|
61
|
+
Worker.new.read( src )
|
62
|
+
end
|
63
|
+
|
50
64
|
end # module Fetcher
|
51
65
|
|
52
66
|
|
data/lib/fetcher/runner.rb
CHANGED
@@ -1,11 +1,6 @@
|
|
1
1
|
|
2
2
|
module Fetcher
|
3
3
|
|
4
|
-
def self.copy( src, dest )
|
5
|
-
Worker.new( Logger.new(STDOUT) ).copy( src, dest )
|
6
|
-
end
|
7
|
-
|
8
|
-
|
9
4
|
class Opts
|
10
5
|
|
11
6
|
def initialize
|
@@ -24,13 +19,12 @@ module Fetcher
|
|
24
19
|
|
25
20
|
|
26
21
|
class Runner
|
22
|
+
|
23
|
+
include LogUtils::Logging
|
27
24
|
|
28
|
-
attr_reader :logger
|
29
25
|
attr_reader :opts
|
30
26
|
|
31
27
|
def initialize
|
32
|
-
@logger = Logger.new(STDOUT)
|
33
|
-
@logger.level = Logger::INFO
|
34
28
|
@opts = Opts.new
|
35
29
|
end
|
36
30
|
|
@@ -43,22 +37,23 @@ module Fetcher
|
|
43
37
|
|
44
38
|
# todo: find different letter for debug trace switch (use v for version?)
|
45
39
|
cmd.on( "-v", "--verbose", "Show debug trace" ) do
|
46
|
-
|
47
|
-
logger.
|
40
|
+
# todo/fix: use/change to logutils settings for level
|
41
|
+
# logger.datetime_format = "%H:%H:%S"
|
42
|
+
# logger.level = Logger::DEBUG
|
48
43
|
end
|
49
44
|
|
50
45
|
usage =<<EOS
|
51
46
|
|
52
|
-
fetch - Lets you fetch text documents or binary blobs via HTTP, HTTPS.
|
47
|
+
fetch #{VERSION} - Lets you fetch text documents or binary blobs via HTTP, HTTPS.
|
53
48
|
|
54
49
|
#{cmd.help}
|
55
50
|
|
56
51
|
Examples:
|
57
|
-
fetch
|
58
|
-
fetch -o downloads
|
52
|
+
fetch https://raw.github.com/openfootball/at-austria/master/2013_14/bl.txt
|
53
|
+
fetch -o downloads https://raw.github.com/openfootball/at-austria/master/2013_14/bl.txt
|
59
54
|
|
60
55
|
Further information:
|
61
|
-
|
56
|
+
https://github.com/geraldb/fetcher
|
62
57
|
|
63
58
|
EOS
|
64
59
|
|
@@ -87,15 +82,14 @@ EOS
|
|
87
82
|
|
88
83
|
## todo: use output path option
|
89
84
|
|
90
|
-
Worker.new
|
85
|
+
Worker.new.copy( src, dest )
|
91
86
|
|
92
87
|
end # each arg
|
93
88
|
|
94
|
-
puts
|
89
|
+
puts 'Done.'
|
95
90
|
|
96
91
|
end # method run
|
97
92
|
|
98
93
|
end # class Runner
|
99
94
|
|
100
95
|
end # module Fetcher
|
101
|
-
|
data/lib/fetcher/worker.rb
CHANGED
@@ -2,18 +2,57 @@ module Fetcher
|
|
2
2
|
|
3
3
|
class Worker
|
4
4
|
|
5
|
-
|
5
|
+
include LogUtils::Logging
|
6
6
|
|
7
|
-
|
8
|
-
|
7
|
+
### todo/fix:
|
8
|
+
# remove logger from c'tor
|
9
|
+
# use logutils instead
|
10
|
+
|
11
|
+
def initialize( old_logger_do_not_use=nil )
|
12
|
+
if old_logger_do_not_use != nil
|
13
|
+
puts "*** depreciated API call [Fetcher.initialize] - do NOT pass in logger; no longer required/needed; logger arg will get removed"
|
14
|
+
end
|
9
15
|
end
|
10
16
|
|
11
17
|
|
12
|
-
|
18
|
+
def read( src )
|
19
|
+
# return contents (response body) a string
|
20
|
+
logger.debug "fetch - copy src: #{src} into string"
|
21
|
+
|
22
|
+
response = get_response( src )
|
23
|
+
|
24
|
+
# on error return empty string - check: better return nil- why? why not??
|
25
|
+
return '' if response.nil?
|
26
|
+
|
27
|
+
response.body.dup # return string copy
|
28
|
+
end
|
29
|
+
|
13
30
|
|
14
31
|
def copy( src, dest )
|
15
|
-
logger.debug "fetch
|
32
|
+
logger.debug "fetch - copy src: #{src} to dest: #{dest}"
|
33
|
+
|
34
|
+
response = get_response( src )
|
35
|
+
|
36
|
+
# on error return; do NOT copy file; sorry
|
37
|
+
return if response.nil?
|
16
38
|
|
39
|
+
# check for content type; use 'wb' for images
|
40
|
+
if response.content_type =~ /image/
|
41
|
+
logger.debug ' switching to binary'
|
42
|
+
flags = 'wb'
|
43
|
+
else
|
44
|
+
flags = 'w'
|
45
|
+
end
|
46
|
+
|
47
|
+
File.open( dest, flags ) do |f|
|
48
|
+
f.write( response.body )
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
|
53
|
+
## todo: add file protocol
|
54
|
+
|
55
|
+
def get_response( src )
|
17
56
|
uri = URI.parse( src )
|
18
57
|
|
19
58
|
# new code: honor proxy env variable HTTP_PROXY
|
@@ -72,25 +111,15 @@ module Fetcher
|
|
72
111
|
else
|
73
112
|
msg = "#{response.code} #{response.message}"
|
74
113
|
puts "*** error: #{msg}"
|
75
|
-
return
|
114
|
+
return nil # todo: throw StandardException?
|
76
115
|
end
|
77
116
|
end
|
78
117
|
|
79
118
|
logger.debug " content_type: #{response.content_type}, content_length: #{response.content_length}"
|
80
|
-
|
81
|
-
#
|
82
|
-
if response.content_type =~ /image/
|
83
|
-
logger.debug ' switching to binary'
|
84
|
-
flags = 'wb'
|
85
|
-
else
|
86
|
-
flags = 'w'
|
87
|
-
end
|
88
|
-
|
89
|
-
File.open( dest, flags ) do |f|
|
90
|
-
f.write( response.body )
|
91
|
-
end
|
119
|
+
|
120
|
+
response # return respone or nil if error
|
92
121
|
end # method copy
|
93
122
|
|
94
123
|
end # class Worker
|
95
124
|
|
96
|
-
end # module Fetcher
|
125
|
+
end # module Fetcher
|
metadata
CHANGED
@@ -1,61 +1,65 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: fetcher
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
prerelease:
|
6
|
-
segments:
|
7
|
-
- 0
|
8
|
-
- 1
|
9
|
-
- 0
|
10
|
-
version: 0.1.0
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.0
|
11
5
|
platform: ruby
|
12
|
-
authors:
|
6
|
+
authors:
|
13
7
|
- Gerald Bauer
|
14
8
|
autorequire:
|
15
9
|
bindir: bin
|
16
10
|
cert_chain: []
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
11
|
+
date: 2013-08-26 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: logutils
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.6'
|
20
|
+
type: :runtime
|
22
21
|
prerelease: false
|
23
|
-
|
24
|
-
|
25
|
-
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ! '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.6'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rdoc
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
26
31
|
- - ~>
|
27
|
-
- !ruby/object:Gem::Version
|
28
|
-
|
29
|
-
segments:
|
30
|
-
- 3
|
31
|
-
- 10
|
32
|
-
version: "3.10"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '4.0'
|
33
34
|
type: :development
|
34
|
-
version_requirements: *id001
|
35
|
-
- !ruby/object:Gem::Dependency
|
36
|
-
name: hoe
|
37
35
|
prerelease: false
|
38
|
-
|
39
|
-
|
40
|
-
requirements:
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
41
38
|
- - ~>
|
42
|
-
- !ruby/object:Gem::Version
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '4.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: hoe
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.6'
|
48
48
|
type: :development
|
49
|
-
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.6'
|
50
55
|
description: fetcher - Fetch Text Documents or Binary Blobs via HTTP, HTTPS
|
51
56
|
email: webslideshow@googlegroups.com
|
52
|
-
executables:
|
57
|
+
executables:
|
53
58
|
- fetch
|
54
59
|
extensions: []
|
55
|
-
|
56
|
-
extra_rdoc_files:
|
60
|
+
extra_rdoc_files:
|
57
61
|
- Manifest.txt
|
58
|
-
files:
|
62
|
+
files:
|
59
63
|
- History.markdown
|
60
64
|
- Manifest.txt
|
61
65
|
- README.markdown
|
@@ -64,39 +68,30 @@ files:
|
|
64
68
|
- lib/fetcher.rb
|
65
69
|
- lib/fetcher/runner.rb
|
66
70
|
- lib/fetcher/worker.rb
|
67
|
-
homepage:
|
68
|
-
licenses:
|
69
|
-
|
71
|
+
homepage: https://github.com/geraldb/fetcher
|
72
|
+
licenses:
|
73
|
+
- Public Domain
|
74
|
+
metadata: {}
|
70
75
|
post_install_message:
|
71
|
-
rdoc_options:
|
76
|
+
rdoc_options:
|
72
77
|
- --main
|
73
78
|
- README.markdown
|
74
|
-
require_paths:
|
79
|
+
require_paths:
|
75
80
|
- lib
|
76
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
none: false
|
87
|
-
requirements:
|
88
|
-
- - ">="
|
89
|
-
- !ruby/object:Gem::Version
|
90
|
-
hash: 3
|
91
|
-
segments:
|
92
|
-
- 0
|
93
|
-
version: "0"
|
81
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: 1.9.2
|
86
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ! '>='
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '0'
|
94
91
|
requirements: []
|
95
|
-
|
96
92
|
rubyforge_project: fetcher
|
97
|
-
rubygems_version:
|
93
|
+
rubygems_version: 2.0.6
|
98
94
|
signing_key:
|
99
|
-
specification_version:
|
95
|
+
specification_version: 4
|
100
96
|
summary: fetcher - Fetch Text Documents or Binary Blobs via HTTP, HTTPS
|
101
97
|
test_files: []
|
102
|
-
|