crawl_kit 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/crawl_kit.gemspec +2 -2
- data/lib/crawl_kit/record/helpers.rb +2 -2
- data/lib/crawl_kit/s3.rb +1 -1
- metadata +5 -5
data/crawl_kit.gemspec
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'crawl_kit'
|
|
3
|
-
s.version = '0.0.
|
|
3
|
+
s.version = '0.0.2'
|
|
4
4
|
s.date = '2011-12-08'
|
|
5
5
|
s.summary = "A collection of serivce interfaces and models to use with the GoTime crawling infrastructure."
|
|
6
6
|
s.author = "Austin Cargol"
|
|
7
|
-
s.email = '
|
|
7
|
+
s.email = 'austin@gotime.com'
|
|
8
8
|
|
|
9
9
|
s.files = `git ls-files`.split("\n")
|
|
10
10
|
s.test_files = `git ls-files -- {test}/*`.split("\n")
|
|
@@ -17,7 +17,7 @@ module CrawlKit
|
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
def hashed_uri
|
|
20
|
-
self.class.
|
|
20
|
+
self.class.hash_uri(uri)
|
|
21
21
|
end
|
|
22
22
|
alias :crawl_id :hashed_uri
|
|
23
23
|
|
|
@@ -26,7 +26,7 @@ module CrawlKit
|
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
def page
|
|
29
|
-
CrawlKit::S3.get_file(crawl_id)
|
|
29
|
+
fetched? ? CrawlKit::S3.get_file(crawl_id) : nil
|
|
30
30
|
end
|
|
31
31
|
|
|
32
32
|
def previous_version
|
data/lib/crawl_kit/s3.rb
CHANGED
|
@@ -33,7 +33,7 @@ module CrawlKit
|
|
|
33
33
|
options.merge!(content_type: 'text/html', content_encoding: 'deflate', storage_class: :reduced_redundancy)
|
|
34
34
|
object = get_object(key)
|
|
35
35
|
object.write(Zlib::Deflate.deflate(body), options)
|
|
36
|
-
end
|
|
36
|
+
end
|
|
37
37
|
|
|
38
38
|
def get_versions(hashed_uri)
|
|
39
39
|
s3_object = get_object(hashed_uri)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawl_kit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.2
|
|
5
5
|
prerelease:
|
|
6
6
|
platform: ruby
|
|
7
7
|
authors:
|
|
@@ -13,7 +13,7 @@ date: 2011-12-08 00:00:00.000000000Z
|
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: aws-sdk
|
|
16
|
-
requirement: &
|
|
16
|
+
requirement: &70324126127720 !ruby/object:Gem::Requirement
|
|
17
17
|
none: false
|
|
18
18
|
requirements:
|
|
19
19
|
- - ! '>='
|
|
@@ -21,9 +21,9 @@ dependencies:
|
|
|
21
21
|
version: 1.2.3
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
|
-
version_requirements: *
|
|
24
|
+
version_requirements: *70324126127720
|
|
25
25
|
description:
|
|
26
|
-
email:
|
|
26
|
+
email: austin@gotime.com
|
|
27
27
|
executables: []
|
|
28
28
|
extensions: []
|
|
29
29
|
extra_rdoc_files: []
|
|
@@ -59,7 +59,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
59
59
|
version: '0'
|
|
60
60
|
requirements: []
|
|
61
61
|
rubyforge_project:
|
|
62
|
-
rubygems_version: 1.8.
|
|
62
|
+
rubygems_version: 1.8.15
|
|
63
63
|
signing_key:
|
|
64
64
|
specification_version: 3
|
|
65
65
|
summary: A collection of serivce interfaces and models to use with the GoTime crawling
|