filey-diff 1.1.1 → 1.1.2
Sign up to get free protection for your applications and to get access to all the features.
- data/changelog.md +4 -0
- data/filey-diff.gemspec +1 -1
- data/lib/filey-diff/data-sources/aws_sdk_s3.rb +41 -17
- data/spec/data_sources_spec.rb +57 -29
- metadata +19 -7
- checksums.yaml +0 -7
data/changelog.md
CHANGED
data/filey-diff.gemspec
CHANGED
@@ -18,32 +18,56 @@ module Filey
|
|
18
18
|
end
|
19
19
|
|
20
20
|
if (s3_object.head[:content_encoding] == "gzip")
|
21
|
-
last_modified, md5 =
|
22
|
-
|
21
|
+
last_modified, md5 = last_modified_and_md5_from_gzipped(
|
22
|
+
s3_object, path
|
23
|
+
)
|
23
24
|
else
|
24
|
-
last_modified = s3_object
|
25
|
-
md5 = s3_object.etag.gsub(/"/, '').split('-',2).first
|
25
|
+
last_modified, md5 = last_modified_and_md5(s3_object)
|
26
26
|
end
|
27
27
|
|
28
28
|
normalised_path = "./#{path}"
|
29
|
-
Filey.new(
|
30
|
-
|
31
|
-
|
32
|
-
|
29
|
+
Filey.new(
|
30
|
+
normalised_path,
|
31
|
+
name,
|
32
|
+
last_modified,
|
33
|
+
md5
|
34
|
+
)
|
33
35
|
}
|
34
36
|
end
|
35
37
|
|
36
|
-
def
|
37
|
-
|
38
|
-
|
39
|
-
tempfile.close
|
40
|
-
|
41
|
-
gz = Zlib::GzipReader.open(tempfile.path)
|
42
|
-
last_modified = gz.mtime
|
43
|
-
md5 = Digest::MD5.hexdigest(gz.read)
|
44
|
-
gz.close
|
38
|
+
def last_modified_and_md5(s3_object)
|
39
|
+
last_modified = s3_object.last_modified
|
40
|
+
md5 = s3_object.etag.gsub(/"/, '').split('-',2).first
|
45
41
|
[last_modified, md5]
|
46
42
|
end
|
43
|
+
|
44
|
+
def last_modified_and_md5_from_gzipped(s3_object, path)
|
45
|
+
s3_object_contents = s3_object.read
|
46
|
+
if is_already_decoded s3_object_contents
|
47
|
+
md5 = Digest::MD5.hexdigest(s3_object_contents)
|
48
|
+
[s3_object.last_modified, md5]
|
49
|
+
else
|
50
|
+
tempfile = Tempfile.new(File.basename(path))
|
51
|
+
tempfile.write s3_object_contents
|
52
|
+
tempfile.close
|
53
|
+
|
54
|
+
gz = Zlib::GzipReader.open(tempfile.path)
|
55
|
+
last_modified = gz.mtime
|
56
|
+
md5 = Digest::MD5.hexdigest(gz.read)
|
57
|
+
gz.close
|
58
|
+
[last_modified, md5]
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
# Check if the two first bytes are the magic numbers of the gzip format.
|
63
|
+
# We double-check here because Ruby 2.0.0 decodes gzip'ed HTTP responses
|
64
|
+
# automatically. As a result, we get decoded gzip data from the
|
65
|
+
# s3_object#read method when we are using Ruby 2.0.0, and encoded data
|
66
|
+
# when we are using previous versions of Ruby.
|
67
|
+
def is_already_decoded(gzipped_on_server)
|
68
|
+
is_gzipped = gzipped_on_server.bytes.to_a[0] == 0x1f && gzipped_on_server.bytes.to_a[1] == 0x8b
|
69
|
+
is_gzipped == false
|
70
|
+
end
|
47
71
|
end
|
48
72
|
end
|
49
73
|
end
|
data/spec/data_sources_spec.rb
CHANGED
@@ -61,40 +61,68 @@ describe Filey::DataSources::AwsSdkS3 do
|
|
61
61
|
|
62
62
|
it_should_behave_like "a data source", s3_bucket
|
63
63
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
# to_i adjusts the original file to match
|
73
|
-
filey.last_modified.to_i.should eq(original_object[:mtime].to_i)
|
74
|
-
end
|
64
|
+
context 'gzip' do
|
65
|
+
let(:gzip_tempfile_and_path) {
|
66
|
+
original_object = objects.first
|
67
|
+
[
|
68
|
+
gzip_into_tmp_file(original_object[:content], original_object[:mtime]),
|
69
|
+
original_object[:path]
|
70
|
+
]
|
71
|
+
}
|
75
72
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
73
|
+
let(:data_source_with_one_gzipped_object) {
|
74
|
+
file, path = gzip_tempfile_and_path
|
75
|
+
file.open
|
76
|
+
data_source = Filey::DataSources::AwsSdkS3.new(S3Bucket.new([S3Object.new(
|
77
|
+
path,
|
78
|
+
file.mtime,
|
79
|
+
file.read,
|
80
|
+
{ :content_encoding => 'gzip' }
|
81
|
+
)]))
|
82
|
+
}
|
85
83
|
|
86
|
-
|
87
|
-
|
84
|
+
it 'provides the original md5/mtime of a gzipped file' do
|
85
|
+
filey = data_source_with_one_gzipped_object.get_fileys[0]
|
86
|
+
filey.md5.should eq(Digest::MD5.hexdigest(objects.first[:content]))
|
87
|
+
# GzipWriter seems to cut off fractions of a second,
|
88
|
+
# to_i adjusts the original file to match
|
89
|
+
filey.last_modified.to_i.should eq(objects.first[:mtime].to_i)
|
90
|
+
end
|
88
91
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
+
context 'working with Ruby 2.0.0 automatic decoding of gzipped HTTP responses' do
|
93
|
+
let(:data_source_with_decoded_object_and_gzip_header) {
|
94
|
+
data_source = Filey::DataSources::AwsSdkS3.new(S3Bucket.new([S3Object.new(
|
95
|
+
objects.first[:path],
|
96
|
+
objects.first[:mtime],
|
97
|
+
objects.first[:content],
|
98
|
+
{ :content_encoding => 'gzip' }
|
99
|
+
)]))
|
100
|
+
}
|
101
|
+
|
102
|
+
it 'detects the case where the gzipped data has already been decoded' do
|
103
|
+
filey = data_source_with_decoded_object_and_gzip_header.get_fileys.first
|
104
|
+
filey.last_modified.should eq(objects.first[:mtime])
|
105
|
+
end
|
106
|
+
|
107
|
+
it 'returns the md5 of the gzip-decoded content' do
|
108
|
+
filey = data_source_with_decoded_object_and_gzip_header.get_fileys.first
|
109
|
+
filey.md5.should eq(Digest::MD5.hexdigest(objects.first[:content]))
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def gzip_into_tmp_file(content, mtime)
|
114
|
+
tempfile = Tempfile.new("temp")
|
92
115
|
|
93
|
-
|
94
|
-
|
116
|
+
gz = Zlib::GzipWriter.open(tempfile.path, Zlib::BEST_COMPRESSION, Zlib::DEFAULT_STRATEGY)
|
117
|
+
gz.mtime = mtime
|
118
|
+
gz.write content
|
95
119
|
|
96
|
-
|
97
|
-
|
120
|
+
gz.flush
|
121
|
+
tempfile.flush
|
122
|
+
|
123
|
+
gz.close
|
124
|
+
tempfile
|
125
|
+
end
|
98
126
|
end
|
99
127
|
end
|
100
128
|
|
metadata
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filey-diff
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.2
|
5
|
+
prerelease:
|
5
6
|
platform: ruby
|
6
7
|
authors:
|
7
8
|
- Lauri Lehmijoki
|
@@ -13,6 +14,7 @@ dependencies:
|
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: rake
|
15
16
|
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
16
18
|
requirements:
|
17
19
|
- - ~>
|
18
20
|
- !ruby/object:Gem::Version
|
@@ -20,6 +22,7 @@ dependencies:
|
|
20
22
|
type: :development
|
21
23
|
prerelease: false
|
22
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
23
26
|
requirements:
|
24
27
|
- - ~>
|
25
28
|
- !ruby/object:Gem::Version
|
@@ -27,6 +30,7 @@ dependencies:
|
|
27
30
|
- !ruby/object:Gem::Dependency
|
28
31
|
name: rspec
|
29
32
|
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
30
34
|
requirements:
|
31
35
|
- - ~>
|
32
36
|
- !ruby/object:Gem::Version
|
@@ -34,11 +38,12 @@ dependencies:
|
|
34
38
|
type: :development
|
35
39
|
prerelease: false
|
36
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
37
42
|
requirements:
|
38
43
|
- - ~>
|
39
44
|
- !ruby/object:Gem::Version
|
40
45
|
version: '2.11'
|
41
|
-
description: "\n Find missing or outdated files.\n For example, compare your
|
46
|
+
description: ! "\n Find missing or outdated files.\n For example, compare your
|
42
47
|
local file system to an AWS S3 bucket.\n "
|
43
48
|
email: lauri.lehmijoki@iki.fi
|
44
49
|
executables: []
|
@@ -64,26 +69,33 @@ files:
|
|
64
69
|
- spec/spec_helper.rb
|
65
70
|
homepage: http://github.com/laurilehmijoki/filey-diff
|
66
71
|
licenses: []
|
67
|
-
metadata: {}
|
68
72
|
post_install_message:
|
69
73
|
rdoc_options: []
|
70
74
|
require_paths:
|
71
75
|
- lib
|
72
76
|
required_ruby_version: !ruby/object:Gem::Requirement
|
77
|
+
none: false
|
73
78
|
requirements:
|
74
|
-
- - '>='
|
79
|
+
- - ! '>='
|
75
80
|
- !ruby/object:Gem::Version
|
76
81
|
version: '0'
|
82
|
+
segments:
|
83
|
+
- 0
|
84
|
+
hash: 4447757573038540080
|
77
85
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
86
|
+
none: false
|
78
87
|
requirements:
|
79
|
-
- - '>='
|
88
|
+
- - ! '>='
|
80
89
|
- !ruby/object:Gem::Version
|
81
90
|
version: '0'
|
91
|
+
segments:
|
92
|
+
- 0
|
93
|
+
hash: 4447757573038540080
|
82
94
|
requirements: []
|
83
95
|
rubyforge_project:
|
84
|
-
rubygems_version:
|
96
|
+
rubygems_version: 1.8.25
|
85
97
|
signing_key:
|
86
|
-
specification_version:
|
98
|
+
specification_version: 3
|
87
99
|
summary: Compare two data sources that contain file-like objects
|
88
100
|
test_files:
|
89
101
|
- spec/comparison_spec.rb
|
checksums.yaml
DELETED
@@ -1,7 +0,0 @@
|
|
1
|
-
---
|
2
|
-
SHA1:
|
3
|
-
metadata.gz: 226c6e3d55d656ea58bb65e10021dcb561c9ad29
|
4
|
-
data.tar.gz: 5c0653b7916ecb2e1f27909512506f5b36d19d99
|
5
|
-
SHA512:
|
6
|
-
metadata.gz: 97e2fc29de0145379df08595789fa77c25d7f60daf12f60a17700a8bbff4638a50d2d3325e231e9461efaedc93c2f4d5459c4334011effa4de8aa39f88372d0c
|
7
|
-
data.tar.gz: 2ef1d709e749ee2c7be3b9d6a8c1c8f462b35b8f7982d44905a5e160199d6d76c361f5b34c30e07df148c258da86d7f8518af03b24840761ccd7aa98ff29a5c6
|