fm 0.0.2 → 0.0.3.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/COPYING +1 -1
- data/bin/fm +46 -1
- data/fm.gemspec +10 -10
- data/lib/fm/db.rb +26 -0
- data/lib/fm/file.rb +23 -0
- data/lib/fm/find.rb +133 -0
- data/lib/fm/index.rb +77 -24
- data/lib/fm/match.rb +23 -0
- data/lib/fm/version.rb +24 -1
- data/lib/fm.rb +24 -0
- data/test/index/test_cmd_index.rb +23 -0
- data/test/test_fm.rb +23 -0
- metadata +12 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c4b7a3d871b0e74198e25ed52f8201aeec1447d7
|
4
|
+
data.tar.gz: 91ed6d0a626b604827f91ec956ecf1af5257dc5a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b2f77a2ef3b9bc7e0c575392da9c7a48a5e20dbe386e5239e0f0c8cc95d7e64742101805e875ac7f4cf63cd059eec3097b2982b6d902d6307a73e7839640b6c7
|
7
|
+
data.tar.gz: 031bdf753e0b43b4a98f7f05179a9d16d43364f18bf58a39de1d70d7c80c13a22e780209284d1859b3a1851e1c8dccfc595d9be2bf50111df35e28d6ab428e9e
|
data/COPYING
CHANGED
data/bin/fm
CHANGED
@@ -1,5 +1,29 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
|
4
|
+
# The MIT License (MIT)
|
5
|
+
#
|
6
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
7
|
+
#
|
8
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
9
|
+
# of this software and associated documentation files (the "Software"), to deal
|
10
|
+
# in the Software without restriction, including without limitation the rights
|
11
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
12
|
+
# copies of the Software, and to permit persons to whom the Software is
|
13
|
+
# furnished to do so, subject to the following conditions:
|
14
|
+
#
|
15
|
+
# The above copyright notice and this permission notice shall be included in
|
16
|
+
# all copies or substantial portions of the Software.
|
17
|
+
#
|
18
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
19
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
20
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
21
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
22
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
23
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
24
|
+
# THE SOFTWARE.
|
25
|
+
|
26
|
+
|
3
27
|
require 'fm'
|
4
28
|
|
5
29
|
|
@@ -10,7 +34,8 @@ fm #{FM::VERSION}
|
|
10
34
|
usage: fm <command> [args]
|
11
35
|
|
12
36
|
Available fm commands:
|
13
|
-
index
|
37
|
+
index Create or update the index on the folder.
|
38
|
+
find Find files on the folder from the DB.
|
14
39
|
EOS
|
15
40
|
end
|
16
41
|
|
@@ -22,8 +47,28 @@ end
|
|
22
47
|
|
23
48
|
cmd = ARGV.shift
|
24
49
|
case cmd
|
50
|
+
when "find"
|
51
|
+
FM.cmd_find(ARGV.clone)
|
25
52
|
when "index"
|
26
53
|
FM.cmd_index(ARGV.clone)
|
54
|
+
when "update"
|
55
|
+
when "init"
|
56
|
+
when "config"
|
57
|
+
when "storage"
|
58
|
+
when "status"
|
59
|
+
when "log"
|
60
|
+
when "version"
|
61
|
+
when "help"
|
62
|
+
when "rm"
|
63
|
+
when "trash"
|
64
|
+
when "clean"
|
65
|
+
when "recover"
|
66
|
+
when "grep"
|
67
|
+
when "locate"
|
68
|
+
when "diff"
|
69
|
+
when "search"
|
70
|
+
when "tag"
|
71
|
+
when "active"
|
27
72
|
else
|
28
73
|
$stderr << "fm: '#{cmd}' is not a fm command.\n"
|
29
74
|
exit
|
data/fm.gemspec
CHANGED
@@ -4,23 +4,23 @@ require 'fm/version'
|
|
4
4
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name
|
8
|
-
spec.version
|
9
|
-
spec.summary
|
10
|
-
spec.description
|
7
|
+
spec.name = 'fm'
|
8
|
+
spec.version = FM::VERSION
|
9
|
+
spec.summary = %q{Compute and record file checksum. Verify file content integrity using checksum. Identify duplicated files or backup locations.}
|
10
|
+
spec.description = %q{Manage files scattered over multiple physical storages. Using file checksum, it's possible to verify content integrity of your backups, look up locations of the backups at various storages.}
|
11
11
|
|
12
|
-
spec.authors
|
13
|
-
spec.email
|
14
|
-
spec.homepage
|
15
|
-
spec.licenses
|
16
|
-
|
17
|
-
spec.required_ruby_version = '>= 2.0.0'
|
12
|
+
spec.authors = ['ShinYee']
|
13
|
+
spec.email = ['shinyee@speedgocomputing.com']
|
14
|
+
spec.homepage = 'http://github.com/xman/ruby-fm'
|
15
|
+
spec.licenses = ['MIT']
|
18
16
|
|
19
17
|
spec.files = `git ls-files`.split($/)
|
20
18
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
19
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
22
20
|
spec.require_paths = ["lib"]
|
23
21
|
|
22
|
+
spec.required_ruby_version = '>= 2.0.0'
|
23
|
+
|
24
24
|
spec.add_development_dependency "bundler", "~> 1.9"
|
25
25
|
spec.add_development_dependency "rake", "~> 10.1"
|
26
26
|
end
|
data/lib/fm/db.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
|
24
|
+
module FM
|
25
|
+
DBFILE = ENV['HOME'] + '/.fm/fdata.db'
|
26
|
+
end
|
data/lib/fm/file.rb
CHANGED
@@ -1,3 +1,26 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
|
1
24
|
module FM
|
2
25
|
|
3
26
|
class FMFile
|
data/lib/fm/find.rb
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
|
24
|
+
require 'digest/md5'
|
25
|
+
require 'fileutils'
|
26
|
+
require 'optparse'
|
27
|
+
require 'yaml'
|
28
|
+
|
29
|
+
require 'fm/db'
|
30
|
+
require 'fm/file'
|
31
|
+
require 'fm/match'
|
32
|
+
|
33
|
+
|
34
|
+
module FM
|
35
|
+
|
36
|
+
def cmd_find(argv)
|
37
|
+
|
38
|
+
#### Parse options.
|
39
|
+
|
40
|
+
banner = "usage: fm find [options] <PATH>"
|
41
|
+
options = {
|
42
|
+
dbfile: DBFILE,
|
43
|
+
}
|
44
|
+
OptionParser.new do |opts|
|
45
|
+
opts.banner = banner
|
46
|
+
|
47
|
+
opts.on "-h", "--help", "Show this message." do
|
48
|
+
puts opts
|
49
|
+
exit
|
50
|
+
end
|
51
|
+
end.parse!(argv)
|
52
|
+
|
53
|
+
if argv.size != 1
|
54
|
+
$stderr << "#{banner}\n"
|
55
|
+
exit
|
56
|
+
end
|
57
|
+
|
58
|
+
|
59
|
+
#### Load DB file.
|
60
|
+
|
61
|
+
puts "Loading DB #{options[:dbfile]} ..."
|
62
|
+
t0 = Time.now
|
63
|
+
h = {}
|
64
|
+
if File.exists?(options[:dbfile])
|
65
|
+
obj = YAML::load(File.read(options[:dbfile]))
|
66
|
+
if obj.is_a?(Hash)
|
67
|
+
h = obj
|
68
|
+
end
|
69
|
+
end
|
70
|
+
t1 = Time.now
|
71
|
+
puts "Loaded DB in #{(t1-t0).round(2)}s."
|
72
|
+
|
73
|
+
|
74
|
+
####
|
75
|
+
|
76
|
+
indexpath = argv[0]
|
77
|
+
indexabspath = File.realpath(indexpath)
|
78
|
+
earr = [] # List of files exists in DB.
|
79
|
+
nearr = [] # List of files not exists in DB.
|
80
|
+
nfailfiles = 0
|
81
|
+
t0 = Time.now
|
82
|
+
|
83
|
+
|
84
|
+
#### Find files in the path recursively.
|
85
|
+
|
86
|
+
# FIXME: Configurable file patterns to be excluded.
|
87
|
+
for fpath in Dir.glob("#{indexpath}/**/*", File::FNM_DOTMATCH).select { |e| e.force_encoding("binary"); File.ftype(e) == "file" && has_folder?(".git", e) == false && has_folder?(".hg", e) == false }
|
88
|
+
begin
|
89
|
+
fpath = File.realpath(fpath).force_encoding("binary")
|
90
|
+
ftype = File.ftype(fpath)
|
91
|
+
fsize = File.size(fpath)
|
92
|
+
fmtime = File.mtime(fpath)
|
93
|
+
# last index time.
|
94
|
+
# storage type: raw or compressed?
|
95
|
+
# content type: text or binary?
|
96
|
+
# hash type.
|
97
|
+
# hash value.
|
98
|
+
digest = Digest::MD5.hexdigest(File.read(fpath))
|
99
|
+
fmfile = FMFile.new(fsize, fpath, digest, fmtime)
|
100
|
+
|
101
|
+
if h[fsize] && h[fsize][digest]
|
102
|
+
earr.push(fmfile)
|
103
|
+
else
|
104
|
+
nearr.push(fmfile)
|
105
|
+
end
|
106
|
+
|
107
|
+
rescue StandardError => e
|
108
|
+
nfailfiles += 1
|
109
|
+
puts "[FAIL]: file=\"#{fpath}\" #{e.message}"
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
#### Report.
|
115
|
+
|
116
|
+
nearr.sort! { |a,b| a.path <=> b.path }
|
117
|
+
|
118
|
+
puts "Number of files found: #{earr.size}"
|
119
|
+
|
120
|
+
puts "Number of files not found: #{nearr.size}"
|
121
|
+
nearr.each do |f|
|
122
|
+
puts " #{f.path}"
|
123
|
+
end
|
124
|
+
|
125
|
+
t1 = Time.now
|
126
|
+
puts "Find completed in #{(t1-t0).round(2)}s."
|
127
|
+
|
128
|
+
end
|
129
|
+
|
130
|
+
|
131
|
+
module_function :cmd_find
|
132
|
+
|
133
|
+
end
|
data/lib/fm/index.rb
CHANGED
@@ -1,18 +1,40 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
|
1
24
|
require 'digest/md5'
|
2
25
|
require 'fileutils'
|
3
26
|
require 'optparse'
|
4
27
|
require 'yaml'
|
5
28
|
|
29
|
+
require 'fm/db'
|
6
30
|
require 'fm/file'
|
7
31
|
require 'fm/match'
|
8
32
|
|
9
33
|
|
10
|
-
DBFILE = ENV['HOME'] + '/.fm/fdata.db'
|
11
|
-
|
12
|
-
|
13
34
|
module FM
|
14
35
|
|
15
36
|
def cmd_index(argv)
|
37
|
+
#### Parse options.
|
16
38
|
|
17
39
|
banner = "usage: fm index [options] <PATH>"
|
18
40
|
options = {
|
@@ -33,33 +55,43 @@ module FM
|
|
33
55
|
end
|
34
56
|
|
35
57
|
|
58
|
+
#### Load DB file.
|
59
|
+
|
36
60
|
puts "Loading DB #{options[:dbfile]} ..."
|
37
61
|
t0 = Time.now
|
38
62
|
h = {}
|
39
|
-
needupdate = true
|
40
63
|
if File.exists?(options[:dbfile])
|
41
64
|
obj = YAML::load(File.read(options[:dbfile]))
|
42
65
|
if obj.is_a?(Hash)
|
43
66
|
h = obj
|
44
|
-
needupdate = false
|
45
67
|
end
|
46
68
|
end
|
47
69
|
t1 = Time.now
|
48
70
|
puts "Loaded DB in #{(t1-t0).round(2)}s."
|
49
71
|
|
72
|
+
|
73
|
+
####
|
74
|
+
|
50
75
|
indexpath = argv[0]
|
76
|
+
indexabspath = File.realpath(indexpath)
|
51
77
|
nnewfiles = 0
|
52
78
|
nskipfiles = 0
|
53
79
|
ndupfiles = 0
|
54
80
|
nfailfiles = 0
|
81
|
+
nupdatefiles = 0
|
82
|
+
nremovefiles = 0
|
55
83
|
duplist = {}
|
84
|
+
needupdate = false
|
56
85
|
puts "Indexing #{indexpath} ..."
|
57
86
|
t0 = Time.now
|
58
87
|
|
88
|
+
|
89
|
+
#### Remove non-existing files and update modified files in the index path from the DB.
|
90
|
+
|
59
91
|
dlist = []
|
60
92
|
h.each do |skey, svalue|
|
61
93
|
h[skey].each do |dkey, dvalue|
|
62
|
-
h[skey][dkey].each do |f|
|
94
|
+
h[skey][dkey].select { |s| s.path.start_with?(indexabspath) }.each do |f|
|
63
95
|
fsize = -1
|
64
96
|
digest = ""
|
65
97
|
fe = File.exists?(f.path)
|
@@ -68,39 +100,46 @@ module FM
|
|
68
100
|
next
|
69
101
|
end
|
70
102
|
|
103
|
+
dlist.push(f)
|
104
|
+
|
71
105
|
if fe
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
106
|
+
nupdatefiles += 1
|
107
|
+
puts "[UPD]: #{f.path} #{f.digest}"
|
108
|
+
else
|
109
|
+
nremovefiles += 1
|
110
|
+
puts "[RMV]: #{f.path} #{f.digest}"
|
77
111
|
end
|
78
112
|
end
|
79
113
|
end
|
80
114
|
end
|
81
115
|
|
82
116
|
dlist.each do |f|
|
83
|
-
fsize
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
if h[fsize][digest].size == 0
|
89
|
-
h[fsize].delete(digest)
|
90
|
-
if h[fsize].size == 0
|
91
|
-
h.delete(fsize)
|
117
|
+
h[f.fsize][f.digest].delete_if { |v| v.path == f.path }
|
118
|
+
if h[f.fsize][f.digest].size == 0
|
119
|
+
h[f.fsize].delete(f.digest)
|
120
|
+
if h[f.fsize].size == 0
|
121
|
+
h.delete(f.fsize)
|
92
122
|
end
|
93
123
|
end
|
94
124
|
|
95
125
|
needupdate = true
|
96
126
|
end
|
97
|
-
dlist = nil
|
98
127
|
|
128
|
+
|
129
|
+
#### Index the path recursively.
|
130
|
+
|
131
|
+
# FIXME: Configurable file patterns to be excluded.
|
99
132
|
for fpath in Dir.glob("#{indexpath}/**/*", File::FNM_DOTMATCH).select { |e| e.force_encoding("binary"); File.ftype(e) == "file" && has_folder?(".git", e) == false && has_folder?(".hg", e) == false }
|
100
133
|
begin
|
101
134
|
fpath = File.realpath(fpath).force_encoding("binary")
|
135
|
+
ftype = File.ftype(fpath)
|
102
136
|
fsize = File.size(fpath)
|
103
137
|
fmtime = File.mtime(fpath)
|
138
|
+
# last index time.
|
139
|
+
# storage type: raw or compressed?
|
140
|
+
# content type: text or binary?
|
141
|
+
# hash type.
|
142
|
+
# hash value.
|
104
143
|
|
105
144
|
if fsize == 0
|
106
145
|
nskipfiles += 1
|
@@ -108,7 +147,7 @@ module FM
|
|
108
147
|
next
|
109
148
|
end
|
110
149
|
|
111
|
-
#
|
150
|
+
# Lazily index the file.
|
112
151
|
if h[fsize].nil?
|
113
152
|
digest = Digest::MD5.hexdigest(File.read(fpath))
|
114
153
|
h[fsize] = { digest => [ FMFile.new(fsize, fpath, digest, fmtime) ] }
|
@@ -116,12 +155,17 @@ module FM
|
|
116
155
|
nnewfiles += 1
|
117
156
|
puts "[NEW]: #{fpath} #{digest}"
|
118
157
|
else
|
119
|
-
|
158
|
+
# Skip if the file had been indexed.
|
159
|
+
is_indexed = false
|
120
160
|
h[fsize].values.each do |v|
|
121
161
|
if v.any? { |f| f.mtime == fmtime && f.path == fpath }
|
122
|
-
|
162
|
+
is_indexed = true
|
163
|
+
break
|
123
164
|
end
|
124
165
|
end
|
166
|
+
if is_indexed
|
167
|
+
next
|
168
|
+
end
|
125
169
|
|
126
170
|
digest = Digest::MD5.hexdigest(File.read(fpath))
|
127
171
|
if h[fsize][digest].nil?
|
@@ -130,6 +174,10 @@ module FM
|
|
130
174
|
nnewfiles += 1
|
131
175
|
puts "[NEW]: #{fpath} #{digest}"
|
132
176
|
elsif h[fsize][digest].size == 1 && h[fsize][digest].first.path == fpath
|
177
|
+
h[fsize][digest] = [ FMFile.new(fsize, fpath, digest, fmtime) ]
|
178
|
+
needupdate = true
|
179
|
+
nupdatefiles += 1
|
180
|
+
puts "[UPD]: #{fpath} #{digest}"
|
133
181
|
else
|
134
182
|
ndupfiles += 1
|
135
183
|
if duplist[digest].nil?
|
@@ -156,12 +204,17 @@ module FM
|
|
156
204
|
end
|
157
205
|
end
|
158
206
|
|
207
|
+
|
208
|
+
#### Report and update DB.
|
209
|
+
|
159
210
|
t1 = Time.now
|
160
211
|
puts "Indexing completed in #{(t1-t0).round(2)}s."
|
161
212
|
|
162
213
|
puts "Indexed:"
|
163
214
|
puts " NEW: #{nnewfiles}"
|
164
215
|
puts " DUP: #{ndupfiles}"
|
216
|
+
puts " UPD: #{nupdatefiles}"
|
217
|
+
puts " RMV: #{nremovefiles}"
|
165
218
|
puts " SKIP: #{nskipfiles}"
|
166
219
|
puts " FAIL: #{nfailfiles}"
|
167
220
|
|
data/lib/fm/match.rb
CHANGED
@@ -1,3 +1,26 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
|
1
24
|
module FM
|
2
25
|
|
3
26
|
def has_folder?(folder, path)
|
data/lib/fm/version.rb
CHANGED
@@ -1,3 +1,26 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
|
1
24
|
module FM
|
2
|
-
VERSION = '0.0.
|
25
|
+
VERSION = '0.0.3.rc1'
|
3
26
|
end
|
data/lib/fm.rb
CHANGED
@@ -1,2 +1,26 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
|
1
24
|
require 'fm/version'
|
25
|
+
require 'fm/find'
|
2
26
|
require 'fm/index'
|
@@ -1,3 +1,26 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
|
1
24
|
require 'test/unit'
|
2
25
|
|
3
26
|
require 'fm'
|
data/test/test_fm.rb
CHANGED
@@ -1,3 +1,26 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2015-2017 SpeedGo Computing
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
|
1
24
|
require 'test/unit'
|
2
25
|
|
3
26
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fm
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- ShinYee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-02-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -38,8 +38,9 @@ dependencies:
|
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '10.1'
|
41
|
-
description:
|
42
|
-
|
41
|
+
description: Manage files scattered over multiple physical storages. Using file checksum,
|
42
|
+
it's possible to verify content integrity of your backups, look up locations of
|
43
|
+
the backups at various storages.
|
43
44
|
email:
|
44
45
|
- shinyee@speedgocomputing.com
|
45
46
|
executables:
|
@@ -54,7 +55,9 @@ files:
|
|
54
55
|
- bin/fm
|
55
56
|
- fm.gemspec
|
56
57
|
- lib/fm.rb
|
58
|
+
- lib/fm/db.rb
|
57
59
|
- lib/fm/file.rb
|
60
|
+
- lib/fm/find.rb
|
58
61
|
- lib/fm/index.rb
|
59
62
|
- lib/fm/match.rb
|
60
63
|
- lib/fm/version.rb
|
@@ -75,15 +78,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
75
78
|
version: 2.0.0
|
76
79
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
77
80
|
requirements:
|
78
|
-
- - "
|
81
|
+
- - ">"
|
79
82
|
- !ruby/object:Gem::Version
|
80
|
-
version:
|
83
|
+
version: 1.3.1
|
81
84
|
requirements: []
|
82
85
|
rubyforge_project:
|
83
|
-
rubygems_version: 2.
|
86
|
+
rubygems_version: 2.6.8
|
84
87
|
signing_key:
|
85
88
|
specification_version: 4
|
86
|
-
summary:
|
89
|
+
summary: Compute and record file checksum. Verify file content integrity using checksum.
|
90
|
+
Identify duplicated files or backup locations.
|
87
91
|
test_files:
|
88
92
|
- test/index/test_cmd_index.rb
|
89
93
|
- test/test_fm.rb
|