ki-repo 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,153 @@
1
+ # encoding: UTF-8
2
+
3
+ # Copyright 2012 Mikko Apo
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ module Ki
18
+ # Combines version's information from all different repositories
19
+ # @see Component
20
+ # @see Repository::Version
21
+ # @see Repository::Component
22
+ # @see VersionIterator
23
+ # @see RepositoryFinder
24
+ # @see FileFinder
25
+ # @see VersionMetadataFile
26
+ # @see VersionStatusFile
27
+ class Version
28
+ attr_chain :component, :require
29
+ attr_chain :name, :require
30
+ attr_chain :version_id, :require
31
+ attr_chain :metadata, -> { find_metadata }
32
+ attr_chain :binaries, -> { find_binaries }
33
+ attr_chain :finder, -> { component.finder }
34
+ attr_chain :versions, :require
35
+ attr_chain :statuses, -> { collect_statuses }
36
+
37
+ # finds first Repository::Version directory for this version that contains binaries
38
+ def find_binaries
39
+ finder.all_repositories.each do |package_root|
40
+ binary_dir = package_root.go(version_id)
41
+ if binary_dir.exists?
42
+ return binary_dir
43
+ end
44
+ end
45
+ nil
46
+ end
47
+
48
+ # finds first Repository::Version directory that contains metadata
49
+ def find_metadata
50
+ versions.each do |v|
51
+ m = v.metadata
52
+ if m.exists?
53
+ return m
54
+ end
55
+ end
56
+ nil
57
+ end
58
+
59
+ # collects all statuses related to this version
60
+ def collect_statuses
61
+ ret = []
62
+ versions.each do |v|
63
+ s = v.statuses
64
+ if s.exists?
65
+ v.statuses.each do |status|
66
+ ret << [status["key"], status["value"]]
67
+ end
68
+ end
69
+ end
70
+ ret
71
+ end
72
+
73
+ # finds all versions referenced by this version
74
+ def version_iterator
75
+ VersionIterator.new.version(self)
76
+ end
77
+
78
+ # finds files from this version (recursive)
79
+ def find_files(*file_patterns)
80
+ FileFinder.new.version(self).files(file_patterns)
81
+ end
82
+
83
+ def exists?
84
+ metadata || binaries
85
+ end
86
+
87
+ # Initializes a Version and Repository::Version for files non-imported files
88
+ # * works for testing and showing
89
+ def self.create_version(file, binary_directory=nil)
90
+ dir = File.dirname(file)
91
+ if dir == "."
92
+ dir = Dir.pwd
93
+ end
94
+ version = Version.new
95
+ repo_ver = Repository::Version.new(dir)
96
+ repo_ver.metadata = VersionMetadataFile.new(File.basename(file)).parent(repo_ver)
97
+ version.versions=[repo_ver]
98
+ if binary_directory
99
+ version.binaries = DirectoryBase.new(binary_directory)
100
+ end
101
+ version
102
+ end
103
+ end
104
+
105
+ # Combine's component's information from all different repositories
106
+ # @see Repository::Component
107
+ # @see RepositoryFinder
108
+ class Component
109
+ attr_chain :component_id, :require
110
+ # Package collector contains
111
+ attr_chain :finder, :require
112
+ attr_chain :versions, -> { find_versions }
113
+ attr_chain :status_info, -> { find_status_info }
114
+ attr_chain :components, :require
115
+
116
+ # Returns version list from first component which has a version list
117
+ def find_versions
118
+ components.each do |c|
119
+ version_list_file = c.versions
120
+ if version_list_file.exists?
121
+ return version_list_file
122
+ end
123
+ end
124
+ nil
125
+ end
126
+
127
+ # Returns Version which references all existing version directories
128
+ # @see Version
129
+ def version_by_id(version_str)
130
+ version_id = File.join(component_id, version_str)
131
+ finder.versions.cache(version_id) do
132
+ info_versions = components.map do |c|
133
+ Repository::Version.new(version_str).version_id(version_id).parent(c)
134
+ end
135
+ existing_versions = info_versions.select do |v|
136
+ v.exists?
137
+ end
138
+ Version.new.component(self).version_id(version_id).name(version_str).versions(existing_versions)
139
+ end
140
+ end
141
+
142
+ def find_status_info
143
+ ret = {}
144
+ components.each do |c|
145
+ si = c.status_info
146
+ if si.exists?
147
+ ret.merge!(si.cached_data)
148
+ end
149
+ end
150
+ ret
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,242 @@
1
+ # encoding: UTF-8
2
+
3
+ # Copyright 2012 Mikko Apo
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ module Ki
18
+ # Tests that a version is intact. Version can be in repository or as file.
19
+ # Checks that all files have correct hashes. If recursive is set to true, goes through all dependencies
20
+ # @see test_version
21
+ class VersionTester
22
+ attr_chain :ki_home, :require
23
+ attr_chain :finder, -> { ki_home.finder }
24
+ attr_chain :recursive, -> { true }
25
+ attr_chain :print, -> { false }
26
+ attr_chain :results, -> { Hash.new }
27
+
28
+ # Tests that a version is intact
29
+ # * test_version(version) expects a Version parameter
30
+ # @see VersionIterator
31
+ # @see RepositoryFinder
32
+ # @return [bool] returns true if there weren't any problems with the version
33
+ def test_version(root_version, &block)
34
+ all_ok = true
35
+ possible_hashes = KiCommand::KiExtensions.find!("/hashing")
36
+ # iterates through all versions
37
+ root_version.version_iterator.iterate_versions do |v|
38
+ binaries = v.binaries
39
+ metadata = v.metadata
40
+ metadata.cached_data
41
+ metadata.files.each do |file_hash|
42
+ file_path = file_hash["path"]
43
+ full_path = binaries.path(file_path)
44
+ issue = nil
45
+ if !File.exists?(full_path)
46
+ issue="missing"
47
+ elsif File.size(full_path) != file_hash["size"]
48
+ issue="wrong size"
49
+ elsif !verify_hash(file_hash, full_path, possible_hashes)
50
+ issue="wrong hash"
51
+ end
52
+ if issue
53
+ all_ok = false
54
+ (results[issue]||=[]) << [v, file_path]
55
+ if block
56
+ block.call(issue, v, file_path)
57
+ end
58
+ if print
59
+ puts "#{v.metadata.path}: '#{file_path}' #{issue} '#{v.binaries.path(file_path)}'"
60
+ end
61
+ end
62
+ end
63
+ if !recursive
64
+ break
65
+ end
66
+ end
67
+ all_ok
68
+ end
69
+
70
+ def verify_hash(file_hash, full_path, possible_hashes)
71
+ file_hashes = possible_hashes.service_names.select { |name| file_hash.include?(name) }
72
+ checked_hashes = VersionMetadataFile.calculate_hashes(full_path, file_hashes)
73
+ checked_hashes.each_pair do |id, result|
74
+ if file_hash[id] != result
75
+ return false
76
+ end
77
+ end
78
+ true
79
+ end
80
+ end
81
+
82
+ # Imports a version to KiHome
83
+ class VersionImporter
84
+ attr_chain :ki_home, :require
85
+ attr_chain :finder, -> { ki_home.finder}
86
+ attr_chain :tester, -> { VersionTester.new.recursive(false).print(true) }
87
+ attr_chain :move_files
88
+ attr_chain :create_new_version
89
+ attr_chain :specific_version_id
90
+
91
+ # Imports a version to KiHome
92
+ # * import(file, binary_directory) expects two String parameters defining version file location and directory base for binaries
93
+ def import(*args)
94
+ if args.size == 2
95
+ file, input = args
96
+ source = DirectoryBase.new(input)
97
+ metadata = VersionMetadataFile.new(file)
98
+ else
99
+ raise "Not supported: '#{args.inspect}'"
100
+ end
101
+ test_version(file, input)
102
+
103
+ import_from_metadata(metadata, source)
104
+ end
105
+
106
+ def import_from_metadata(metadata, source=nil)
107
+ if defined?(@specific_version_id) && defined?(@create_new_version)
108
+ raise "Can't define both specific_version_id '#{specific_version_id}' and create_new_version '#{create_new_version}'!"
109
+ end
110
+
111
+ if defined?(@specific_version_id)
112
+ version_id = @specific_version_id
113
+ elsif defined? @create_new_version
114
+ component_id = @create_new_version
115
+ version = finder.version(component_id)
116
+ if version
117
+ id = version.version_id.split("/").last
118
+ version_number = (Integer(id) + 1).to_s
119
+ else
120
+ version_number = "1"
121
+ end
122
+ version_id = File.join(component_id, version_number)
123
+ else
124
+ version_id = metadata.version_id
125
+ end
126
+
127
+ version_arr = version_id.split("/")
128
+ version_number = version_arr.delete_at(-1)
129
+ component_id = version_arr.join("/")
130
+
131
+ version = finder.version(version_id)
132
+ if version && version.exists?
133
+ raise "'#{version_id}' exists in repository already!"
134
+ end
135
+
136
+ # creates directories
137
+ components_dir = ki_home.repositories.add_item("local").mkdir.components
138
+ binary_dest = metadata_dir = components_dir.add_item(component_id).mkdir.versions.add_version(version_number).mkdir
139
+
140
+ metadata_dir.metadata.cached_data = metadata.cached_data
141
+ metadata_dir.metadata.version_id = version_id
142
+ metadata_dir.metadata.save
143
+ if defined? @move_files
144
+ FileUtils.rm(metadata.path)
145
+ end
146
+ source_dirs = []
147
+ metadata_dir.metadata.files.each do |file_info|
148
+ file_path = file_info["path"]
149
+ dir = File.dirname(file_path)
150
+ if dir != "."
151
+ source_dirs << dir
152
+ binary_dest.mkdir(dir)
153
+ end
154
+ to_repo(source.path(file_path), binary_dest.path(file_path))
155
+ end
156
+ delete_empty_source_dirs(source, source_dirs)
157
+ end
158
+
159
+
160
+ def delete_empty_source_dirs(source, source_dirs)
161
+ if defined? @move_files
162
+ expanded_source_dirs = {}
163
+ source_dirs.each do |d|
164
+ dir_entries(d).each do |expanded|
165
+ expanded_source_dirs[expanded] = true
166
+ end
167
+ end
168
+ expanded_source_dirs.keys.each do |dir|
169
+ checked_dir = source.path(dir)
170
+ if Dir.entries(checked_dir) == [".", ".."]
171
+ FileUtils.rmdir(checked_dir)
172
+ end
173
+ end
174
+ end
175
+ end
176
+
177
+ # splits dir path in to all components: foo/bar/baz, foo/bar, foo
178
+ def dir_entries(str)
179
+ arr = str.split("/")
180
+ ret = []
181
+ c = arr.size
182
+ while (c > 0)
183
+ ret << File.join(arr[0..c])
184
+ c-=1
185
+ end
186
+ ret
187
+ end
188
+
189
+ def to_repo(src, dest)
190
+ if defined? @move_files
191
+ FileUtils.mv(src, dest)
192
+ else
193
+ FileUtils.cp(src, dest)
194
+ end
195
+ end
196
+
197
+ def test_version(file, input)
198
+ all_ok = tester.ki_home(ki_home).test_version(Version.create_version(file, input))
199
+ if !all_ok
200
+ raise "Files are not ok!"
201
+ end
202
+ end
203
+ end
204
+
205
+ # Exports a version to directory
206
+ # * if test_dependencies set to true, tests the version before exporting
207
+ class VersionExporter
208
+ attr_chain :ki_home, :require
209
+ attr_chain :finder, -> { ki_home.finder }
210
+ attr_chain :test_dependencies
211
+ attr_chain :find_files, -> { FileFinder.new }
212
+ attr_chain :copy
213
+
214
+ # Exports a version to directory
215
+ def export(version, out)
216
+ ver = finder.version(version)
217
+ if test_dependencies
218
+ test_version(ver)
219
+ end
220
+ files = find_files.version(ver).file_map.sort
221
+ files.each do |file_path, full_path|
222
+ dir = File.dirname(file_path)
223
+ if dir != "."
224
+ FileUtils.mkdir_p File.join(out, dir)
225
+ end
226
+ if defined? @copy
227
+ FileUtils.cp(full_path, File.join(out, file_path))
228
+ else
229
+ FileUtils.ln_sf(full_path, File.join(out, file_path))
230
+ end
231
+ end
232
+ end
233
+
234
+ def test_version(version)
235
+ tester = VersionTester.new.ki_home(ki_home).finder(finder).recursive(true).print(true)
236
+ all_ok = tester.test_version(version)
237
+ if !all_ok
238
+ raise "Files are not ok!"
239
+ end
240
+ end
241
+ end
242
+ end
@@ -0,0 +1,145 @@
1
+ # encoding: UTF-8
2
+
3
+ # Copyright 2012 Mikko Apo
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ module Ki
18
+ #
19
+ #
20
+ class VersionIterator
21
+ attr_chain :version, :require
22
+ attr_chain :dependency
23
+ attr_chain :package_path
24
+ attr_chain :dependency_path
25
+ attr_chain :finder, -> { version.component.finder }
26
+ attr_chain :block
27
+ attr_chain :internals
28
+ attr_chain :exclude_dependencies, -> { [] }, :convert => lambda { |list| Array.wrap(list).map { |s| /#{s}/ } }
29
+
30
+ def iterate_versions(&block)
31
+ start_iteration do |version_iterator|
32
+ block.call(version_iterator.version)
33
+ version_iterator.iterate_dependencies
34
+ end
35
+ end
36
+
37
+ def start_iteration(&block)
38
+ @block = block
39
+ @internals = true
40
+ block.call(self)
41
+ end
42
+
43
+ def iterate_dependencies
44
+ version.metadata.dependencies.map do |dep|
45
+ if internals || !dep["internal"]
46
+ dep_v = VersionIterator.new
47
+ dep_v.version = finder.version(dep["version_id"])
48
+ dep_v.dependency = dep
49
+ dep_v.exclude_dependencies.concat(exclude_dependencies).concat(select_dep_rm(dep))
50
+ dep_v.block = block
51
+ if dep["path"] || package_path
52
+ dep_v.package_path = File.join([package_path, dep["path"]].compact)
53
+ end
54
+ if dependency_path || dep["name"]
55
+ dep_v.dependency_path = File.join([dependency_path, dep["name"]].compact)
56
+ end
57
+ if ok_to_iterate_dependency(dep_v)
58
+ [dep, dep_v.version, block.call(dep_v)]
59
+ end
60
+ end
61
+ end.compact
62
+ end
63
+
64
+ def select_dep_rm(dep_v)
65
+ ops = dep_v["operations"]
66
+ if ops
67
+ ops.map do |op|
68
+ if op.first == "dep-rm"
69
+ op[1..-1]
70
+ end
71
+ end.compact.flatten
72
+ else
73
+ []
74
+ end
75
+ end
76
+
77
+ def ok_to_iterate_dependency(dep_v)
78
+ !(exclude_dependencies.size > 0 && (exclude_dependencies.any_matches?(dep_v.version.version_id) || dep_v.dependency_path && exclude_dependencies.any_matches?(dep_v.dependency_path)))
79
+ end
80
+ end
81
+
82
+ class FileFinder < VersionIterator
83
+ attr_chain :files, -> { [] }, :convert => lambda { |list| Array.wrap(list).map { |s| FileRegexp.matcher(s) } }
84
+ attr_chain :exclude_files, -> { [] }, :convert => lambda { |list| Array.wrap(list).map { |s| FileRegexp.matcher(s) } }
85
+ attr_chain :tags, -> { [] }, :convert => lambda { |list| Array.wrap(list)}
86
+ attr_chain :exclude_tags, -> { [] }, :convert => lambda { |list| Array.wrap(list)}
87
+
88
+ def file_map
89
+ start_iteration do |ver_iterator|
90
+ ret = {}
91
+ ver_iterator.iterate_dependencies.each do |dependency, version, file_map|
92
+ file_operations(file_map, dependency)
93
+ ret.merge!(file_map)
94
+ end
95
+ ver = ver_iterator.version
96
+ binaries = ver.binaries
97
+ metadata = ver.metadata
98
+ # TODO: file operations should be applied to the files before the files are filtered
99
+ metadata.files.each do |file|
100
+ path = file["path"]
101
+ file_path = File.join([ver_iterator.package_path, path].compact)
102
+ if ok_to_add_file(file, file_path)
103
+ ret[file_path]=binaries.path(path)
104
+ end
105
+ end
106
+ file_operations(ret, metadata.cached_data)
107
+ ret
108
+ end
109
+ end
110
+
111
+ def file_list
112
+ file_map.values
113
+ end
114
+
115
+ # Modifies
116
+ def file_operations(file_map, dependency)
117
+ operations = dependency["operations"]
118
+ if operations
119
+ VersionFileOperations.new.edit_file_map(file_map, operations)
120
+ end
121
+ end
122
+
123
+ # File is added to the list if
124
+ # - files pattern list is empty (select all files) or file path matches any files pattern
125
+ # - it does not match any file exclude patterns
126
+ # - tags selection list is empty or file has any tags from tags selection list
127
+ # - no tags match tags from tags exclusion list
128
+ def ok_to_add_file(file, file_path)
129
+ file_tags = file["tags"] || []
130
+ (files.size == 0 || files.any_matches?(file_path)) &&
131
+ !exclude_files.any_matches?(file_path) &&
132
+ (tags.size == 0 || (cross_any_matches?(file_tags, tags)) &&
133
+ !cross_any_matches?(file_tags, exclude_tags))
134
+ end
135
+
136
+ def cross_any_matches?(arr, dest_arr)
137
+ arr.each do |i|
138
+ if dest_arr.any_matches?(i)
139
+ return true
140
+ end
141
+ end
142
+ false
143
+ end
144
+ end
145
+ end
@@ -0,0 +1,80 @@
1
+ # encoding: UTF-8
2
+
3
+ # Copyright 2012 Mikko Apo
4
+ #
5
+ # Licensed under the Apache License, Version 2.0 (the "License");
6
+ # you may not use this file except in compliance with the License.
7
+ # You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS,
13
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+ # See the License for the specific language governing permissions and
15
+ # limitations under the License.
16
+
17
+ module Ki
18
+ class VersionFileOperations
19
+ def resolve_dest_file(file, dest, matching_pattern)
20
+ matcher = matching_pattern.match(file)
21
+ dest = dest.gsub(/\$\d/) do |str|
22
+ matcher[Integer(str[1..-1])]
23
+ end
24
+ if dest.end_with?("/")
25
+ dest = File.join(dest, File.basename(file))
26
+ end
27
+ if dest.start_with?("/")
28
+ dest = dest[1..-1]
29
+ end
30
+ dest
31
+ end
32
+
33
+ def copy_or_move(file_map, args, op)
34
+ delete = op == "mv"
35
+ dest = args.delete_at(-1)
36
+ patterns = args.map { |pattern| FileRegexp.matcher(pattern) }
37
+ matching_files = []
38
+ file_map.keys.each do |file|
39
+ matching_pattern = patterns.any_matches?(file)
40
+ if matching_pattern
41
+ matching_files << file
42
+ dest_file = resolve_dest_file(file, dest, matching_pattern)
43
+ file_map[dest_file]=file_map[file]
44
+ if delete
45
+ file_map.delete(file)
46
+ end
47
+ end
48
+ end
49
+ end
50
+
51
+ def delete(file_map, args, op)
52
+ patterns = args.map { |pattern| FileRegexp.matcher(pattern) }
53
+ file_map.keys.each do |file|
54
+ if patterns.any_matches?(file)
55
+ file_map.delete(file)
56
+ end
57
+ end
58
+ end
59
+
60
+ def edit_file_map(file_map, operations)
61
+ operations.each do |op, *args|
62
+ case op
63
+ when "cp"
64
+ copy_or_move(file_map, args, op)
65
+ when "mv"
66
+ copy_or_move(file_map, args, op)
67
+ when "rm"
68
+ delete(file_map, args, op)
69
+ end
70
+ end
71
+ end
72
+
73
+ end
74
+
75
+ class FileRegexp
76
+ def FileRegexp.matcher(s)
77
+ /^#{s.gsub(/\./, '\.').gsub('*', ".*")}$/
78
+ end
79
+ end
80
+ end