cocoapods-core 0.30.0 → 1.15.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +7 -10
- data/lib/cocoapods-core/build_type.rb +121 -0
- data/lib/cocoapods-core/cdn_source.rb +501 -0
- data/lib/cocoapods-core/core_ui.rb +4 -3
- data/lib/cocoapods-core/dependency.rb +100 -73
- data/lib/cocoapods-core/gem_version.rb +1 -2
- data/lib/cocoapods-core/github.rb +32 -5
- data/lib/cocoapods-core/http.rb +86 -0
- data/lib/cocoapods-core/lockfile.rb +161 -56
- data/lib/cocoapods-core/metrics.rb +47 -0
- data/lib/cocoapods-core/platform.rb +99 -11
- data/lib/cocoapods-core/podfile/dsl.rb +623 -124
- data/lib/cocoapods-core/podfile/target_definition.rb +662 -109
- data/lib/cocoapods-core/podfile.rb +138 -65
- data/lib/cocoapods-core/requirement.rb +37 -8
- data/lib/cocoapods-core/source/acceptor.rb +16 -13
- data/lib/cocoapods-core/source/aggregate.rb +79 -103
- data/lib/cocoapods-core/source/health_reporter.rb +9 -18
- data/lib/cocoapods-core/source/manager.rb +488 -0
- data/lib/cocoapods-core/source/metadata.rb +79 -0
- data/lib/cocoapods-core/source.rb +241 -70
- data/lib/cocoapods-core/specification/consumer.rb +187 -47
- data/lib/cocoapods-core/specification/dsl/attribute.rb +49 -85
- data/lib/cocoapods-core/specification/dsl/attribute_support.rb +6 -8
- data/lib/cocoapods-core/specification/dsl/deprecations.rb +9 -126
- data/lib/cocoapods-core/specification/dsl/platform_proxy.rb +30 -20
- data/lib/cocoapods-core/specification/dsl.rb +943 -296
- data/lib/cocoapods-core/specification/json.rb +64 -23
- data/lib/cocoapods-core/specification/linter/analyzer.rb +218 -0
- data/lib/cocoapods-core/specification/linter/result.rb +128 -0
- data/lib/cocoapods-core/specification/linter.rb +310 -309
- data/lib/cocoapods-core/specification/root_attribute_accessors.rb +90 -39
- data/lib/cocoapods-core/specification/set/presenter.rb +35 -71
- data/lib/cocoapods-core/specification/set.rb +42 -96
- data/lib/cocoapods-core/specification.rb +368 -130
- data/lib/cocoapods-core/standard_error.rb +45 -24
- data/lib/cocoapods-core/trunk_source.rb +14 -0
- data/lib/cocoapods-core/vendor/requirement.rb +133 -53
- data/lib/cocoapods-core/vendor/version.rb +197 -156
- data/lib/cocoapods-core/vendor.rb +1 -5
- data/lib/cocoapods-core/version.rb +137 -42
- data/lib/cocoapods-core/yaml_helper.rb +334 -0
- data/lib/cocoapods-core.rb +10 -4
- metadata +100 -27
- data/lib/cocoapods-core/source/abstract_data_provider.rb +0 -71
- data/lib/cocoapods-core/source/file_system_data_provider.rb +0 -150
- data/lib/cocoapods-core/source/github_data_provider.rb +0 -143
- data/lib/cocoapods-core/specification/set/statistics.rb +0 -266
- data/lib/cocoapods-core/yaml_converter.rb +0 -192
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 28b58aee1d429faed6b698b36263cc7e86e1ebe4540b2c7e03d1883e28e74f59
|
4
|
+
data.tar.gz: 81da93773c4f3cff54ae532ef20b3fd9b3190fbca1fc4d7fa89e57da4c840ab3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6ab7a1a1c329774e398f25080dd03f3fda96bc96be0c5582088010b317e2c64876f2d3b01e1ae4d0ea48f32839443ff907f90b9cd27aa681cf9a9e1c94eee43d
|
7
|
+
data.tar.gz: 7a82bc2bda4f3f2cf4f54bf70300bc5c633073696d5882cd282d302ec5b00d8a33072edd7cbaa60812b52525b51cce47ca10597dcefd1b82fb001c55e7852aa4
|
data/README.md
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
# CocoaPods Core
|
2
2
|
|
3
|
-
[![Build Status](https://
|
4
|
-
[![Coverage
|
5
|
-
[![
|
3
|
+
[![Build Status](https://github.com/CocoaPods/Core/workflows/Specs/badge.svg)](https://github.com/CocoaPods/Core/actions/workflows/Specs.yml)
|
4
|
+
[![Test Coverage](https://api.codeclimate.com/v1/badges/91a2d70b9ed977815c66/test_coverage)](https://codeclimate.com/github/CocoaPods/Core/test_coverage)
|
5
|
+
[![Maintainability](https://api.codeclimate.com/v1/badges/91a2d70b9ed977815c66/maintainability)](https://codeclimate.com/github/CocoaPods/Core/maintainability)
|
6
6
|
|
7
7
|
The CocoaPods-Core gem provides support to work with the models of CocoaPods.
|
8
|
-
It is intended to be used in place of the CocoaPods when the installation
|
8
|
+
It is intended to be used in place of the CocoaPods gem when the installation
|
9
9
|
of the dependencies is not needed. Therefore, it is suitable for web services.
|
10
10
|
|
11
11
|
Provides support for working with the following models:
|
12
12
|
|
13
|
-
- `Pod::Specification` - [
|
14
|
-
- `Pod::Podfile` - [
|
13
|
+
- `Pod::Specification` - [Podspec Syntax Reference](https://guides.cocoapods.org/syntax/podspec.html).
|
14
|
+
- `Pod::Podfile` - [Podfile Syntax Reference](https://guides.cocoapods.org/syntax/podfile.html).
|
15
15
|
- `Pod::Source` - collections of podspec files like the [CocoaPods Spec repo](https://github.com/CocoaPods/Specs).
|
16
16
|
|
17
17
|
The gem also provides support for ancillary features like
|
@@ -25,10 +25,7 @@ files.
|
|
25
25
|
$ [sudo] gem install cocoapods-core
|
26
26
|
```
|
27
27
|
|
28
|
-
The `cocoapods-core` gem requires
|
29
|
-
|
30
|
-
- Ruby 1.8.7 (shipped with OS X 10.8).
|
31
|
-
- Ruby 1.9.3 (recommended).
|
28
|
+
The `cocoapods-core` gem requires Ruby 2.6.0 or later.
|
32
29
|
|
33
30
|
## Collaborate
|
34
31
|
|
@@ -0,0 +1,121 @@
|
|
1
|
+
module Pod
|
2
|
+
class BuildType
|
3
|
+
# @return [Array<Symbol>] known packaging options.
|
4
|
+
#
|
5
|
+
KNOWN_PACKAGING_OPTIONS = %i(library framework).freeze
|
6
|
+
|
7
|
+
# @return [Array<Symbol>] known linking options.
|
8
|
+
#
|
9
|
+
KNOWN_LINKAGE_OPTIONS = %i(static dynamic).freeze
|
10
|
+
|
11
|
+
# @return [Symbol] the packaging for this build type, one of #KNOWN_PACKAGING_OPTIONS
|
12
|
+
#
|
13
|
+
attr_reader :packaging
|
14
|
+
|
15
|
+
# @return [Symbol] the linkage for this build type, one of #KNOWN_LINKAGE_OPTIONS
|
16
|
+
#
|
17
|
+
attr_reader :linkage
|
18
|
+
|
19
|
+
attr_reader :hash
|
20
|
+
|
21
|
+
def initialize(linkage: :static, packaging: :library)
|
22
|
+
unless KNOWN_LINKAGE_OPTIONS.include?(linkage)
|
23
|
+
raise ArgumentError, "Invalid linkage option #{linkage.inspect}, valid options are #{KNOWN_LINKAGE_OPTIONS.inspect}"
|
24
|
+
end
|
25
|
+
unless KNOWN_PACKAGING_OPTIONS.include?(packaging)
|
26
|
+
raise ArgumentError, "Invalid packaging option #{packaging.inspect}, valid options are #{KNOWN_PACKAGING_OPTIONS.inspect}"
|
27
|
+
end
|
28
|
+
@packaging = packaging
|
29
|
+
@linkage = linkage
|
30
|
+
@hash = packaging.hash ^ linkage.hash
|
31
|
+
end
|
32
|
+
|
33
|
+
# @return [BuildType] the build type for a dynamic library
|
34
|
+
def self.dynamic_library
|
35
|
+
new(:linkage => :dynamic, :packaging => :library)
|
36
|
+
end
|
37
|
+
|
38
|
+
# @return [BuildType] the build type for a static library
|
39
|
+
#
|
40
|
+
def self.static_library
|
41
|
+
new(:linkage => :static, :packaging => :library)
|
42
|
+
end
|
43
|
+
|
44
|
+
# @return [BuildType] the build type for a dynamic framework
|
45
|
+
#
|
46
|
+
def self.dynamic_framework
|
47
|
+
new(:linkage => :dynamic, :packaging => :framework)
|
48
|
+
end
|
49
|
+
|
50
|
+
# @return [BuildType] the build type for a static framework
|
51
|
+
#
|
52
|
+
def self.static_framework
|
53
|
+
new(:linkage => :static, :packaging => :framework)
|
54
|
+
end
|
55
|
+
|
56
|
+
# @return [Boolean] whether the target is built dynamically
|
57
|
+
#
|
58
|
+
def dynamic?
|
59
|
+
linkage == :dynamic
|
60
|
+
end
|
61
|
+
|
62
|
+
# @return [Boolean] whether the target is built statically
|
63
|
+
#
|
64
|
+
def static?
|
65
|
+
linkage == :static
|
66
|
+
end
|
67
|
+
|
68
|
+
# @return [Boolean] whether the target is built as a framework
|
69
|
+
#
|
70
|
+
def framework?
|
71
|
+
packaging == :framework
|
72
|
+
end
|
73
|
+
|
74
|
+
# @return [Boolean] whether the target is built as a library
|
75
|
+
#
|
76
|
+
def library?
|
77
|
+
packaging == :library
|
78
|
+
end
|
79
|
+
|
80
|
+
# @return [Boolean] whether the target is built as a dynamic framework
|
81
|
+
#
|
82
|
+
def dynamic_framework?
|
83
|
+
dynamic? && framework?
|
84
|
+
end
|
85
|
+
|
86
|
+
# @return [Boolean] whether the target is built as a dynamic library
|
87
|
+
#
|
88
|
+
def dynamic_library?
|
89
|
+
dynamic? && library?
|
90
|
+
end
|
91
|
+
|
92
|
+
# @return [Boolean] whether the target is built as a static framework
|
93
|
+
#
|
94
|
+
def static_framework?
|
95
|
+
static? && framework?
|
96
|
+
end
|
97
|
+
|
98
|
+
# @return [Boolean] whether the target is built as a static library
|
99
|
+
#
|
100
|
+
def static_library?
|
101
|
+
static? && library?
|
102
|
+
end
|
103
|
+
|
104
|
+
def to_s
|
105
|
+
"#{linkage} #{packaging}"
|
106
|
+
end
|
107
|
+
|
108
|
+
def to_hash
|
109
|
+
{ :linkage => linkage, :packaging => packaging }
|
110
|
+
end
|
111
|
+
|
112
|
+
def inspect
|
113
|
+
"#<#{self.class} linkage=#{linkage} packaging=#{packaging}>"
|
114
|
+
end
|
115
|
+
|
116
|
+
def ==(other)
|
117
|
+
linkage == other.linkage &&
|
118
|
+
packaging == other.packaging
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,501 @@
|
|
1
|
+
require 'cocoapods-core/source'
|
2
|
+
require 'rest'
|
3
|
+
require 'concurrent'
|
4
|
+
require 'netrc'
|
5
|
+
require 'addressable'
|
6
|
+
|
7
|
+
module Pod
|
8
|
+
# Subclass of Pod::Source to provide support for CDN-based Specs repositories
|
9
|
+
#
|
10
|
+
class CDNSource < Source
|
11
|
+
include Concurrent
|
12
|
+
|
13
|
+
MAX_NUMBER_OF_RETRIES = (ENV['COCOAPODS_CDN_MAX_NUMBER_OF_RETRIES'] || 5).to_i
|
14
|
+
# Single thread executor for all network activity.
|
15
|
+
HYDRA_EXECUTOR = Concurrent::SingleThreadExecutor.new
|
16
|
+
|
17
|
+
private_constant :HYDRA_EXECUTOR
|
18
|
+
|
19
|
+
# @param [String] repo The name of the repository
|
20
|
+
#
|
21
|
+
def initialize(repo)
|
22
|
+
@check_existing_files_for_update = false
|
23
|
+
# Optimization: we initialize startup_time when the source is first initialized
|
24
|
+
# and then test file modification dates against it. Any file that was touched
|
25
|
+
# after the source was initialized, is considered fresh enough.
|
26
|
+
@startup_time = Time.new
|
27
|
+
|
28
|
+
@version_arrays_by_fragment_by_name = {}
|
29
|
+
|
30
|
+
super(repo)
|
31
|
+
end
|
32
|
+
|
33
|
+
# @return [String] The URL of the source.
|
34
|
+
#
|
35
|
+
def url
|
36
|
+
@url ||= File.read(repo.join('.url')).chomp.chomp('/') + '/'
|
37
|
+
end
|
38
|
+
|
39
|
+
# @return [String] The type of the source.
|
40
|
+
#
|
41
|
+
def type
|
42
|
+
'CDN'
|
43
|
+
end
|
44
|
+
|
45
|
+
def refresh_metadata
|
46
|
+
if metadata.nil?
|
47
|
+
unless repo.exist?
|
48
|
+
debug "CDN: Repo #{name} does not exist!"
|
49
|
+
return
|
50
|
+
end
|
51
|
+
|
52
|
+
specs_dir.mkpath
|
53
|
+
download_file('CocoaPods-version.yml')
|
54
|
+
end
|
55
|
+
|
56
|
+
super
|
57
|
+
end
|
58
|
+
|
59
|
+
def preheat_existing_files
|
60
|
+
files_to_update = files_definitely_to_update + deprecated_local_podspecs - ['deprecated_podspecs.txt']
|
61
|
+
debug "CDN: #{name} Going to update #{files_to_update.count} files"
|
62
|
+
|
63
|
+
concurrent_requests_catching_errors do
|
64
|
+
# Queue all tasks first
|
65
|
+
loaders = files_to_update.map do |file|
|
66
|
+
download_file_async(file)
|
67
|
+
end
|
68
|
+
# Block and wait for all to complete running on Hydra
|
69
|
+
Promises.zip_futures_on(HYDRA_EXECUTOR, *loaders).wait!
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def files_definitely_to_update
|
74
|
+
Pathname.glob(repo.join('**/*.{txt,yml}')).map { |f| f.relative_path_from(repo).to_s }
|
75
|
+
end
|
76
|
+
|
77
|
+
def deprecated_local_podspecs
|
78
|
+
download_file('deprecated_podspecs.txt')
|
79
|
+
local_file('deprecated_podspecs.txt', &:to_a).
|
80
|
+
map { |f| Pathname.new(f.chomp) }.
|
81
|
+
select { |f| repo.join(f).exist? }
|
82
|
+
end
|
83
|
+
|
84
|
+
# @return [Pathname] The directory where the specs are stored.
|
85
|
+
#
|
86
|
+
def specs_dir
|
87
|
+
@specs_dir ||= repo + 'Specs'
|
88
|
+
end
|
89
|
+
|
90
|
+
# @!group Querying the source
|
91
|
+
#-------------------------------------------------------------------------#
|
92
|
+
|
93
|
+
# @return [Array<String>] the list of the name of all the Pods.
|
94
|
+
#
|
95
|
+
def pods
|
96
|
+
download_file('all_pods.txt')
|
97
|
+
local_file('all_pods.txt', &:to_a).map(&:chomp)
|
98
|
+
end
|
99
|
+
|
100
|
+
# @return [Array<Version>] all the available versions for the Pod, sorted
|
101
|
+
# from highest to lowest.
|
102
|
+
#
|
103
|
+
# @param [String] name
|
104
|
+
# the name of the Pod.
|
105
|
+
#
|
106
|
+
def versions(name)
|
107
|
+
return nil unless specs_dir
|
108
|
+
raise ArgumentError, 'No name' unless name
|
109
|
+
|
110
|
+
fragment = pod_shard_fragment(name)
|
111
|
+
|
112
|
+
ensure_versions_file_loaded(fragment)
|
113
|
+
|
114
|
+
return @versions_by_name[name] unless @versions_by_name[name].nil?
|
115
|
+
|
116
|
+
pod_path_actual = pod_path(name)
|
117
|
+
pod_path_relative = relative_pod_path(name)
|
118
|
+
|
119
|
+
return nil if @version_arrays_by_fragment_by_name[fragment][name].nil?
|
120
|
+
|
121
|
+
concurrent_requests_catching_errors do
|
122
|
+
loaders = []
|
123
|
+
|
124
|
+
@versions_by_name[name] ||= @version_arrays_by_fragment_by_name[fragment][name].map do |version|
|
125
|
+
# Optimization: ensure all the podspec files at least exist. The correct one will get refreshed
|
126
|
+
# in #specification_path regardless.
|
127
|
+
podspec_version_path_relative = Pathname.new(version).join("#{name}.podspec.json")
|
128
|
+
|
129
|
+
unless pod_path_actual.join(podspec_version_path_relative).exist?
|
130
|
+
# Queue all podspec download tasks first
|
131
|
+
loaders << download_file_async(pod_path_relative.join(podspec_version_path_relative).to_s)
|
132
|
+
end
|
133
|
+
|
134
|
+
begin
|
135
|
+
Version.new(version) if version[0, 1] != '.'
|
136
|
+
rescue ArgumentError
|
137
|
+
raise Informative, 'An unexpected version directory ' \
|
138
|
+
"`#{version}` was encountered for the " \
|
139
|
+
"`#{pod_path_actual}` Pod in the `#{name}` repository."
|
140
|
+
end
|
141
|
+
end.compact.sort.reverse
|
142
|
+
|
143
|
+
# Block and wait for all to complete running on Hydra
|
144
|
+
Promises.zip_futures_on(HYDRA_EXECUTOR, *loaders).wait!
|
145
|
+
end
|
146
|
+
|
147
|
+
@versions_by_name[name]
|
148
|
+
end
|
149
|
+
|
150
|
+
# Returns the path of the specification with the given name and version.
|
151
|
+
#
|
152
|
+
# @param [String] name
|
153
|
+
# the name of the Pod.
|
154
|
+
#
|
155
|
+
# @param [Version,String] version
|
156
|
+
# the version for the specification.
|
157
|
+
#
|
158
|
+
# @return [Pathname] The path of the specification.
|
159
|
+
#
|
160
|
+
def specification_path(name, version)
|
161
|
+
raise ArgumentError, 'No name' unless name
|
162
|
+
raise ArgumentError, 'No version' unless version
|
163
|
+
unless versions(name).include?(Version.new(version))
|
164
|
+
raise StandardError, "Unable to find the specification #{name} " \
|
165
|
+
"(#{version}) in the #{self.name} source."
|
166
|
+
end
|
167
|
+
|
168
|
+
podspec_version_path_relative = Pathname.new(version.to_s).join("#{name}.podspec.json")
|
169
|
+
relative_podspec = relative_pod_path(name).join(podspec_version_path_relative).to_s
|
170
|
+
download_file(relative_podspec)
|
171
|
+
pod_path(name).join(podspec_version_path_relative)
|
172
|
+
end
|
173
|
+
|
174
|
+
# @return [Array<Specification>] all the specifications contained by the
|
175
|
+
# source.
|
176
|
+
#
|
177
|
+
def all_specs
|
178
|
+
raise Informative, "Can't retrieve all the specs for a CDN-backed source, it will take forever"
|
179
|
+
end
|
180
|
+
|
181
|
+
# @return [Array<Sets>] the sets of all the Pods.
|
182
|
+
#
|
183
|
+
def pod_sets
|
184
|
+
raise Informative, "Can't retrieve all the pod sets for a CDN-backed source, it will take forever"
|
185
|
+
end
|
186
|
+
|
187
|
+
# @!group Searching the source
|
188
|
+
#-------------------------------------------------------------------------#
|
189
|
+
|
190
|
+
# @return [Set] a set for a given dependency. The set is identified by the
|
191
|
+
# name of the dependency and takes into account subspecs.
|
192
|
+
#
|
193
|
+
# @note This method is optimized for fast lookups by name, i.e. it does
|
194
|
+
# *not* require iterating through {#pod_sets}
|
195
|
+
#
|
196
|
+
# @todo Rename to #load_set
|
197
|
+
#
|
198
|
+
def search(query)
|
199
|
+
unless specs_dir
|
200
|
+
raise Informative, "Unable to find a source named: `#{name}`"
|
201
|
+
end
|
202
|
+
if query.is_a?(Dependency)
|
203
|
+
query = query.root_name
|
204
|
+
end
|
205
|
+
|
206
|
+
fragment = pod_shard_fragment(query)
|
207
|
+
|
208
|
+
ensure_versions_file_loaded(fragment)
|
209
|
+
|
210
|
+
version_arrays_by_name = @version_arrays_by_fragment_by_name[fragment] || {}
|
211
|
+
|
212
|
+
found = version_arrays_by_name[query].nil? ? nil : query
|
213
|
+
|
214
|
+
if found
|
215
|
+
set = set(query)
|
216
|
+
set if set.specification_name == query
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
# @return [Array<Set>] The list of the sets that contain the search term.
|
221
|
+
#
|
222
|
+
# @param [String] query
|
223
|
+
# the search term. Can be a regular expression.
|
224
|
+
#
|
225
|
+
# @param [Boolean] full_text_search
|
226
|
+
# performed using Algolia
|
227
|
+
#
|
228
|
+
# @note full text search requires to load the specification for each pod,
|
229
|
+
# and therefore not supported.
|
230
|
+
#
|
231
|
+
def search_by_name(query, full_text_search = false)
|
232
|
+
if full_text_search
|
233
|
+
require 'algoliasearch'
|
234
|
+
begin
|
235
|
+
algolia_result = algolia_search_index.search(query, :attributesToRetrieve => 'name')
|
236
|
+
names = algolia_result['hits'].map { |r| r['name'] }
|
237
|
+
names.map { |n| set(n) }.reject { |s| s.versions.compact.empty? }
|
238
|
+
rescue Algolia::AlgoliaError => e
|
239
|
+
raise Informative, "CDN: #{name} - Cannot perform full-text search because Algolia returned an error: #{e}"
|
240
|
+
end
|
241
|
+
else
|
242
|
+
super(query)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
# Check update dates for all existing files.
|
247
|
+
# Does not download non-existing specs, since CDN-backed repo is updated live.
|
248
|
+
#
|
249
|
+
# @param [Boolean] show_output
|
250
|
+
#
|
251
|
+
# @return [Array<String>] Always returns empty array, as it cannot know
|
252
|
+
# everything that actually changed.
|
253
|
+
#
|
254
|
+
def update(_show_output)
|
255
|
+
@check_existing_files_for_update = true
|
256
|
+
begin
|
257
|
+
preheat_existing_files
|
258
|
+
ensure
|
259
|
+
@check_existing_files_for_update = false
|
260
|
+
end
|
261
|
+
[]
|
262
|
+
end
|
263
|
+
|
264
|
+
def updateable?
|
265
|
+
true
|
266
|
+
end
|
267
|
+
|
268
|
+
def git?
|
269
|
+
false
|
270
|
+
end
|
271
|
+
|
272
|
+
def indexable?
|
273
|
+
false
|
274
|
+
end
|
275
|
+
|
276
|
+
private
|
277
|
+
|
278
|
+
def ensure_versions_file_loaded(fragment)
|
279
|
+
return if !@version_arrays_by_fragment_by_name[fragment].nil? && !@check_existing_files_for_update
|
280
|
+
|
281
|
+
# Index file that contains all the versions for all the pods in the shard.
|
282
|
+
# We use those because you can't get a directory listing from a CDN.
|
283
|
+
index_file_name = index_file_name_for_fragment(fragment)
|
284
|
+
download_file(index_file_name)
|
285
|
+
versions_raw = local_file(index_file_name, &:to_a).map(&:chomp)
|
286
|
+
@version_arrays_by_fragment_by_name[fragment] = versions_raw.reduce({}) do |hash, row|
|
287
|
+
row = row.split('/')
|
288
|
+
pod = row.shift
|
289
|
+
versions = row
|
290
|
+
|
291
|
+
hash[pod] = versions
|
292
|
+
hash
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
def algolia_search_index
|
297
|
+
@index ||= begin
|
298
|
+
require 'algoliasearch'
|
299
|
+
|
300
|
+
raise Informative, "Cannot perform full-text search in repo #{name} because it's missing Algolia config" if download_file('AlgoliaSearch.yml').nil?
|
301
|
+
algolia_config = YAMLHelper.load_string(local_file('AlgoliaSearch.yml', &:read))
|
302
|
+
|
303
|
+
client = Algolia::Client.new(:application_id => algolia_config['application_id'], :api_key => algolia_config['api_key'])
|
304
|
+
Algolia::Index.new(algolia_config['index'], client)
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
def index_file_name_for_fragment(fragment)
|
309
|
+
fragment_joined = fragment.join('_')
|
310
|
+
fragment_joined = '_' + fragment_joined unless fragment.empty?
|
311
|
+
"all_pods_versions#{fragment_joined}.txt"
|
312
|
+
end
|
313
|
+
|
314
|
+
def pod_shard_fragment(pod_name)
|
315
|
+
metadata.path_fragment(pod_name)[0..-2]
|
316
|
+
end
|
317
|
+
|
318
|
+
def local_file_okay?(partial_url)
|
319
|
+
file_path = repo.join(partial_url)
|
320
|
+
File.exist?(file_path) && File.size(file_path) > 0
|
321
|
+
end
|
322
|
+
|
323
|
+
def local_file(partial_url)
|
324
|
+
file_path = repo.join(partial_url)
|
325
|
+
File.open(file_path) do |file|
|
326
|
+
yield file if block_given?
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
330
|
+
def relative_pod_path(pod_name)
|
331
|
+
pod_path(pod_name).relative_path_from(repo)
|
332
|
+
end
|
333
|
+
|
334
|
+
def download_file(partial_url)
|
335
|
+
# Block the main thread waiting for Hydra to finish
|
336
|
+
#
|
337
|
+
# Used for single-file downloads
|
338
|
+
download_file_async(partial_url).wait!
|
339
|
+
end
|
340
|
+
|
341
|
+
def download_file_async(partial_url)
|
342
|
+
file_remote_url = Addressable::URI.encode(url + partial_url.to_s)
|
343
|
+
path = repo + partial_url
|
344
|
+
|
345
|
+
file_okay = local_file_okay?(partial_url)
|
346
|
+
if file_okay
|
347
|
+
if @startup_time < File.mtime(path)
|
348
|
+
debug "CDN: #{name} Relative path: #{partial_url} modified during this run! Returning local"
|
349
|
+
return Promises.fulfilled_future(partial_url, HYDRA_EXECUTOR)
|
350
|
+
end
|
351
|
+
|
352
|
+
unless @check_existing_files_for_update
|
353
|
+
debug "CDN: #{name} Relative path: #{partial_url} exists! Returning local because checking is only performed in repo update"
|
354
|
+
return Promises.fulfilled_future(partial_url, HYDRA_EXECUTOR)
|
355
|
+
end
|
356
|
+
end
|
357
|
+
|
358
|
+
path.dirname.mkpath
|
359
|
+
|
360
|
+
etag_path = path.sub_ext(path.extname + '.etag')
|
361
|
+
|
362
|
+
etag = File.read(etag_path) if file_okay && File.exist?(etag_path)
|
363
|
+
debug "CDN: #{name} Relative path: #{partial_url}, has ETag? #{etag}" unless etag.nil?
|
364
|
+
|
365
|
+
download_and_save_with_retries_async(partial_url, file_remote_url, etag)
|
366
|
+
end
|
367
|
+
|
368
|
+
def download_and_save_with_retries_async(partial_url, file_remote_url, etag, retries = MAX_NUMBER_OF_RETRIES)
|
369
|
+
path = repo + partial_url
|
370
|
+
etag_path = path.sub_ext(path.extname + '.etag')
|
371
|
+
|
372
|
+
download_task = download_typhoeus_impl_async(file_remote_url, etag).then do |response|
|
373
|
+
case response.response_code
|
374
|
+
when 301, 302
|
375
|
+
redirect_location = response.headers['location']
|
376
|
+
debug "CDN: #{name} Redirecting from #{file_remote_url} to #{redirect_location}"
|
377
|
+
download_and_save_with_retries_async(partial_url, redirect_location, etag)
|
378
|
+
when 304
|
379
|
+
debug "CDN: #{name} Relative path not modified: #{partial_url}"
|
380
|
+
# We need to update the file modification date, as it is later used for freshness
|
381
|
+
# optimization. See #initialize for more information.
|
382
|
+
FileUtils.touch path
|
383
|
+
partial_url
|
384
|
+
when 200
|
385
|
+
File.open(path, 'w') { |f| f.write(response.response_body.force_encoding('UTF-8')) }
|
386
|
+
|
387
|
+
etag_new = response.headers['etag'] unless response.headers.nil?
|
388
|
+
debug "CDN: #{name} Relative path downloaded: #{partial_url}, save ETag: #{etag_new}"
|
389
|
+
File.open(etag_path, 'w') { |f| f.write(etag_new) } unless etag_new.nil?
|
390
|
+
partial_url
|
391
|
+
when 404
|
392
|
+
debug "CDN: #{name} Relative path couldn't be downloaded: #{partial_url} Response: #{response.response_code}"
|
393
|
+
nil
|
394
|
+
when 502, 503, 504
|
395
|
+
# Retryable HTTP errors, usually related to server overloading
|
396
|
+
if retries <= 1
|
397
|
+
raise Informative, "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.response_code} #{response.response_body}"
|
398
|
+
else
|
399
|
+
debug "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.response_code} #{response.response_body}, retries: #{retries - 1}"
|
400
|
+
exponential_backoff_async(retries).then do
|
401
|
+
download_and_save_with_retries_async(partial_url, file_remote_url, etag, retries - 1)
|
402
|
+
end
|
403
|
+
end
|
404
|
+
when 0
|
405
|
+
# Non-HTTP errors, usually network layer
|
406
|
+
if retries <= 1
|
407
|
+
raise Informative, "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.return_message}"
|
408
|
+
else
|
409
|
+
debug "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.return_message}, retries: #{retries - 1}"
|
410
|
+
exponential_backoff_async(retries).then do
|
411
|
+
download_and_save_with_retries_async(partial_url, file_remote_url, etag, retries - 1)
|
412
|
+
end
|
413
|
+
end
|
414
|
+
else
|
415
|
+
raise Informative, "CDN: #{name} URL couldn't be downloaded: #{file_remote_url} Response: #{response.response_code} #{response.response_body}"
|
416
|
+
end
|
417
|
+
end
|
418
|
+
|
419
|
+
# Calling `Future#run` flattens the chained futures created by retries or redirects
|
420
|
+
#
|
421
|
+
# Does not, in fact, run the task - that is already happening in Hydra at this point
|
422
|
+
download_task.run
|
423
|
+
end
|
424
|
+
|
425
|
+
def exponential_backoff_async(retries)
|
426
|
+
sleep_async(backoff_time(retries))
|
427
|
+
end
|
428
|
+
|
429
|
+
def backoff_time(retries)
|
430
|
+
current_retry = MAX_NUMBER_OF_RETRIES - retries
|
431
|
+
4 * 2**current_retry
|
432
|
+
end
|
433
|
+
|
434
|
+
def sleep_async(seconds)
|
435
|
+
# Async sleep to avoid blocking either the main or the Hydra thread
|
436
|
+
Promises.schedule_on(HYDRA_EXECUTOR, seconds)
|
437
|
+
end
|
438
|
+
|
439
|
+
def download_typhoeus_impl_async(file_remote_url, etag)
|
440
|
+
require 'typhoeus'
|
441
|
+
|
442
|
+
# Create a prefereably HTTP/2 request - the protocol is ultimately responsible for picking
|
443
|
+
# the maximum supported protocol
|
444
|
+
# When debugging with proxy, use the following extra options:
|
445
|
+
# :proxy => 'http://localhost:8888',
|
446
|
+
# :ssl_verifypeer => false,
|
447
|
+
# :ssl_verifyhost => 0,
|
448
|
+
request = Typhoeus::Request.new(
|
449
|
+
file_remote_url,
|
450
|
+
:method => :get,
|
451
|
+
:http_version => :httpv2_0,
|
452
|
+
:timeout => 10,
|
453
|
+
:connecttimeout => 10,
|
454
|
+
:accept_encoding => 'gzip',
|
455
|
+
:netrc => :optional,
|
456
|
+
:netrc_file => Netrc.default_path,
|
457
|
+
:headers => etag.nil? ? {} : { 'If-None-Match' => etag },
|
458
|
+
)
|
459
|
+
|
460
|
+
future = Promises.resolvable_future_on(HYDRA_EXECUTOR)
|
461
|
+
queue_request(request)
|
462
|
+
request.on_complete do |response|
|
463
|
+
future.fulfill(response)
|
464
|
+
end
|
465
|
+
|
466
|
+
# This `Future` should never reject, network errors are exposed on `Typhoeus::Response`
|
467
|
+
future
|
468
|
+
end
|
469
|
+
|
470
|
+
def debug(message)
|
471
|
+
if defined?(Pod::UI)
|
472
|
+
Pod::UI.message(message)
|
473
|
+
else
|
474
|
+
CoreUI.puts(message)
|
475
|
+
end
|
476
|
+
end
|
477
|
+
|
478
|
+
def concurrent_requests_catching_errors
|
479
|
+
yield
|
480
|
+
rescue MultipleErrors => e
|
481
|
+
# aggregated error message from `Concurrent`
|
482
|
+
errors = e.errors
|
483
|
+
raise Informative, "CDN: #{name} Repo update failed - #{e.errors.size} error(s):\n#{errors.join("\n")}"
|
484
|
+
end
|
485
|
+
|
486
|
+
def queue_request(request)
|
487
|
+
@hydra ||= Typhoeus::Hydra.new
|
488
|
+
|
489
|
+
# Queue the request into the Hydra (libcurl reactor).
|
490
|
+
@hydra.queue(request)
|
491
|
+
|
492
|
+
# Cycle the reactor on a separate thread
|
493
|
+
#
|
494
|
+
# The way it works is that if more requests are queued while Hydra is in the `#run`
|
495
|
+
# method, it will keep executing them
|
496
|
+
#
|
497
|
+
# The upcoming calls to `#run` will simply run empty.
|
498
|
+
HYDRA_EXECUTOR.post(@hydra, &:run)
|
499
|
+
end
|
500
|
+
end
|
501
|
+
end
|
@@ -1,18 +1,19 @@
|
|
1
1
|
module Pod
|
2
|
-
|
3
2
|
# Manages the UI output so dependent gems can customize it.
|
4
3
|
#
|
5
4
|
module CoreUI
|
6
|
-
|
7
5
|
def self.puts(message)
|
8
6
|
STDOUT.puts message
|
9
7
|
end
|
10
8
|
|
9
|
+
def self.print(message)
|
10
|
+
STDOUT.print(message)
|
11
|
+
end
|
12
|
+
|
11
13
|
def self.warn(message)
|
12
14
|
STDERR.puts message
|
13
15
|
end
|
14
16
|
|
15
17
|
#-------------------------------------------------------------------------#
|
16
|
-
|
17
18
|
end
|
18
19
|
end
|