apt_stage_artifacts 0.6.0 → 0.8.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/apt_add_to_freight_library.rb +19 -15
- data/lib/apt_stage_artifacts.rb +10 -10
- data/lib/apt_stage_from_tarball.rb +15 -17
- data/lib/apt_update_freight_cache.rb +6 -12
- data/lib/mixins/constants.rb +9 -2
- data/lib/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ee106eeb8e60ac6a9d942ded6cd6fed67bc799ec2c77f4740d59a6c4c0667b27
|
4
|
+
data.tar.gz: 1961201b900674777a81a9ec22959873cd5782bc5141b647b89fb893efe20356
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 988779891f6d6729d1fb70beacdd721b46ffc08a0d0cb5694f5461f3c987b839f87dcdda8fa557050232d85ce539e501759da7440638a2ffc59eac1dc10e35a0
|
7
|
+
data.tar.gz: 04e70a00aaef203277698f49e521ba0daf83c83d7b2f1752a39727e793ab53b1c6c06d25a95f1c4c07ed5c1d620bc599052a32fe311b27cc6ca45d057a6e810e
|
@@ -10,8 +10,9 @@ require 'tmpdir'
|
|
10
10
|
## This is a wrapper for https://github.com/freight-team/freight, which creates APT repos
|
11
11
|
## from deb files.
|
12
12
|
##
|
13
|
-
## In our case, we maintain a set of debian repos, based on puppet version and a
|
14
|
-
## 'nightly'
|
13
|
+
## In our case, we maintain a set of debian repos, based on puppet version and a APT component
|
14
|
+
## 'nightly' or 'stable' (we've also used 'archive' in the past; there's some remnants of it
|
15
|
+
## left).
|
15
16
|
##
|
16
17
|
## freight provides signing services, so this script needs to exist on a machine where GPG
|
17
18
|
## signing has been set up.
|
@@ -19,14 +20,14 @@ require 'tmpdir'
|
|
19
20
|
## Typical usage would be for a remote process to call this script with a list of .deb
|
20
21
|
## URLS:
|
21
22
|
##
|
22
|
-
## apt-add-to-freight-library --puppet-version=7 --
|
23
|
+
## apt-add-to-freight-library --puppet-version=7 --component=nightly --codename=stretch
|
23
24
|
## https://some-server.net/somepackage-a_0.19.2-1stretch_amd64.deb
|
24
25
|
## https://some-server.net/somepackage-b_0.19.2-1stretch_amd64.deb
|
25
26
|
##
|
26
|
-
## Alternately one
|
27
|
+
## Alternately one could scp a bunch of .deb files to the machine
|
27
28
|
## this is installed on then call this script with all the deb files as arguments. For example:
|
28
29
|
##
|
29
|
-
## apt-add-to-freight-library --puppet-version=7 --
|
30
|
+
## apt-add-to-freight-library --puppet-version=7 --component=nightly --codename=buster
|
30
31
|
## /tmp/xxtmpdir/somepackage-a_0.0.0-1buster_amd64.deb
|
31
32
|
## /tmp/xxtmpdir/somepackage-b_1.2.3-5buster_amd64.deb
|
32
33
|
## (etc.)
|
@@ -38,14 +39,14 @@ class AptAddToFreightLibrary
|
|
38
39
|
Adds debian .deb files to a freight library.
|
39
40
|
|
40
41
|
Usage:
|
41
|
-
apt-add-to-freight-library [--verbose] --puppet-version=VERSION --
|
42
|
+
apt-add-to-freight-library [--verbose] --puppet-version=VERSION --component=COMPONENT --codename=DEBIAN_CODENAME <deb_file> ...
|
42
43
|
apt-add-to-freight-library --help
|
43
44
|
apt-add-to-freight-library --version
|
44
45
|
|
45
46
|
Options:
|
47
|
+
-c --component=COMPONENT APT component (Valid: #{AptStageArtifacts::VALID_APT_COMPONENTS.join(', ')})
|
48
|
+
-n --codename=DEBIAN_CODENAME Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
|
46
49
|
-p --puppet-version=VERSION Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
|
47
|
-
-r --repo-type=REPO_TYPE Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
|
48
|
-
-c --codename=DEBIAN_CODENAME Debian codename to deliver to (Valid: #{AptStageArtifacts::VALID_CODENAMES.join(', ')})
|
49
50
|
|
50
51
|
-v --verbose Be verbose
|
51
52
|
-h --help Show this help
|
@@ -79,9 +80,9 @@ class AptAddToFreightLibrary
|
|
79
80
|
fatal "Invalid Puppet version '#{@puppet_version}'" unless
|
80
81
|
AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
|
81
82
|
|
82
|
-
@
|
83
|
-
unless AptStageArtifacts::
|
84
|
-
fatal "Invalid
|
83
|
+
@apt_component = @user_options['--component']
|
84
|
+
unless AptStageArtifacts::VALID_APT_COMPONENTS.include? @apt_component
|
85
|
+
fatal "Invalid APT component '#{@apt_component}'"
|
85
86
|
end
|
86
87
|
|
87
88
|
@codename = @user_options['--codename']
|
@@ -91,7 +92,7 @@ class AptAddToFreightLibrary
|
|
91
92
|
|
92
93
|
freight_config_file = File.join(
|
93
94
|
AptStageArtifacts::FREIGHT_CONFIG_DIRECTORY,
|
94
|
-
"
|
95
|
+
"puppet#{@puppet_version}.conf"
|
95
96
|
)
|
96
97
|
fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
|
97
98
|
File.readable?(freight_config_file)
|
@@ -101,11 +102,14 @@ class AptAddToFreightLibrary
|
|
101
102
|
|
102
103
|
Dir.mktmpdir do |temporary_directory|
|
103
104
|
deb_artifacts.each do |artifact_path|
|
104
|
-
logger.info "Adding #{artifact_path} to
|
105
|
+
logger.info "Adding #{artifact_path} to puppet#{@puppet_version}/#{@apt_component}"
|
105
106
|
deb_artifact_path = artifact_path
|
106
107
|
if deb_artifact_path.start_with?('https://', 'http://')
|
107
108
|
download_data = URI.parse(deb_artifact_path).open
|
108
|
-
download_path = File.join(
|
109
|
+
download_path = File.join(
|
110
|
+
temporary_directory,
|
111
|
+
download_data.base_uri.to_s.split('/').last
|
112
|
+
)
|
109
113
|
IO.copy_stream(download_data, download_path)
|
110
114
|
deb_artifact_path = download_path
|
111
115
|
end
|
@@ -113,7 +117,7 @@ class AptAddToFreightLibrary
|
|
113
117
|
freight_add_command = %W[
|
114
118
|
sudo --user=jenkins --set-home
|
115
119
|
#{AptStageArtifacts::FREIGHT_COMMAND} add #{@verbose} --conf="#{freight_config_file}"
|
116
|
-
#{deb_artifact_path} "apt/#{@codename}"
|
120
|
+
#{deb_artifact_path} "apt/#{@codename}/#{@apt_component}"
|
117
121
|
].join(' ')
|
118
122
|
|
119
123
|
%x(#{freight_add_command})
|
data/lib/apt_stage_artifacts.rb
CHANGED
@@ -16,7 +16,7 @@ class AptStageArtifacts
|
|
16
16
|
Stages .deb artifacts into a freight library from a Vanagon or packaging directory.
|
17
17
|
|
18
18
|
Usage:
|
19
|
-
apt-stage-artifacts [--
|
19
|
+
apt-stage-artifacts [--component=<apt_component>] [<top_directory>]
|
20
20
|
apt-stage-artifacts --version
|
21
21
|
apt-stage-artifacts --help
|
22
22
|
|
@@ -26,7 +26,7 @@ class AptStageArtifacts
|
|
26
26
|
@script_name = File.basename($PROGRAM_NAME)
|
27
27
|
@log_level = Logger::INFO
|
28
28
|
@user_options = parse_options(argv)
|
29
|
-
@
|
29
|
+
@apt_component = @user_options['--component'] || 'stable'
|
30
30
|
@version_string = "Version #{AptStageArtifacts::VERSION}"
|
31
31
|
|
32
32
|
@artifacts_tarball_name = 'artifacts.tgz'
|
@@ -47,11 +47,12 @@ class AptStageArtifacts
|
|
47
47
|
def run
|
48
48
|
logger.info @version_string
|
49
49
|
exit 0 if @user_options['--version']
|
50
|
-
unless AptStageArtifacts::
|
51
|
-
fatal "Unknown
|
52
|
-
AptStageArtifacts::
|
50
|
+
unless AptStageArtifacts::VALID_APT_COMPONENTS.include? @apt_component
|
51
|
+
fatal "Unknown APT component: \"#{@apt_component}\". Valid APT components are: " +
|
52
|
+
AptStageArtifacts::VALID_APT_COMPONENTS.join(', ')
|
53
53
|
end
|
54
54
|
load_builder_data_yaml
|
55
|
+
logger.info "Artifacts will be shipped to '#{@staging_server}'."
|
55
56
|
collect_artifacts
|
56
57
|
create_remote_staging_directory
|
57
58
|
generate_manifest
|
@@ -116,7 +117,6 @@ class AptStageArtifacts
|
|
116
117
|
yaml_content = URI.parse(@builder_data_yaml).open(&:read)
|
117
118
|
yaml_data = YAML.safe_load(yaml_content)
|
118
119
|
@staging_server = yaml_data['staging_server']
|
119
|
-
logger.info "Artifacts will be staged on '#{@staging_server}'."
|
120
120
|
end
|
121
121
|
|
122
122
|
# A manifest is a staging to-do list represented in JSON as an array of hashes
|
@@ -125,14 +125,14 @@ class AptStageArtifacts
|
|
125
125
|
# codename: debian codename for this artifact, needed by freight
|
126
126
|
# puppet_version: part of the destination apt repo. We maintain apt repos by puppet major
|
127
127
|
# version
|
128
|
-
#
|
128
|
+
# apt_component: either 'stable' or 'nightly'. 'archive' is supported but unused.
|
129
129
|
def generate_manifest
|
130
130
|
manifest = []
|
131
131
|
|
132
132
|
@debian_artifacts.each do |deb_file_path|
|
133
|
-
|
133
|
+
artifact_items = deb_file_path.split('/')
|
134
134
|
|
135
|
-
repo_name =
|
135
|
+
repo_name = artifact_items[-2]
|
136
136
|
case repo_name
|
137
137
|
when /^puppet(\d+)+$/
|
138
138
|
puppet_version = Regexp.last_match(1)
|
@@ -149,7 +149,7 @@ class AptStageArtifacts
|
|
149
149
|
deb_file_path: deb_file_path,
|
150
150
|
puppet_version: puppet_version,
|
151
151
|
codename: codename,
|
152
|
-
|
152
|
+
apt_component: @apt_component
|
153
153
|
}
|
154
154
|
end
|
155
155
|
|
@@ -13,11 +13,13 @@ class AptStageFromTarball
|
|
13
13
|
@tarball_path = tarball_path
|
14
14
|
@manifest_json_file_name = AptStageArtifacts::MANIFEST_JSON_FILENAME
|
15
15
|
@log_level = Logger::INFO
|
16
|
-
@repo_types_updated = {}
|
17
16
|
|
18
17
|
@script_name = File.basename($PROGRAM_NAME)
|
19
18
|
@version_string = "Version #{AptStageArtifacts::VERSION}"
|
20
19
|
|
20
|
+
# Keep a list of puppet versions that will require a freight-cache update
|
21
|
+
@cache_update_versions = []
|
22
|
+
|
21
23
|
# External commands
|
22
24
|
@apt_add_to_freight_library_command = 'apt-add-to-freight-library'
|
23
25
|
@apt_update_freight_cache_command = 'apt-update-freight-cache'
|
@@ -77,7 +79,7 @@ class AptStageFromTarball
|
|
77
79
|
#{@apt_add_to_freight_library_command}
|
78
80
|
--verbose
|
79
81
|
--puppet-version=#{artifact[:puppet_version]}
|
80
|
-
--
|
82
|
+
--component=#{artifact[:apt_component]}
|
81
83
|
--codename=#{artifact[:codename]}
|
82
84
|
#{File.join(Dir.pwd, artifact[:deb_file_path])}
|
83
85
|
].join(' ')
|
@@ -85,29 +87,25 @@ class AptStageFromTarball
|
|
85
87
|
%x(#{add_to_freight_library_command})
|
86
88
|
fatal "#{add_to_freight_library_command} failed." unless $CHILD_STATUS.success?
|
87
89
|
|
88
|
-
# Keep a hash, key by puppet_version, of
|
90
|
+
# Keep a hash, key by puppet_version, of apt_components so that we can
|
89
91
|
# efficiently build out the freight caches later.
|
90
|
-
|
91
|
-
repo_type = artifact[:repo_type]
|
92
|
-
@repo_types_updated[puppet_version] = [] unless @repo_types_updated.key?(puppet_version)
|
93
|
-
@repo_types_updated[puppet_version] |= [repo_type]
|
92
|
+
@cache_update_versions << artifact[:puppet_version]
|
94
93
|
end
|
95
94
|
end
|
95
|
+
@cache_update_versions = @cache_update_versions.uniq
|
96
96
|
end
|
97
97
|
|
98
|
+
# Executes the 'freight-cache' command through a local wrapper to create an APT repo
|
98
99
|
def update_freight_cache
|
99
|
-
@
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
--puppet-version=#{puppet_version}
|
105
|
-
--repo-type=#{repo_type}
|
100
|
+
@cache_update_versions.each do |puppet_version|
|
101
|
+
update_freight_cache_command = %W[
|
102
|
+
#{@apt_update_freight_cache_command}
|
103
|
+
--verbose
|
104
|
+
--puppet-version=#{puppet_version}
|
106
105
|
].join(' ')
|
107
106
|
|
108
|
-
|
109
|
-
|
110
|
-
end
|
107
|
+
%x(#{update_freight_cache_command})
|
108
|
+
fatal "#{update_freight_cache_command} failed." unless $CHILD_STATUS.success?
|
111
109
|
end
|
112
110
|
end
|
113
111
|
end
|
@@ -20,9 +20,8 @@ require 'yaml'
|
|
20
20
|
## freight provides signing services, so this script needs to exist on a machine where GPG
|
21
21
|
## signing has been set up.
|
22
22
|
##
|
23
|
-
## Example
|
24
|
-
##
|
25
|
-
## apt-update-freight-cache --puppet-version=7 --repo-type=nightly
|
23
|
+
## Example usage:
|
24
|
+
## apt-update-freight-cache --puppet-version=7
|
26
25
|
|
27
26
|
class AptUpdateFreightCache
|
28
27
|
include Logging
|
@@ -31,13 +30,12 @@ class AptUpdateFreightCache
|
|
31
30
|
Generates a freight cache from a freight library.
|
32
31
|
|
33
32
|
Usage:
|
34
|
-
apt-update-freight-cache [--verbose] --puppet-version=VERSION
|
33
|
+
apt-update-freight-cache [--verbose] --puppet-version=VERSION
|
35
34
|
apt-update-freight-cache --help
|
36
35
|
apt-update-freight-cache --version
|
37
36
|
|
38
37
|
Options:
|
39
38
|
-p --puppet-version=VERSION Puppet version (Valid: #{AptStageArtifacts::VALID_PUPPET_VERSIONS.join(', ')})
|
40
|
-
-r --repo-type=REPO_TYPE Puppet debian repo type (Valid: #{AptStageArtifacts::VALID_REPO_TYPES.join(', ')})
|
41
39
|
|
42
40
|
-v --verbose Be verbose
|
43
41
|
-h --help Show this help
|
@@ -69,15 +67,11 @@ class AptUpdateFreightCache
|
|
69
67
|
logger.info @version_string
|
70
68
|
load_builder_data_yaml
|
71
69
|
@puppet_version = @user_options['--puppet-version']
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
@repo_type = @user_options['--repo-type']
|
76
|
-
unless AptStageArtifacts::VALID_REPO_TYPES.include? @repo_type
|
77
|
-
fatal "Invalid repo type '#{@repo_type}'"
|
70
|
+
unless AptStageArtifacts::VALID_PUPPET_VERSIONS.include? @puppet_version
|
71
|
+
fatal "Invalid Puppet version '#{@puppet_version}'"
|
78
72
|
end
|
79
73
|
|
80
|
-
freight_config_file = "/etc/freight.conf.d/
|
74
|
+
freight_config_file = "/etc/freight.conf.d/puppet#{@puppet_version}.conf"
|
81
75
|
fatal "Cannot read '#{freight_config_file} on #{`hostname`.chomp}." unless
|
82
76
|
File.readable?(freight_config_file)
|
83
77
|
|
data/lib/mixins/constants.rb
CHANGED
@@ -17,6 +17,13 @@ class AptStageArtifacts
|
|
17
17
|
VALID_CODENAMES = DEBIAN_CODENAMES | UBUNTU_CODENAMES
|
18
18
|
|
19
19
|
# Validation constraints
|
20
|
-
VALID_PUPPET_VERSIONS = %w[6 7 8]
|
21
|
-
|
20
|
+
VALID_PUPPET_VERSIONS = %w[6 7 8 9]
|
21
|
+
|
22
|
+
# From freight-add:
|
23
|
+
# freight-add registers package with one or more manager/distro[/component] pairs (or triples).
|
24
|
+
# component is optional and for apt defaults to 'main'.
|
25
|
+
|
26
|
+
# These are valid APT components. We use 'stable' instead of 'main' established by convention
|
27
|
+
# from docker dists (see https://download.docker.com/linux/ubuntu/dists/groovy)
|
28
|
+
VALID_APT_COMPONENTS = %w[archive nightly stable]
|
22
29
|
end
|
data/lib/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: apt_stage_artifacts
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Puppet Release Engineering
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-09-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pry
|
@@ -179,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '0'
|
181
181
|
requirements: []
|
182
|
-
rubygems_version: 3.1.
|
182
|
+
rubygems_version: 3.1.6
|
183
183
|
signing_key:
|
184
184
|
specification_version: 4
|
185
185
|
summary: Stages .deb artifacts to a remote freight repository
|