terraspace-bundler 0.3.4 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/exe/{terraform-bundler → terraspace-bundler} +0 -0
- data/lib/terraspace_bundler/cli/purge_cache.rb +1 -1
- data/lib/terraspace_bundler/dsl/syntax.rb +4 -0
- data/lib/terraspace_bundler/exporter.rb +4 -4
- data/lib/terraspace_bundler/exporter/base.rb +1 -1
- data/lib/terraspace_bundler/exporter/copy.rb +2 -1
- data/lib/terraspace_bundler/extract.rb +21 -0
- data/lib/terraspace_bundler/extract/tar.rb +34 -0
- data/lib/terraspace_bundler/extract/zip.rb +16 -0
- data/lib/terraspace_bundler/info.rb +7 -0
- data/lib/terraspace_bundler/mod.rb +18 -7
- data/lib/terraspace_bundler/mod/concerns/local_concern.rb +10 -0
- data/lib/terraspace_bundler/mod/concerns/notation_concern.rb +42 -0
- data/lib/terraspace_bundler/mod/concerns/path_concern.rb +90 -0
- data/lib/terraspace_bundler/mod/{stack_concern.rb → concerns/stack_concern.rb} +1 -1
- data/lib/terraspace_bundler/mod/fetcher.rb +18 -0
- data/lib/terraspace_bundler/mod/fetcher/base.rb +27 -0
- data/lib/terraspace_bundler/mod/fetcher/gcs.rb +48 -0
- data/lib/terraspace_bundler/mod/{downloader.rb → fetcher/git.rb} +10 -16
- data/lib/terraspace_bundler/mod/fetcher/local.rb +15 -0
- data/lib/terraspace_bundler/mod/fetcher/s3.rb +66 -0
- data/lib/terraspace_bundler/mod/http/concern.rb +45 -0
- data/lib/terraspace_bundler/mod/http/source.rb +19 -0
- data/lib/terraspace_bundler/mod/props.rb +101 -0
- data/lib/terraspace_bundler/mod/{props_extension.rb → props/extension.rb} +6 -2
- data/lib/terraspace_bundler/mod/props/typer.rb +39 -0
- data/lib/terraspace_bundler/mod/registry.rb +77 -23
- data/lib/terraspace_bundler/terrafile.rb +3 -3
- data/lib/terraspace_bundler/version.rb +1 -1
- data/spec/fixtures/terrafiles/example/Terrafile +29 -0
- data/spec/terraform_bundler/mod/props_spec.rb +34 -0
- data/terraspace-bundler.gemspec +4 -0
- metadata +82 -9
- data/lib/terraspace_bundler/mod/path_concern.rb +0 -37
- data/lib/terraspace_bundler/mod/props_builder.rb +0 -70
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c8245c4a3b7fd5b149298bdf0084bddeee9153ff1cd7016e7e13facb0f3bede2
|
4
|
+
data.tar.gz: 761d564519e6958842cc5386e7b84f4a2b2cc90e3801fc2c3845f752983241b2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d23b49ddabd983bf1927c62aabc2a9bf4165970a32b7325e41965bc804267297cf269741677582e4c0c4f7603e15dc21fef9d5370200b18adb2f369733b653ae
|
7
|
+
data.tar.gz: 38bf841d2c9d721b34930786c46567a47e881e3951153272eb7f9dea66e23e974994071e5514a4e92091db0fb19d787ce22893bfb514464ae967fcc04bb34959
|
data/CHANGELOG.md
CHANGED
@@ -3,6 +3,10 @@
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
4
4
|
This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
|
5
5
|
|
6
|
+
## [0.4.0] - 2021-08-10
|
7
|
+
- [#10](https://github.com/boltops-tools/terraspace-bundler/pull/10) terraspace bundle info: warning when no lockfile
|
8
|
+
- [#12](https://github.com/boltops-tools/terraspace-bundler/pull/12) support for: private registry, s3, gcs, local
|
9
|
+
|
6
10
|
## [0.3.4] - 2021-04-15
|
7
11
|
- [#11](https://github.com/boltops-tools/terraspace-bundler/pull/11) improve registry dectection and gitlab repository support
|
8
12
|
|
File without changes
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module TerraspaceBundler
|
2
2
|
class Exporter
|
3
|
-
include TB::Mod::PathConcern
|
3
|
+
include TB::Mod::Concerns::PathConcern
|
4
4
|
include TB::Util::Logging
|
5
5
|
|
6
6
|
def initialize(options={})
|
@@ -25,9 +25,9 @@ module TerraspaceBundler
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def export(mod)
|
28
|
-
|
29
|
-
|
30
|
-
|
28
|
+
fetcher = Mod::Fetcher.new(mod).instance
|
29
|
+
fetcher.run
|
30
|
+
fetcher.switch_version(mod.sha)
|
31
31
|
copy = Copy.new(mod)
|
32
32
|
copy.mod
|
33
33
|
copy.stacks
|
@@ -7,6 +7,7 @@ class TerraspaceBundler::Exporter
|
|
7
7
|
def mod
|
8
8
|
FileUtils.rm_rf(mod_path)
|
9
9
|
FileUtils.mkdir_p(File.dirname(mod_path))
|
10
|
+
logger.debug "Copy: cp -r #{src_path} #{mod_path}"
|
10
11
|
FileUtils.cp_r(src_path, mod_path)
|
11
12
|
FileUtils.rm_rf("#{mod_path}/.git")
|
12
13
|
end
|
@@ -17,7 +18,7 @@ class TerraspaceBundler::Exporter
|
|
17
18
|
|
18
19
|
# src path is from the stage area
|
19
20
|
def src_path
|
20
|
-
path = stage_path(
|
21
|
+
path = stage_path(rel_dest_dir)
|
21
22
|
path = "#{path}/#{@mod.subfolder}" if @mod.subfolder
|
22
23
|
path
|
23
24
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module TerraspaceBundler
|
2
|
+
class Extract
|
3
|
+
def self.extract(archive, dest)
|
4
|
+
FileUtils.rm_rf(dest)
|
5
|
+
FileUtils.mkdir_p(File.dirname(dest))
|
6
|
+
|
7
|
+
if archive.ends_with?('.tgz') || archive.ends_with?('.tar.gz')
|
8
|
+
Tar.extract(archive, dest)
|
9
|
+
elsif archive.ends_with?('.zip')
|
10
|
+
Zip.extract(archive, dest)
|
11
|
+
else
|
12
|
+
puts <<~EOL.color(:red)
|
13
|
+
ERROR: Unable to extract. Unsupported archive extension for:
|
14
|
+
|
15
|
+
#{archive}
|
16
|
+
EOL
|
17
|
+
exit 1
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'rubygems/package'
|
2
|
+
require 'zlib'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
# Thanks: https://gist.github.com/ForeverZer0/2adbba36fd452738e7cca6a63aee2f30
|
6
|
+
class TerraspaceBundler::Extract
|
7
|
+
class Tar
|
8
|
+
# tar_gz_archive = '/tmp/terraspace/bundler/stage/s3-us-west-2.amazonaws.com/demo-terraform-test/modules/example-module.tgz'
|
9
|
+
# destination = '/tmp/terraspace/where/extract/to'
|
10
|
+
def self.extract(tar_gz_archive, destination)
|
11
|
+
reader = Zlib::GzipReader.open(tar_gz_archive)
|
12
|
+
Gem::Package::TarReader.new(reader) do |tar|
|
13
|
+
dest = nil
|
14
|
+
tar.each do |entry|
|
15
|
+
dest ||= File.join destination, entry.full_name
|
16
|
+
if entry.directory?
|
17
|
+
FileUtils.rm_rf dest unless File.directory? dest
|
18
|
+
FileUtils.mkdir_p dest, :mode => entry.header.mode, :verbose => false
|
19
|
+
elsif entry.file?
|
20
|
+
FileUtils.rm_rf dest unless File.file? dest
|
21
|
+
File.open dest, "wb" do |f|
|
22
|
+
f.print entry.read
|
23
|
+
end
|
24
|
+
FileUtils.chmod entry.header.mode, dest, :verbose => false
|
25
|
+
elsif entry.header.typeflag == '2' #Symlink!
|
26
|
+
File.symlink entry.header.linkname, dest
|
27
|
+
end
|
28
|
+
dest = nil
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'zip'
|
3
|
+
|
4
|
+
class TerraspaceBundler::Extract
|
5
|
+
class Zip
|
6
|
+
def self.extract(file, dest)
|
7
|
+
::Zip::File.open(file) { |zip_file|
|
8
|
+
zip_file.each { |f|
|
9
|
+
f_path=File.join(dest, f.name)
|
10
|
+
FileUtils.mkdir_p(File.dirname(f_path))
|
11
|
+
zip_file.extract(f, f_path) unless File.exist?(f_path)
|
12
|
+
}
|
13
|
+
}
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -1,6 +1,13 @@
|
|
1
1
|
module TerraspaceBundler
|
2
2
|
class Info < TB::CLI::Base
|
3
3
|
def run
|
4
|
+
file = TB.config.lockfile
|
5
|
+
unless File.exist?(file)
|
6
|
+
logger.info "No #{file} found".color(:red)
|
7
|
+
logger.info "Maybe run: terraspace bundle"
|
8
|
+
return
|
9
|
+
end
|
10
|
+
|
4
11
|
name = @options[:mod]
|
5
12
|
found = lockfile.mods.find { |m| m.name == @options[:mod] }
|
6
13
|
if found
|
@@ -1,9 +1,10 @@
|
|
1
1
|
module TerraspaceBundler
|
2
2
|
class Mod
|
3
|
-
extend
|
4
|
-
props :export_to, :name, :sha, :source, :subfolder, :type, :url
|
3
|
+
extend Props::Extension
|
4
|
+
props :export_to, :name, :sha, :source, :subfolder, :type, :url, :clone_with
|
5
5
|
|
6
|
-
include StackConcern
|
6
|
+
include Concerns::StackConcern
|
7
|
+
include Concerns::LocalConcern
|
7
8
|
|
8
9
|
attr_reader :props, :version, :ref, :tag, :branch
|
9
10
|
def initialize(props={})
|
@@ -19,7 +20,7 @@ module TerraspaceBundler
|
|
19
20
|
|
20
21
|
# use url instead of source because for registry modules, the repo name is different
|
21
22
|
def repo
|
22
|
-
url_words[-1]
|
23
|
+
url_words[-1].sub(/\.git$/,'')
|
23
24
|
end
|
24
25
|
|
25
26
|
# https://github.com/tongueroo/pet - 2nd to last word
|
@@ -34,9 +35,19 @@ module TerraspaceBundler
|
|
34
35
|
end
|
35
36
|
|
36
37
|
def latest_sha
|
37
|
-
|
38
|
-
|
39
|
-
|
38
|
+
fetcher = Fetcher.new(self).instance
|
39
|
+
fetcher.run
|
40
|
+
fetcher.sha
|
41
|
+
end
|
42
|
+
|
43
|
+
def vcs_provider
|
44
|
+
if url.include?('http')
|
45
|
+
# "https://github.com/org/repo" => github.com
|
46
|
+
url.match(%r{http[s]?://(.*?)/})[1]
|
47
|
+
else # git@
|
48
|
+
# "git@github.com:org/repo" => github.com
|
49
|
+
url.match(%r{git@(.*?):})[1]
|
50
|
+
end
|
40
51
|
end
|
41
52
|
|
42
53
|
private
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module TerraspaceBundler::Mod::Concerns
|
4
|
+
module NotationConcern
|
5
|
+
def remove_notations(source)
|
6
|
+
remove_subfolder_notation(remove_ref_notation(source))
|
7
|
+
end
|
8
|
+
|
9
|
+
def remove_ref_notation(source)
|
10
|
+
source.sub(/\?.*/,'')
|
11
|
+
end
|
12
|
+
|
13
|
+
def remove_subfolder_notation(source)
|
14
|
+
parts = clean_notation(source).split('//')
|
15
|
+
if parts.size == 2 # has subfolder
|
16
|
+
source.split('//')[0..-2].join('//') # remove only subfolder, keep rest of original source
|
17
|
+
else
|
18
|
+
source
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def subfolder(source)
|
23
|
+
parts = clean_notation(source).split('//')
|
24
|
+
if parts.size == 2 # has subfolder
|
25
|
+
remove_ref_notation(parts.last)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def ref(source)
|
30
|
+
url = clean_notation(source)
|
31
|
+
uri = URI(url)
|
32
|
+
if uri.query
|
33
|
+
params = URI::decode_www_form(uri.query).to_h # if you are in 2.1 or later version of Ruby
|
34
|
+
params['ref']
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def clean_notation(source)
|
39
|
+
source.sub(/.*::/,'').sub(%r{http[s?]://},'').sub(%r{git@(.*?):},'') # also remove git@ notation
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
module TerraspaceBundler::Mod::Concerns
|
2
|
+
module PathConcern
|
3
|
+
def setup_tmp
|
4
|
+
FileUtils.mkdir_p(cache_root)
|
5
|
+
FileUtils.mkdir_p(stage_root)
|
6
|
+
end
|
7
|
+
|
8
|
+
def tmp_root
|
9
|
+
"/tmp/terraspace/bundler"
|
10
|
+
end
|
11
|
+
|
12
|
+
def cache_root
|
13
|
+
"#{tmp_root}/cache"
|
14
|
+
end
|
15
|
+
|
16
|
+
def stage_root
|
17
|
+
"#{tmp_root}/stage"
|
18
|
+
end
|
19
|
+
|
20
|
+
def cache_path(name)
|
21
|
+
[cache_root, parent_stage_folder, name].compact.join('/')
|
22
|
+
end
|
23
|
+
|
24
|
+
def stage_path(name)
|
25
|
+
[stage_root, parent_stage_folder, name].compact.join('/')
|
26
|
+
end
|
27
|
+
|
28
|
+
# Fetcher: Downloader/Local copies to a slightly different folder.
|
29
|
+
# Also, Copy will use this and reference same method so it's consistent.
|
30
|
+
def rel_dest_dir
|
31
|
+
case @mod.type
|
32
|
+
when 'local'
|
33
|
+
@mod.name # example-module
|
34
|
+
when 's3'
|
35
|
+
path = type_path # https://s3-us-west-2.amazonaws.com/demo-terraform-test/example-module.zip
|
36
|
+
remove_ext(path) # demo-terraform-test/modules/example-module
|
37
|
+
when 'gcs'
|
38
|
+
path = type_path # https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH/TO/module.zip
|
39
|
+
path.sub!(%r{storage/v\d+/},'')
|
40
|
+
remove_ext(path) # terraform-example-modules/modules/example-module
|
41
|
+
when 'http'
|
42
|
+
path = type_path # https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH/TO/module.zip
|
43
|
+
remove_ext(path) # terraform-example-modules/modules/example-module
|
44
|
+
when -> (_) { @mod.source.include?('git::') }
|
45
|
+
@mod.name # example-module
|
46
|
+
else # inferred git, registry
|
47
|
+
@mod.full_repo # tongueroo/example-module
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def parent_stage_folder
|
52
|
+
case @mod.type
|
53
|
+
when 'local'
|
54
|
+
'local'
|
55
|
+
when 'http'
|
56
|
+
'http'
|
57
|
+
else # gcs, s3, git, registry
|
58
|
+
@mod.vcs_provider
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def type_path
|
63
|
+
source = @mod.source
|
64
|
+
url = source.sub("#{@mod.type}::",'')
|
65
|
+
uri = URI(url)
|
66
|
+
uri.path.sub('/','') # removing leading slash to bucket name is the first thing
|
67
|
+
.sub(%r{//(.*)},'') # remove subfolder
|
68
|
+
end
|
69
|
+
|
70
|
+
def remove_ext(path)
|
71
|
+
ext = File.extname(path)
|
72
|
+
path.sub(ext,'')
|
73
|
+
end
|
74
|
+
|
75
|
+
def get_bucket_key(path)
|
76
|
+
bucket, *rest = path.split('/')
|
77
|
+
key = rest.join('/')
|
78
|
+
[bucket, key]
|
79
|
+
end
|
80
|
+
|
81
|
+
def mod_path
|
82
|
+
get_mod_path(@mod)
|
83
|
+
end
|
84
|
+
|
85
|
+
def get_mod_path(mod)
|
86
|
+
export_to = mod.export_to || TB.config.export_to
|
87
|
+
"#{export_to}/#{mod.name}"
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# Delegates to:
|
2
|
+
#
|
3
|
+
# 1. Local
|
4
|
+
# 2. Git
|
5
|
+
#
|
6
|
+
class TerraspaceBundler::Mod
|
7
|
+
class Fetcher
|
8
|
+
def initialize(mod)
|
9
|
+
@mod = mod
|
10
|
+
end
|
11
|
+
|
12
|
+
def instance
|
13
|
+
type = @mod.type == "registry" ? "git" : @mod.type
|
14
|
+
klass = "TerraspaceBundler::Mod::Fetcher::#{type.camelize}".constantize
|
15
|
+
klass.new(@mod) # IE: Local.new(@mod), Git.new(@mod), S3.new(@mod), etc
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# Interface of subclasses should implement
|
2
|
+
#
|
3
|
+
# run
|
4
|
+
# switch_version(mod.sha)
|
5
|
+
# sha
|
6
|
+
#
|
7
|
+
class TerraspaceBundler::Mod::Fetcher
|
8
|
+
class Base
|
9
|
+
include TB::Util::Git
|
10
|
+
include TB::Util::Logging
|
11
|
+
include TB::Mod::Concerns::PathConcern
|
12
|
+
|
13
|
+
attr_reader :sha # returns nil for Local
|
14
|
+
def initialize(mod)
|
15
|
+
@mod = mod
|
16
|
+
end
|
17
|
+
|
18
|
+
def switch_version(*)
|
19
|
+
# noop
|
20
|
+
end
|
21
|
+
|
22
|
+
def extract(archive, dest)
|
23
|
+
TerraspaceBundler::Extract.extract(archive, dest)
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require "google/cloud/storage"
|
2
|
+
|
3
|
+
class TerraspaceBundler::Mod::Fetcher
|
4
|
+
class Gcs < Base
|
5
|
+
extend Memoist
|
6
|
+
|
7
|
+
def run
|
8
|
+
bucket, key, path = gcs_info
|
9
|
+
download(bucket, key, path)
|
10
|
+
end
|
11
|
+
|
12
|
+
def download(bucket_name, key, path)
|
13
|
+
# Download to cache area
|
14
|
+
bucket = storage.bucket(bucket_name)
|
15
|
+
unless bucket
|
16
|
+
logger.error "ERROR: bucket #{bucket_name} does not exist".color(:red)
|
17
|
+
exit 1
|
18
|
+
end
|
19
|
+
file = bucket.file(key)
|
20
|
+
unless file
|
21
|
+
logger.error "ERROR: Unable to find gs://#{bucket_name}/#{key}".color(:red)
|
22
|
+
exit 1
|
23
|
+
end
|
24
|
+
|
25
|
+
archive = cache_path(path) # temporary path
|
26
|
+
logger.debug "Gcs save archive to #{archive}"
|
27
|
+
FileUtils.mkdir_p(File.dirname(archive))
|
28
|
+
file.download(archive)
|
29
|
+
|
30
|
+
# Save to stage
|
31
|
+
dest = stage_path(rel_dest_dir)
|
32
|
+
extract(archive, dest)
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
def gcs_info
|
37
|
+
path = type_path
|
38
|
+
path.sub!(%r{storage/v\d+/},'')
|
39
|
+
bucket, key = get_bucket_key(path)
|
40
|
+
[bucket, key, path]
|
41
|
+
end
|
42
|
+
|
43
|
+
def storage
|
44
|
+
Google::Cloud::Storage.new
|
45
|
+
end
|
46
|
+
memoize :storage
|
47
|
+
end
|
48
|
+
end
|