lexicon-common 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lexicon-common.gemspec +2 -0
- data/lib/lexicon/common/database/database.rb +24 -2
- data/lib/lexicon/common/mixin/logger_aware.rb +2 -2
- data/lib/lexicon/common/mixin/nameable.rb +16 -0
- data/lib/lexicon/common/package/directory_package_loader.rb +56 -12
- data/lib/lexicon/common/package/package.rb +11 -43
- data/lib/lexicon/common/package/package_file.rb +50 -0
- data/lib/lexicon/common/package/v1/package.rb +83 -0
- data/lib/lexicon/common/package/v1/package_builder.rb +70 -0
- data/lib/lexicon/common/package/v1/source_file_set.rb +28 -0
- data/lib/lexicon/common/package/v2/package.rb +56 -0
- data/lib/lexicon/common/package/v2/package_builder.rb +72 -0
- data/lib/lexicon/common/package/v2/source_file_set.rb +26 -0
- data/lib/lexicon/common/production/datasource_loader.rb +119 -60
- data/lib/lexicon/common/production/table_locker.rb +58 -0
- data/lib/lexicon/common/psql.rb +47 -0
- data/lib/lexicon/common/remote/package_downloader.rb +61 -43
- data/lib/lexicon/common/remote/package_uploader.rb +37 -28
- data/lib/lexicon/common/remote/s3_client.rb +6 -0
- data/lib/lexicon/common/schema/validator_factory.rb +1 -1
- data/lib/lexicon/common/shell_executor.rb +0 -3
- data/lib/lexicon/common/version.rb +1 -1
- data/lib/lexicon/common.rb +1 -0
- data/lib/lexicon-common.rb +3 -0
- data/resources/lexicon.schema.json +116 -31
- metadata +45 -9
- data/lib/lexicon/common/package/package_builder.rb +0 -68
- data/lib/lexicon/common/package/source_file_set.rb +0 -24
@@ -1,61 +1,139 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
using Corindon::Result::Ext
|
4
|
+
|
3
5
|
module Lexicon
|
4
6
|
module Common
|
5
7
|
module Production
|
6
8
|
class DatasourceLoader
|
9
|
+
include Mixin::LoggerAware
|
7
10
|
include Mixin::SchemaNamer
|
11
|
+
|
8
12
|
# @param [ShellExecutor] shell
|
9
13
|
# @param [Database::Factory] database_factory
|
10
14
|
# @param [FileLoader] file_loader
|
11
15
|
# @param [String] database_url
|
12
|
-
|
16
|
+
# @param [TableLocker] table_locker
|
17
|
+
# @param [Psql] psql
|
18
|
+
def initialize(shell:, database_factory:, file_loader:, database_url:, table_locker:, psql:)
|
13
19
|
@shell = shell
|
14
20
|
@database_factory = database_factory
|
15
21
|
@file_loader = file_loader
|
16
22
|
@database_url = database_url
|
23
|
+
@table_locker = table_locker
|
24
|
+
@psql = psql
|
17
25
|
end
|
18
26
|
|
19
27
|
# @param [Package::Package] package
|
20
28
|
# @param [Array<String>, nil] only
|
21
|
-
# @param [Array<String>] without
|
22
29
|
# If nil, all datasets are loaded.
|
23
30
|
# If present, only listed datasets are loaded.
|
24
31
|
# Structures are ALWAYS loaded
|
32
|
+
# @param [Array<String>] without
|
25
33
|
def load_package(package, only: nil, without: [])
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
34
|
+
case package.schema_version
|
35
|
+
when 1
|
36
|
+
load_v1(package, only: only, without: without)
|
37
|
+
when 2
|
38
|
+
load_v2(package, only: only, without: without)
|
39
|
+
else
|
40
|
+
log("Schema version #{package.schema_version} is not supported")
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
30
45
|
|
31
|
-
|
32
|
-
|
46
|
+
# @param [Package::V1::Package] package
|
47
|
+
def load_v1(package, only: nil, without: [])
|
48
|
+
file_sets = filter_file_sets(package.file_sets, only: only, without: without)
|
49
|
+
.unwrap!
|
50
|
+
.select(&:data_path)
|
33
51
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
52
|
+
load_structure_files(
|
53
|
+
package.files.select(&:structure?).map(&:path),
|
54
|
+
schema: version_to_schema(package.version),
|
55
|
+
dir: package.dir
|
56
|
+
)
|
38
57
|
|
39
|
-
|
40
|
-
.select(&:data_path)
|
41
|
-
end
|
58
|
+
remaining = ::Concurrent::Set.new(file_sets.map(&:name))
|
42
59
|
|
43
|
-
|
60
|
+
file_sets.map do |fs|
|
61
|
+
Thread.new do
|
62
|
+
file_loader.load_file(package.data_path(fs))
|
63
|
+
remaining.delete(fs.name)
|
64
|
+
|
65
|
+
puts '[ OK ] '.green + fs.name.yellow + ", #{remaining_message(remaining)}"
|
66
|
+
end
|
67
|
+
end.each(&:join)
|
44
68
|
|
45
|
-
|
69
|
+
table_locker.lock_tables(package: package, tables: package.file_sets.flat_map(&:tables))
|
70
|
+
end
|
46
71
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
72
|
+
def remaining_message(remaining)
|
73
|
+
if remaining.size.zero?
|
74
|
+
'All done!'
|
75
|
+
elsif remaining.size > 5
|
76
|
+
"#{remaining.size} remaining"
|
77
|
+
else
|
78
|
+
"Remaining: #{remaining.to_a.sort.join(', ')}"
|
52
79
|
end
|
53
|
-
end
|
80
|
+
end
|
54
81
|
|
55
|
-
|
56
|
-
|
82
|
+
# @param [Package::V2::Package] package
|
83
|
+
# @param [Array<String>, nil] only
|
84
|
+
# @param [Array<String>] without
|
85
|
+
def load_v2(package, only: nil, without: [])
|
86
|
+
file_sets = filter_file_sets(package.file_sets, only: only, without: without)
|
87
|
+
.unwrap!
|
88
|
+
.select { |fs| fs.tables.any? }
|
57
89
|
|
58
|
-
|
90
|
+
schema = version_to_schema(package.version)
|
91
|
+
|
92
|
+
load_structure_files(package.files.select(&:structure?).map(&:path), schema: schema, dir: package.dir)
|
93
|
+
|
94
|
+
remaining = ::Concurrent::Set.new(file_sets.flat_map{|fs| fs.tables.values.flatten(1) })
|
95
|
+
|
96
|
+
threads = file_sets.flat_map do |fs|
|
97
|
+
fs.tables.flat_map do |name, files|
|
98
|
+
files.map do |file|
|
99
|
+
Thread.new do
|
100
|
+
load_csv(package.data_dir.join(file), into: name, schema: schema)
|
101
|
+
remaining.delete(file)
|
102
|
+
|
103
|
+
puts '[ OK ] '.green + file.to_s.yellow + ", #{remaining_message(remaining)}"
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
threads.each(&:join)
|
110
|
+
end
|
111
|
+
|
112
|
+
# @param [Array<Package::Mixin::Nameable>] file_sets
|
113
|
+
# @param [Array<String>, nil] only
|
114
|
+
# @param [Array<String>] without
|
115
|
+
# @return [Corindon::Result::Result]
|
116
|
+
def filter_file_sets(file_sets, only:, without:)
|
117
|
+
sets = if only.nil?
|
118
|
+
file_sets
|
119
|
+
else
|
120
|
+
sets_by_name = file_sets.map { |fs| [fs.name, fs] }.to_h
|
121
|
+
|
122
|
+
missing, present = only.map { |name| [name, sets_by_name.fetch(name, nil)] }
|
123
|
+
.partition { |(_name, value)| value.nil? }
|
124
|
+
|
125
|
+
if missing.any?
|
126
|
+
puts "[ NOK ] Datasources #{missing.map(&:first).join(', ')} don't exist!"
|
127
|
+
|
128
|
+
return Failure(StandardError.new("Datasources #{missing.map(&:first).join(', ')} don't exist!"))
|
129
|
+
end
|
130
|
+
|
131
|
+
present.map(&:second)
|
132
|
+
.select(&:data_path)
|
133
|
+
end
|
134
|
+
|
135
|
+
Success(sets.reject { |fs| without.include?(fs.name) })
|
136
|
+
end
|
59
137
|
|
60
138
|
# @return [Database::Factory]
|
61
139
|
attr_reader :database_factory
|
@@ -65,44 +143,25 @@ module Lexicon
|
|
65
143
|
attr_reader :file_loader
|
66
144
|
# @return [String]
|
67
145
|
attr_reader :database_url
|
146
|
+
# @return [TableLocker]
|
147
|
+
attr_reader :table_locker
|
148
|
+
# @return [Psql]
|
149
|
+
attr_reader :psql
|
68
150
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
151
|
+
# @param [Pathname] file
|
152
|
+
# @param [String] into
|
153
|
+
# @param [String] schema
|
154
|
+
def load_csv(file, into:, schema:)
|
155
|
+
psql.execute_raw(<<~SQL)
|
156
|
+
\\copy "#{schema}"."#{into}" FROM PROGRAM 'zcat < #{file}' WITH csv
|
157
|
+
SQL
|
76
158
|
end
|
77
159
|
|
78
|
-
|
79
|
-
def lock_tables(package)
|
160
|
+
def load_structure_files(files, schema:, dir:)
|
80
161
|
database = database_factory.new_instance(url: database_url)
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
database.prepend_search_path schema do
|
85
|
-
database.query <<~SQL
|
86
|
-
CREATE OR REPLACE FUNCTION #{schema}.deny_changes()
|
87
|
-
RETURNS TRIGGER
|
88
|
-
AS $$
|
89
|
-
BEGIN
|
90
|
-
RAISE EXCEPTION '% denied on % (master data)', TG_OP, TG_RELNAME;
|
91
|
-
END;
|
92
|
-
$$
|
93
|
-
LANGUAGE plpgsql;
|
94
|
-
SQL
|
95
|
-
package.file_sets.flat_map(&:tables).each do |table_name|
|
96
|
-
database.query <<~SQL
|
97
|
-
CREATE TRIGGER deny_changes
|
98
|
-
BEFORE INSERT
|
99
|
-
OR UPDATE
|
100
|
-
OR DELETE
|
101
|
-
OR TRUNCATE
|
102
|
-
ON #{schema}.#{table_name}
|
103
|
-
FOR EACH STATEMENT
|
104
|
-
EXECUTE PROCEDURE #{schema}.deny_changes()
|
105
|
-
SQL
|
162
|
+
database.prepend_search_path(schema) do
|
163
|
+
files.each do |file|
|
164
|
+
database.query(dir.join(file).read)
|
106
165
|
end
|
107
166
|
end
|
108
167
|
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Lexicon
|
4
|
+
module Common
|
5
|
+
module Production
|
6
|
+
class TableLocker
|
7
|
+
include Mixin::SchemaNamer
|
8
|
+
|
9
|
+
# @param [Database::Factory] database_factory
|
10
|
+
# @param [String] database_url
|
11
|
+
def initialize(database_factory:, database_url:)
|
12
|
+
@database_factory = database_factory
|
13
|
+
@database_url = database_url
|
14
|
+
end
|
15
|
+
|
16
|
+
# @param [Package::Package] package
|
17
|
+
# @param [Array<String>] tables
|
18
|
+
def lock_tables(package:, tables: [])
|
19
|
+
database = database_factory.new_instance(url: database_url)
|
20
|
+
|
21
|
+
schema = version_to_schema(package.version)
|
22
|
+
|
23
|
+
database.prepend_search_path schema do
|
24
|
+
database.query <<~SQL
|
25
|
+
CREATE OR REPLACE FUNCTION #{schema}.deny_changes()
|
26
|
+
RETURNS TRIGGER
|
27
|
+
AS $$
|
28
|
+
BEGIN
|
29
|
+
RAISE EXCEPTION '% denied on % (master data)', TG_OP, TG_RELNAME;
|
30
|
+
END;
|
31
|
+
$$
|
32
|
+
LANGUAGE plpgsql;
|
33
|
+
SQL
|
34
|
+
tables.each do |table_name|
|
35
|
+
database.query <<~SQL
|
36
|
+
CREATE TRIGGER deny_changes
|
37
|
+
BEFORE INSERT
|
38
|
+
OR UPDATE
|
39
|
+
OR DELETE
|
40
|
+
OR TRUNCATE
|
41
|
+
ON #{schema}.#{table_name}
|
42
|
+
FOR EACH STATEMENT
|
43
|
+
EXECUTE PROCEDURE #{schema}.deny_changes()
|
44
|
+
SQL
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
# @return [Database::Factory]
|
52
|
+
attr_reader :database_factory
|
53
|
+
# @return [String]
|
54
|
+
attr_reader :database_url
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Lexicon
|
4
|
+
module Common
|
5
|
+
class Psql
|
6
|
+
# @param [String] url
|
7
|
+
# @param [ShellExecutor] executor
|
8
|
+
def initialize(url:, executor:)
|
9
|
+
@url = url
|
10
|
+
@executor = executor
|
11
|
+
end
|
12
|
+
|
13
|
+
# @param [String] command
|
14
|
+
# @param [String, Array<String>] search_path
|
15
|
+
def execute(command, search_path:)
|
16
|
+
command = <<~SQL
|
17
|
+
SET search_path TO #{Array(search_path).join(', ')};
|
18
|
+
#{command}
|
19
|
+
SQL
|
20
|
+
|
21
|
+
execute_raw(command)
|
22
|
+
end
|
23
|
+
|
24
|
+
# @param [String] command
|
25
|
+
def execute_raw(command)
|
26
|
+
@executor.execute <<~BASH
|
27
|
+
psql '#{url}' --quiet -c #{Shellwords.escape(command)}
|
28
|
+
BASH
|
29
|
+
end
|
30
|
+
|
31
|
+
# @param [Pathname] file
|
32
|
+
# @param [String, Array<String>] search_path
|
33
|
+
def load_sql(file, search_path:)
|
34
|
+
@executor.execute <<~BASH
|
35
|
+
echo 'SET SEARCH_PATH TO #{Array(search_path).join(', ')};' | cat - #{file} | psql '#{url}'
|
36
|
+
BASH
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
# @return [ShellExecutor]
|
42
|
+
attr_reader :executor
|
43
|
+
# @return [String]
|
44
|
+
attr_reader :url
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
using Corindon::Result::Ext
|
4
|
+
|
3
5
|
module Lexicon
|
4
6
|
module Common
|
5
7
|
module Remote
|
@@ -9,71 +11,87 @@ module Lexicon
|
|
9
11
|
# @param [DirectoryPackageLoader] package_loader
|
10
12
|
def initialize(s3:, out_dir:, package_loader:)
|
11
13
|
super(s3: s3)
|
14
|
+
|
12
15
|
@out_dir = out_dir
|
13
16
|
@package_loader = package_loader
|
14
17
|
end
|
15
18
|
|
16
19
|
# @param [Semantic::Version] version
|
17
|
-
# @return [
|
20
|
+
# @return [Corindon::Result::Result]
|
18
21
|
def download(version)
|
19
|
-
|
22
|
+
rescue_failure do
|
23
|
+
bucket = version.to_s
|
20
24
|
|
21
|
-
|
22
|
-
|
23
|
-
|
25
|
+
if s3.bucket_exist?(bucket)
|
26
|
+
Dir.mktmpdir(nil, out_dir) do |tmp_dir|
|
27
|
+
tmp_dir = Pathname.new(tmp_dir)
|
24
28
|
|
25
|
-
|
26
|
-
bucket: bucket,
|
27
|
-
key: Package::Package::SPEC_FILE_NAME,
|
28
|
-
response_target: tmp_dir.join(Package::Package::SPEC_FILE_NAME).to_s
|
29
|
-
)
|
30
|
-
s3.raw.get_object(
|
31
|
-
bucket: bucket,
|
32
|
-
key: Package::Package::CHECKSUM_FILE_NAME,
|
33
|
-
response_target: tmp_dir.join(Package::Package::CHECKSUM_FILE_NAME).to_s
|
34
|
-
)
|
29
|
+
download_spec_files(bucket, tmp_dir).unwrap!
|
35
30
|
|
36
|
-
|
37
|
-
|
38
|
-
|
31
|
+
package = package_loader.load_package(tmp_dir.basename.to_s)
|
32
|
+
if !package.nil?
|
33
|
+
puts "[ OK ] Found package with key #{version}, version is #{package.version}".green
|
39
34
|
|
40
|
-
|
35
|
+
download_data_files(package, bucket).unwrap!
|
41
36
|
|
42
|
-
|
43
|
-
|
44
|
-
s3.raw.get_object(bucket: bucket, key: "data/#{file.basename.to_s}", response_target: file.to_s)
|
45
|
-
puts "[ OK ] Downloaded #{file.basename}".green
|
46
|
-
end
|
47
|
-
end.each(&:join)
|
37
|
+
dest_dir = out_dir.join(version.to_s)
|
38
|
+
FileUtils.mkdir_p(dest_dir)
|
48
39
|
|
49
|
-
|
50
|
-
|
51
|
-
path = package.data_path(fs)
|
52
|
-
s3.raw.get_object(bucket: bucket, key: "data/#{path.basename.to_s}", response_target: path.to_s)
|
53
|
-
puts "[ OK ] Downloaded #{path.basename}".green
|
40
|
+
tmp_dir.children.each do |child|
|
41
|
+
FileUtils.mv(child.to_s, dest_dir.join(child.basename).to_s)
|
54
42
|
end
|
55
|
-
end.each(&:join)
|
56
|
-
|
57
|
-
dest_dir = out_dir.join(version.to_s)
|
58
|
-
FileUtils.mkdir_p(dest_dir)
|
59
|
-
tmp_dir.children.each do |child|
|
60
|
-
FileUtils.mv(child.to_s, dest_dir.join(child.basename).to_s)
|
61
|
-
end
|
62
43
|
|
63
|
-
|
64
|
-
|
65
|
-
|
44
|
+
Success(package)
|
45
|
+
else
|
46
|
+
puts "[ NOK ] The remote contains a bucket '#{version}' but it does not contains a valid package.".red
|
66
47
|
|
67
|
-
|
48
|
+
Failure(StandardError.new("The folder #{bucket} on the server does not contain a valid package"))
|
49
|
+
end
|
68
50
|
end
|
51
|
+
else
|
52
|
+
Failure(StandardError.new("The server does not have a directory named #{bucket}"))
|
69
53
|
end
|
70
|
-
else
|
71
|
-
false
|
72
54
|
end
|
73
55
|
end
|
74
56
|
|
75
57
|
private
|
76
58
|
|
59
|
+
def download_data_files(package, bucket)
|
60
|
+
rescue_failure do
|
61
|
+
threads = package.files.map do |file|
|
62
|
+
Thread.new do
|
63
|
+
destination = package.dir.join(file.path)
|
64
|
+
FileUtils.mkdir_p(destination.dirname)
|
65
|
+
|
66
|
+
s3.raw.get_object(bucket: bucket, key: file.to_s, response_target: destination)
|
67
|
+
|
68
|
+
puts "[ OK ] Downloaded #{file}".green
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
threads.each(&:join)
|
73
|
+
|
74
|
+
Success(nil)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def download_spec_files(bucket, tmp_dir)
|
79
|
+
rescue_failure do
|
80
|
+
s3.raw.get_object(
|
81
|
+
bucket: bucket,
|
82
|
+
key: Package::Package::SPEC_FILE_NAME,
|
83
|
+
response_target: tmp_dir.join(Package::Package::SPEC_FILE_NAME).to_s
|
84
|
+
)
|
85
|
+
s3.raw.get_object(
|
86
|
+
bucket: bucket,
|
87
|
+
key: Package::Package::CHECKSUM_FILE_NAME,
|
88
|
+
response_target: tmp_dir.join(Package::Package::CHECKSUM_FILE_NAME).to_s
|
89
|
+
)
|
90
|
+
|
91
|
+
Success(nil)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
77
95
|
# @return [DirectoryPackageLoader]
|
78
96
|
attr_reader :package_loader
|
79
97
|
# @return [Pathname]
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
using Corindon::Result::Ext
|
4
|
+
|
3
5
|
module Lexicon
|
4
6
|
module Common
|
5
7
|
module Remote
|
@@ -7,51 +9,58 @@ module Lexicon
|
|
7
9
|
include Mixin::LoggerAware
|
8
10
|
|
9
11
|
# @param [Package] package
|
10
|
-
# @return [
|
12
|
+
# @return [Corindon::Result::Result]
|
11
13
|
def upload(package)
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
14
|
+
rescue_failure do
|
15
|
+
bucket_name = package.version.to_s
|
16
|
+
|
17
|
+
if s3.bucket_exist?(bucket_name)
|
18
|
+
Failure(StandardError.new("The server already has a folder named #{bucket_name}"))
|
19
|
+
else
|
20
|
+
upload_package(package, bucket_name)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
16
24
|
|
17
|
-
|
18
|
-
puts '[ OK ] Structure uploaded.'.green
|
25
|
+
private
|
19
26
|
|
20
|
-
|
21
|
-
|
22
|
-
|
27
|
+
# @return [Corindon::Result::Result]
|
28
|
+
def upload_package(package, bucket_name)
|
29
|
+
s3.raw.create_bucket(bucket: bucket_name)
|
23
30
|
|
24
|
-
|
25
|
-
puts "[ OK ] #{path.basename}".green
|
26
|
-
end
|
31
|
+
relative_paths = [*base_files, *package.files.map(&:path)]
|
27
32
|
|
28
|
-
upload_files(
|
33
|
+
upload_files(*relative_paths, from: package.dir, bucket: bucket_name) do |path|
|
29
34
|
puts "[ OK ] #{path.basename}".green
|
30
35
|
end
|
31
36
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
end
|
36
|
-
rescue StandardError => e
|
37
|
-
log_error(e)
|
38
|
-
|
39
|
-
false
|
40
|
-
end
|
37
|
+
Success(package)
|
38
|
+
rescue StandardError => e
|
39
|
+
s3.ensure_bucket_absent(bucket_name)
|
41
40
|
|
42
|
-
|
41
|
+
Failure(e)
|
42
|
+
end
|
43
43
|
|
44
44
|
# @param [Array<Pathname>] files
|
45
|
-
#
|
45
|
+
# @param [Pathname] from
|
46
46
|
# @yieldparam [Pathname] path
|
47
|
-
def upload_files(*files, bucket:,
|
47
|
+
def upload_files(*files, bucket:, from:)
|
48
48
|
files.each do |path|
|
49
|
-
path.open do |f|
|
50
|
-
s3.put_object(bucket: bucket, key:
|
49
|
+
from.join(path).open do |f|
|
50
|
+
s3.raw.put_object(bucket: bucket, key: path.to_s, body: f)
|
51
51
|
end
|
52
|
+
|
52
53
|
yield path if block_given?
|
53
54
|
end
|
54
55
|
end
|
56
|
+
|
57
|
+
# @return [Array<Pathname>]
|
58
|
+
def base_files
|
59
|
+
[
|
60
|
+
Pathname.new(Package::Package::CHECKSUM_FILE_NAME),
|
61
|
+
Pathname.new(Package::Package::SPEC_FILE_NAME),
|
62
|
+
]
|
63
|
+
end
|
55
64
|
end
|
56
65
|
end
|
57
66
|
end
|
@@ -4,7 +4,6 @@ module Lexicon
|
|
4
4
|
module Common
|
5
5
|
class ShellExecutor
|
6
6
|
include Mixin::Finalizable
|
7
|
-
include Mixin::LoggerAware
|
8
7
|
|
9
8
|
def initialize
|
10
9
|
@command_dir = Dir.mktmpdir
|
@@ -13,8 +12,6 @@ module Lexicon
|
|
13
12
|
# @param [String] command
|
14
13
|
# @return [String]
|
15
14
|
def execute(command)
|
16
|
-
log(command.cyan)
|
17
|
-
|
18
15
|
cmd = Tempfile.new('command-', @command_dir)
|
19
16
|
cmd.write <<~BASH
|
20
17
|
#!/usr/bin/env bash
|
data/lib/lexicon/common.rb
CHANGED
data/lib/lexicon-common.rb
CHANGED
@@ -3,10 +3,13 @@
|
|
3
3
|
# require 'lexicon/common'
|
4
4
|
require 'aws-sdk-s3'
|
5
5
|
require 'colored'
|
6
|
+
require 'concurrent-ruby'
|
7
|
+
require 'corindon'
|
6
8
|
require 'logger'
|
7
9
|
require 'json_schemer'
|
8
10
|
require 'pg'
|
9
11
|
require 'semantic'
|
12
|
+
require 'shellwords'
|
10
13
|
require 'zeitwerk'
|
11
14
|
|
12
15
|
# Require the common file as loading the version first through the gemspec prevents Zeitwerk to load it.
|