lexicon-common 0.1.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,61 +1,139 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ using Corindon::Result::Ext
4
+
3
5
  module Lexicon
4
6
  module Common
5
7
  module Production
6
8
  class DatasourceLoader
9
+ include Mixin::LoggerAware
7
10
  include Mixin::SchemaNamer
11
+
8
12
  # @param [ShellExecutor] shell
9
13
  # @param [Database::Factory] database_factory
10
14
  # @param [FileLoader] file_loader
11
15
  # @param [String] database_url
12
- def initialize(shell:, database_factory:, file_loader:, database_url:)
16
+ # @param [TableLocker] table_locker
17
+ # @param [Psql] psql
18
+ def initialize(shell:, database_factory:, file_loader:, database_url:, table_locker:, psql:)
13
19
  @shell = shell
14
20
  @database_factory = database_factory
15
21
  @file_loader = file_loader
16
22
  @database_url = database_url
23
+ @table_locker = table_locker
24
+ @psql = psql
17
25
  end
18
26
 
19
27
  # @param [Package::Package] package
20
28
  # @param [Array<String>, nil] only
21
- # @param [Array<String>] without
22
29
  # If nil, all datasets are loaded.
23
30
  # If present, only listed datasets are loaded.
24
31
  # Structures are ALWAYS loaded
32
+ # @param [Array<String>] without
25
33
  def load_package(package, only: nil, without: [])
26
- file_sets = if only.nil?
27
- package.file_sets.select(&:data_path)
28
- else
29
- sets_by_name = package.file_sets.map { |fs| [fs.name, fs] }.to_h
34
+ case package.schema_version
35
+ when 1
36
+ load_v1(package, only: only, without: without)
37
+ when 2
38
+ load_v2(package, only: only, without: without)
39
+ else
40
+ log("Schema version #{package.schema_version} is not supported")
41
+ end
42
+ end
43
+
44
+ private
30
45
 
31
- missing, present = only.map { |name| [name, sets_by_name.fetch(name, nil)] }
32
- .partition { |(_name, value)| value.nil? }
46
+ # @param [Package::V1::Package] package
47
+ def load_v1(package, only: nil, without: [])
48
+ file_sets = filter_file_sets(package.file_sets, only: only, without: without)
49
+ .unwrap!
50
+ .select(&:data_path)
33
51
 
34
- if missing.any?
35
- puts "[ NOK ] Datasources #{missing.map(&:first).join(', ')} don't exist!"
36
- return
37
- end
52
+ load_structure_files(
53
+ package.files.select(&:structure?).map(&:path),
54
+ schema: version_to_schema(package.version),
55
+ dir: package.dir
56
+ )
38
57
 
39
- present.map(&:second)
40
- .select(&:data_path)
41
- end
58
+ remaining = ::Concurrent::Set.new(file_sets.map(&:name))
42
59
 
43
- file_sets = file_sets.reject { |fs| without.include?(fs.name) }
60
+ file_sets.map do |fs|
61
+ Thread.new do
62
+ file_loader.load_file(package.data_path(fs))
63
+ remaining.delete(fs.name)
64
+
65
+ puts '[ OK ] '.green + fs.name.yellow + ", #{remaining_message(remaining)}"
66
+ end
67
+ end.each(&:join)
44
68
 
45
- load_structure_files(package.structure_files, schema: version_to_schema(package.version))
69
+ table_locker.lock_tables(package: package, tables: package.file_sets.flat_map(&:tables))
70
+ end
46
71
 
47
- file_sets.map do |fs|
48
- Thread.new do
49
- puts "Loading #{fs.name}"
50
- file_loader.load_file(package.data_path(fs))
51
- puts '[ OK ] '.green + fs.name.yellow
72
+ def remaining_message(remaining)
73
+ if remaining.size.zero?
74
+ 'All done!'
75
+ elsif remaining.size > 5
76
+ "#{remaining.size} remaining"
77
+ else
78
+ "Remaining: #{remaining.to_a.sort.join(', ')}"
52
79
  end
53
- end.each(&:join)
80
+ end
54
81
 
55
- lock_tables(package)
56
- end
82
+ # @param [Package::V2::Package] package
83
+ # @param [Array<String>, nil] only
84
+ # @param [Array<String>] without
85
+ def load_v2(package, only: nil, without: [])
86
+ file_sets = filter_file_sets(package.file_sets, only: only, without: without)
87
+ .unwrap!
88
+ .select { |fs| fs.tables.any? }
57
89
 
58
- private
90
+ schema = version_to_schema(package.version)
91
+
92
+ load_structure_files(package.files.select(&:structure?).map(&:path), schema: schema, dir: package.dir)
93
+
94
+ remaining = ::Concurrent::Set.new(file_sets.flat_map{|fs| fs.tables.values.flatten(1) })
95
+
96
+ threads = file_sets.flat_map do |fs|
97
+ fs.tables.flat_map do |name, files|
98
+ files.map do |file|
99
+ Thread.new do
100
+ load_csv(package.data_dir.join(file), into: name, schema: schema)
101
+ remaining.delete(file)
102
+
103
+ puts '[ OK ] '.green + file.to_s.yellow + ", #{remaining_message(remaining)}"
104
+ end
105
+ end
106
+ end
107
+ end
108
+
109
+ threads.each(&:join)
110
+ end
111
+
112
+ # @param [Array<Package::Mixin::Nameable>] file_sets
113
+ # @param [Array<String>, nil] only
114
+ # @param [Array<String>] without
115
+ # @return [Corindon::Result::Result]
116
+ def filter_file_sets(file_sets, only:, without:)
117
+ sets = if only.nil?
118
+ file_sets
119
+ else
120
+ sets_by_name = file_sets.map { |fs| [fs.name, fs] }.to_h
121
+
122
+ missing, present = only.map { |name| [name, sets_by_name.fetch(name, nil)] }
123
+ .partition { |(_name, value)| value.nil? }
124
+
125
+ if missing.any?
126
+ puts "[ NOK ] Datasources #{missing.map(&:first).join(', ')} don't exist!"
127
+
128
+ return Failure(StandardError.new("Datasources #{missing.map(&:first).join(', ')} don't exist!"))
129
+ end
130
+
131
+ present.map(&:second)
132
+ .select(&:data_path)
133
+ end
134
+
135
+ Success(sets.reject { |fs| without.include?(fs.name) })
136
+ end
59
137
 
60
138
  # @return [Database::Factory]
61
139
  attr_reader :database_factory
@@ -65,44 +143,25 @@ module Lexicon
65
143
  attr_reader :file_loader
66
144
  # @return [String]
67
145
  attr_reader :database_url
146
+ # @return [TableLocker]
147
+ attr_reader :table_locker
148
+ # @return [Psql]
149
+ attr_reader :psql
68
150
 
69
- def load_structure_files(files, schema:)
70
- database = database_factory.new_instance(url: database_url)
71
- database.prepend_search_path(schema) do
72
- files.each do |file|
73
- database.query(file.read)
74
- end
75
- end
151
+ # @param [Pathname] file
152
+ # @param [String] into
153
+ # @param [String] schema
154
+ def load_csv(file, into:, schema:)
155
+ psql.execute_raw(<<~SQL)
156
+ \\copy "#{schema}"."#{into}" FROM PROGRAM 'zcat < #{file}' WITH csv
157
+ SQL
76
158
  end
77
159
 
78
- # @param [Package::Package] package
79
- def lock_tables(package)
160
+ def load_structure_files(files, schema:, dir:)
80
161
  database = database_factory.new_instance(url: database_url)
81
-
82
- schema = version_to_schema(package.version)
83
-
84
- database.prepend_search_path schema do
85
- database.query <<~SQL
86
- CREATE OR REPLACE FUNCTION #{schema}.deny_changes()
87
- RETURNS TRIGGER
88
- AS $$
89
- BEGIN
90
- RAISE EXCEPTION '% denied on % (master data)', TG_OP, TG_RELNAME;
91
- END;
92
- $$
93
- LANGUAGE plpgsql;
94
- SQL
95
- package.file_sets.flat_map(&:tables).each do |table_name|
96
- database.query <<~SQL
97
- CREATE TRIGGER deny_changes
98
- BEFORE INSERT
99
- OR UPDATE
100
- OR DELETE
101
- OR TRUNCATE
102
- ON #{schema}.#{table_name}
103
- FOR EACH STATEMENT
104
- EXECUTE PROCEDURE #{schema}.deny_changes()
105
- SQL
162
+ database.prepend_search_path(schema) do
163
+ files.each do |file|
164
+ database.query(dir.join(file).read)
106
165
  end
107
166
  end
108
167
  end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lexicon
4
+ module Common
5
+ module Production
6
+ class TableLocker
7
+ include Mixin::SchemaNamer
8
+
9
+ # @param [Database::Factory] database_factory
10
+ # @param [String] database_url
11
+ def initialize(database_factory:, database_url:)
12
+ @database_factory = database_factory
13
+ @database_url = database_url
14
+ end
15
+
16
+ # @param [Package::Package] package
17
+ # @param [Array<String>] tables
18
+ def lock_tables(package:, tables: [])
19
+ database = database_factory.new_instance(url: database_url)
20
+
21
+ schema = version_to_schema(package.version)
22
+
23
+ database.prepend_search_path schema do
24
+ database.query <<~SQL
25
+ CREATE OR REPLACE FUNCTION #{schema}.deny_changes()
26
+ RETURNS TRIGGER
27
+ AS $$
28
+ BEGIN
29
+ RAISE EXCEPTION '% denied on % (master data)', TG_OP, TG_RELNAME;
30
+ END;
31
+ $$
32
+ LANGUAGE plpgsql;
33
+ SQL
34
+ tables.each do |table_name|
35
+ database.query <<~SQL
36
+ CREATE TRIGGER deny_changes
37
+ BEFORE INSERT
38
+ OR UPDATE
39
+ OR DELETE
40
+ OR TRUNCATE
41
+ ON #{schema}.#{table_name}
42
+ FOR EACH STATEMENT
43
+ EXECUTE PROCEDURE #{schema}.deny_changes()
44
+ SQL
45
+ end
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ # @return [Database::Factory]
52
+ attr_reader :database_factory
53
+ # @return [String]
54
+ attr_reader :database_url
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lexicon
4
+ module Common
5
+ class Psql
6
+ # @param [String] url
7
+ # @param [ShellExecutor] executor
8
+ def initialize(url:, executor:)
9
+ @url = url
10
+ @executor = executor
11
+ end
12
+
13
+ # @param [String] command
14
+ # @param [String, Array<String>] search_path
15
+ def execute(command, search_path:)
16
+ command = <<~SQL
17
+ SET search_path TO #{Array(search_path).join(', ')};
18
+ #{command}
19
+ SQL
20
+
21
+ execute_raw(command)
22
+ end
23
+
24
+ # @param [String] command
25
+ def execute_raw(command)
26
+ @executor.execute <<~BASH
27
+ psql '#{url}' --quiet -c #{Shellwords.escape(command)}
28
+ BASH
29
+ end
30
+
31
+ # @param [Pathname] file
32
+ # @param [String, Array<String>] search_path
33
+ def load_sql(file, search_path:)
34
+ @executor.execute <<~BASH
35
+ echo 'SET SEARCH_PATH TO #{Array(search_path).join(', ')};' | cat - #{file} | psql '#{url}'
36
+ BASH
37
+ end
38
+
39
+ private
40
+
41
+ # @return [ShellExecutor]
42
+ attr_reader :executor
43
+ # @return [String]
44
+ attr_reader :url
45
+ end
46
+ end
47
+ end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ using Corindon::Result::Ext
4
+
3
5
  module Lexicon
4
6
  module Common
5
7
  module Remote
@@ -9,71 +11,87 @@ module Lexicon
9
11
  # @param [DirectoryPackageLoader] package_loader
10
12
  def initialize(s3:, out_dir:, package_loader:)
11
13
  super(s3: s3)
14
+
12
15
  @out_dir = out_dir
13
16
  @package_loader = package_loader
14
17
  end
15
18
 
16
19
  # @param [Semantic::Version] version
17
- # @return [Boolean]
20
+ # @return [Corindon::Result::Result]
18
21
  def download(version)
19
- bucket = version.to_s
22
+ rescue_failure do
23
+ bucket = version.to_s
20
24
 
21
- if s3.bucket_exist?(bucket)
22
- Dir.mktmpdir(nil, out_dir) do |tmp_dir|
23
- tmp_dir = Pathname.new(tmp_dir)
25
+ if s3.bucket_exist?(bucket)
26
+ Dir.mktmpdir(nil, out_dir) do |tmp_dir|
27
+ tmp_dir = Pathname.new(tmp_dir)
24
28
 
25
- s3.raw.get_object(
26
- bucket: bucket,
27
- key: Package::Package::SPEC_FILE_NAME,
28
- response_target: tmp_dir.join(Package::Package::SPEC_FILE_NAME).to_s
29
- )
30
- s3.raw.get_object(
31
- bucket: bucket,
32
- key: Package::Package::CHECKSUM_FILE_NAME,
33
- response_target: tmp_dir.join(Package::Package::CHECKSUM_FILE_NAME).to_s
34
- )
29
+ download_spec_files(bucket, tmp_dir).unwrap!
35
30
 
36
- package = package_loader.load_package(tmp_dir.basename.to_s)
37
- if !package.nil?
38
- puts "[ OK ] Found package with key #{version}, version is #{package.version}".green
31
+ package = package_loader.load_package(tmp_dir.basename.to_s)
32
+ if !package.nil?
33
+ puts "[ OK ] Found package with key #{version}, version is #{package.version}".green
39
34
 
40
- FileUtils.mkdir_p package.data_dir
35
+ download_data_files(package, bucket).unwrap!
41
36
 
42
- package.structure_files.map do |file|
43
- Thread.new do
44
- s3.raw.get_object(bucket: bucket, key: "data/#{file.basename.to_s}", response_target: file.to_s)
45
- puts "[ OK ] Downloaded #{file.basename}".green
46
- end
47
- end.each(&:join)
37
+ dest_dir = out_dir.join(version.to_s)
38
+ FileUtils.mkdir_p(dest_dir)
48
39
 
49
- package.file_sets.map do |fs|
50
- Thread.new do
51
- path = package.data_path(fs)
52
- s3.raw.get_object(bucket: bucket, key: "data/#{path.basename.to_s}", response_target: path.to_s)
53
- puts "[ OK ] Downloaded #{path.basename}".green
40
+ tmp_dir.children.each do |child|
41
+ FileUtils.mv(child.to_s, dest_dir.join(child.basename).to_s)
54
42
  end
55
- end.each(&:join)
56
-
57
- dest_dir = out_dir.join(version.to_s)
58
- FileUtils.mkdir_p(dest_dir)
59
- tmp_dir.children.each do |child|
60
- FileUtils.mv(child.to_s, dest_dir.join(child.basename).to_s)
61
- end
62
43
 
63
- true
64
- else
65
- puts "[ NOK ] The remote contains a bucket '#{version}' but it does not contains a valid package.".red
44
+ Success(package)
45
+ else
46
+ puts "[ NOK ] The remote contains a bucket '#{version}' but it does not contains a valid package.".red
66
47
 
67
- false
48
+ Failure(StandardError.new("The folder #{bucket} on the server does not contain a valid package"))
49
+ end
68
50
  end
51
+ else
52
+ Failure(StandardError.new("The server does not have a directory named #{bucket}"))
69
53
  end
70
- else
71
- false
72
54
  end
73
55
  end
74
56
 
75
57
  private
76
58
 
59
+ def download_data_files(package, bucket)
60
+ rescue_failure do
61
+ threads = package.files.map do |file|
62
+ Thread.new do
63
+ destination = package.dir.join(file.path)
64
+ FileUtils.mkdir_p(destination.dirname)
65
+
66
+ s3.raw.get_object(bucket: bucket, key: file.to_s, response_target: destination)
67
+
68
+ puts "[ OK ] Downloaded #{file}".green
69
+ end
70
+ end
71
+
72
+ threads.each(&:join)
73
+
74
+ Success(nil)
75
+ end
76
+ end
77
+
78
+ def download_spec_files(bucket, tmp_dir)
79
+ rescue_failure do
80
+ s3.raw.get_object(
81
+ bucket: bucket,
82
+ key: Package::Package::SPEC_FILE_NAME,
83
+ response_target: tmp_dir.join(Package::Package::SPEC_FILE_NAME).to_s
84
+ )
85
+ s3.raw.get_object(
86
+ bucket: bucket,
87
+ key: Package::Package::CHECKSUM_FILE_NAME,
88
+ response_target: tmp_dir.join(Package::Package::CHECKSUM_FILE_NAME).to_s
89
+ )
90
+
91
+ Success(nil)
92
+ end
93
+ end
94
+
77
95
  # @return [DirectoryPackageLoader]
78
96
  attr_reader :package_loader
79
97
  # @return [Pathname]
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ using Corindon::Result::Ext
4
+
3
5
  module Lexicon
4
6
  module Common
5
7
  module Remote
@@ -7,51 +9,58 @@ module Lexicon
7
9
  include Mixin::LoggerAware
8
10
 
9
11
  # @param [Package] package
10
- # @return [Boolean]
12
+ # @return [Corindon::Result::Result]
11
13
  def upload(package)
12
- bucket_name = package.version.to_s
13
- if !s3.bucket_exist?(bucket_name)
14
- s3.create_bucket(bucket: bucket_name)
15
- puts 'Uploading structures...'
14
+ rescue_failure do
15
+ bucket_name = package.version.to_s
16
+
17
+ if s3.bucket_exist?(bucket_name)
18
+ Failure(StandardError.new("The server already has a folder named #{bucket_name}"))
19
+ else
20
+ upload_package(package, bucket_name)
21
+ end
22
+ end
23
+ end
16
24
 
17
- upload_files(*package.structure_files, bucket: bucket_name, prefix: 'data')
18
- puts '[ OK ] Structure uploaded.'.green
25
+ private
19
26
 
20
- data_files = package.file_sets
21
- .select(&:data_path)
22
- .map { |fs| package.data_path(fs) }
27
+ # @return [Corindon::Result::Result]
28
+ def upload_package(package, bucket_name)
29
+ s3.raw.create_bucket(bucket: bucket_name)
23
30
 
24
- upload_files(*data_files, bucket: bucket_name, prefix: 'data') do |path|
25
- puts "[ OK ] #{path.basename}".green
26
- end
31
+ relative_paths = [*base_files, *package.files.map(&:path)]
27
32
 
28
- upload_files(package.checksum_file, package.spec_file, bucket: bucket_name) do |path|
33
+ upload_files(*relative_paths, from: package.dir, bucket: bucket_name) do |path|
29
34
  puts "[ OK ] #{path.basename}".green
30
35
  end
31
36
 
32
- true
33
- else
34
- false
35
- end
36
- rescue StandardError => e
37
- log_error(e)
38
-
39
- false
40
- end
37
+ Success(package)
38
+ rescue StandardError => e
39
+ s3.ensure_bucket_absent(bucket_name)
41
40
 
42
- private
41
+ Failure(e)
42
+ end
43
43
 
44
44
  # @param [Array<Pathname>] files
45
- #
45
+ # @param [Pathname] from
46
46
  # @yieldparam [Pathname] path
47
- def upload_files(*files, bucket:, prefix: nil)
47
+ def upload_files(*files, bucket:, from:)
48
48
  files.each do |path|
49
- path.open do |f|
50
- s3.put_object(bucket: bucket, key: [prefix, path.basename.to_s].compact.join('/'), body: f)
49
+ from.join(path).open do |f|
50
+ s3.raw.put_object(bucket: bucket, key: path.to_s, body: f)
51
51
  end
52
+
52
53
  yield path if block_given?
53
54
  end
54
55
  end
56
+
57
+ # @return [Array<Pathname>]
58
+ def base_files
59
+ [
60
+ Pathname.new(Package::Package::CHECKSUM_FILE_NAME),
61
+ Pathname.new(Package::Package::SPEC_FILE_NAME),
62
+ ]
63
+ end
55
64
  end
56
65
  end
57
66
  end
@@ -31,6 +31,12 @@ module Lexicon
31
31
  false
32
32
  end
33
33
 
34
+ # @param [String] name
35
+ def ensure_bucket_absent(name)
36
+ if bucket_exist?(name)
37
+ raw.delete_bucket(bucket: name)
38
+ end
39
+ end
34
40
  end
35
41
  end
36
42
  end
@@ -9,7 +9,7 @@ module Lexicon
9
9
  @schema_path = schema_path
10
10
  end
11
11
 
12
- # @return [JSONSchemer::Schema]
12
+ # @return [JSONSchemer::Schema::Base]
13
13
  def build
14
14
  JSONSchemer.schema(schema_path)
15
15
  end
@@ -4,7 +4,6 @@ module Lexicon
4
4
  module Common
5
5
  class ShellExecutor
6
6
  include Mixin::Finalizable
7
- include Mixin::LoggerAware
8
7
 
9
8
  def initialize
10
9
  @command_dir = Dir.mktmpdir
@@ -13,8 +12,6 @@ module Lexicon
13
12
  # @param [String] command
14
13
  # @return [String]
15
14
  def execute(command)
16
- log(command.cyan)
17
-
18
15
  cmd = Tempfile.new('command-', @command_dir)
19
16
  cmd.write <<~BASH
20
17
  #!/usr/bin/env bash
@@ -1,5 +1,5 @@
1
1
  module Lexicon
2
2
  module Common
3
- VERSION = '0.1.0'.freeze
3
+ VERSION = '0.2.1'.freeze
4
4
  end
5
5
  end
@@ -3,5 +3,6 @@
3
3
  module Lexicon
4
4
  module Common
5
5
  LEXICON_SCHEMA_RELATIVE_PATH = 'resources/lexicon.schema.json'
6
+ LEXICON_SCHEMA_ABSOLUTE_PATH = Pathname.new(__dir__).join('..', '..', LEXICON_SCHEMA_RELATIVE_PATH).freeze
6
7
  end
7
8
  end
@@ -3,10 +3,13 @@
3
3
  # require 'lexicon/common'
4
4
  require 'aws-sdk-s3'
5
5
  require 'colored'
6
+ require 'concurrent-ruby'
7
+ require 'corindon'
6
8
  require 'logger'
7
9
  require 'json_schemer'
8
10
  require 'pg'
9
11
  require 'semantic'
12
+ require 'shellwords'
10
13
  require 'zeitwerk'
11
14
 
12
15
  # Require the common file as loading the version first through the gemspec prevents Zeitwerk to load it.