lexicon-common 0.1.0 → 0.2.1

data/lib/lexicon/common/production/datasource_loader.rb

@@ -1,61 +1,139 @@
 # frozen_string_literal: true
 
+using Corindon::Result::Ext
+
 module Lexicon
   module Common
     module Production
       class DatasourceLoader
+        include Mixin::LoggerAware
         include Mixin::SchemaNamer
+
         # @param [ShellExecutor] shell
         # @param [Database::Factory] database_factory
         # @param [FileLoader] file_loader
         # @param [String] database_url
-        def initialize(shell:, database_factory:, file_loader:, database_url:)
+        # @param [TableLocker] table_locker
+        # @param [Psql] psql
+        def initialize(shell:, database_factory:, file_loader:, database_url:, table_locker:, psql:)
           @shell = shell
           @database_factory = database_factory
           @file_loader = file_loader
           @database_url = database_url
+          @table_locker = table_locker
+          @psql = psql
         end
 
         # @param [Package::Package] package
         # @param [Array<String>, nil] only
-        # @param [Array<String>] without
         #   If nil, all datasets are loaded.
         #   If present, only listed datasets are loaded.
         #   Structures are ALWAYS loaded
+        # @param [Array<String>] without
         def load_package(package, only: nil, without: [])
-          file_sets = if only.nil?
-                        package.file_sets.select(&:data_path)
-                      else
-                        sets_by_name = package.file_sets.map { |fs| [fs.name, fs] }.to_h
+          case package.schema_version
+          when 1
+            load_v1(package, only: only, without: without)
+          when 2
+            load_v2(package, only: only, without: without)
+          else
+            log("Schema version #{package.schema_version} is not supported")
+          end
+        end
+
+        private
 
-                        missing, present = only.map { |name| [name, sets_by_name.fetch(name, nil)] }
-                                               .partition { |(_name, value)| value.nil? }
+          # @param [Package::V1::Package] package
+          def load_v1(package, only: nil, without: [])
+            file_sets = filter_file_sets(package.file_sets, only: only, without: without)
+                          .unwrap!
+                          .select(&:data_path)
 
-                        if missing.any?
-                          puts "[ NOK ] Datasources #{missing.map(&:first).join(', ')} don't exist!"
-                          return
-                        end
+            load_structure_files(
+              package.files.select(&:structure?).map(&:path),
+              schema: version_to_schema(package.version),
+              dir: package.dir
+            )
 
-                        present.map(&:second)
-                               .select(&:data_path)
-                      end
+            remaining = ::Concurrent::Set.new(file_sets.map(&:name))
 
-          file_sets = file_sets.reject { |fs| without.include?(fs.name) }
+            file_sets.map do |fs|
+              Thread.new do
+                file_loader.load_file(package.data_path(fs))
+                remaining.delete(fs.name)
+
+                puts '[  OK ] '.green + fs.name.yellow + ", #{remaining_message(remaining)}"
+              end
+            end.each(&:join)
 
-          load_structure_files(package.structure_files, schema: version_to_schema(package.version))
+            table_locker.lock_tables(package: package, tables: package.file_sets.flat_map(&:tables))
+          end
 
-          file_sets.map do |fs|
-            Thread.new do
-              puts "Loading #{fs.name}"
-              file_loader.load_file(package.data_path(fs))
-              puts '[  OK ] '.green + fs.name.yellow
+          def remaining_message(remaining)
+            if remaining.size.zero?
+              'All done!'
+            elsif remaining.size > 5
+              "#{remaining.size} remaining"
+            else
+              "Remaining: #{remaining.to_a.sort.join(', ')}"
             end
-          end.each(&:join)
+          end
 
-          lock_tables(package)
-        end
+          # @param [Package::V2::Package] package
+          # @param [Array<String>, nil] only
+          # @param [Array<String>] without
+          def load_v2(package, only: nil, without: [])
+            file_sets = filter_file_sets(package.file_sets, only: only, without: without)
+                          .unwrap!
+                          .select { |fs| fs.tables.any? }
 
-        private
+            schema = version_to_schema(package.version)
+
+            load_structure_files(package.files.select(&:structure?).map(&:path), schema: schema, dir: package.dir)
+
+            remaining = ::Concurrent::Set.new(file_sets.flat_map{|fs| fs.tables.values.flatten(1) })
+
+            threads = file_sets.flat_map do |fs|
+              fs.tables.flat_map do |name, files|
+                files.map do |file|
+                  Thread.new do
+                    load_csv(package.data_dir.join(file), into: name, schema: schema)
+                    remaining.delete(file)
+
+                    puts '[  OK ] '.green + file.to_s.yellow + ", #{remaining_message(remaining)}"
+                  end
+                end
+              end
+            end
+
+            threads.each(&:join)
+          end
+
+          # @param [Array<Package::Mixin::Nameable>] file_sets
+          # @param [Array<String>, nil] only
+          # @param [Array<String>] without
+          # @return [Corindon::Result::Result]
+          def filter_file_sets(file_sets, only:, without:)
+            sets = if only.nil?
+                     file_sets
+                   else
+                     sets_by_name = file_sets.map { |fs| [fs.name, fs] }.to_h
+
+                     missing, present = only.map { |name| [name, sets_by_name.fetch(name, nil)] }
+                                            .partition { |(_name, value)| value.nil? }
+
+                     if missing.any?
+                       puts "[ NOK ] Datasources #{missing.map(&:first).join(', ')} don't exist!"
+
+                       return Failure(StandardError.new("Datasources #{missing.map(&:first).join(', ')} don't exist!"))
+                     end
+
+                     present.map(&:second)
+                            .select(&:data_path)
+                   end
+
+            Success(sets.reject { |fs| without.include?(fs.name) })
+          end
 
           # @return [Database::Factory]
           attr_reader :database_factory
@@ -65,44 +143,25 @@ module Lexicon
           attr_reader :file_loader
           # @return [String]
           attr_reader :database_url
+          # @return [TableLocker]
+          attr_reader :table_locker
+          # @return [Psql]
+          attr_reader :psql
 
-          def load_structure_files(files, schema:)
-            database = database_factory.new_instance(url: database_url)
-            database.prepend_search_path(schema) do
-              files.each do |file|
-                database.query(file.read)
-              end
-            end
+          # @param [Pathname] file
+          # @param [String] into
+          # @param [String] schema
+          def load_csv(file, into:, schema:)
+            psql.execute_raw(<<~SQL)
+              \\copy "#{schema}"."#{into}" FROM PROGRAM 'zcat < #{file}' WITH csv
+            SQL
           end
 
-          # @param [Package::Package] package
-          def lock_tables(package)
+          def load_structure_files(files, schema:, dir:)
             database = database_factory.new_instance(url: database_url)
-
-            schema = version_to_schema(package.version)
-
-            database.prepend_search_path schema do
-              database.query <<~SQL
-                CREATE OR REPLACE FUNCTION #{schema}.deny_changes()
-                  RETURNS TRIGGER
-                AS $$
-                  BEGIN
-                    RAISE EXCEPTION '% denied on % (master data)', TG_OP, TG_RELNAME;
-                  END;
-                $$
-                LANGUAGE plpgsql;
-              SQL
-              package.file_sets.flat_map(&:tables).each do |table_name|
-                database.query <<~SQL
-                  CREATE TRIGGER deny_changes
-                    BEFORE INSERT
-                        OR UPDATE
-                        OR DELETE
-                        OR TRUNCATE
-                    ON #{schema}.#{table_name}
-                    FOR EACH STATEMENT
-                      EXECUTE PROCEDURE #{schema}.deny_changes()
-                SQL
+            database.prepend_search_path(schema) do
+              files.each do |file|
+                database.query(dir.join(file).read)
               end
             end
           end

data/lib/lexicon/common/production/table_locker.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Production
+      class TableLocker
+        include Mixin::SchemaNamer
+
+        # @param [Database::Factory] database_factory
+        # @param [String] database_url
+        def initialize(database_factory:, database_url:)
+          @database_factory = database_factory
+          @database_url = database_url
+        end
+
+        # @param [Package::Package] package
+        # @param [Array<String>] tables
+        def lock_tables(package:, tables: [])
+          database = database_factory.new_instance(url: database_url)
+
+          schema = version_to_schema(package.version)
+
+          database.prepend_search_path schema do
+            database.query <<~SQL
+              CREATE OR REPLACE FUNCTION #{schema}.deny_changes()
+                RETURNS TRIGGER
+              AS $$
+                BEGIN
+                  RAISE EXCEPTION '% denied on % (master data)', TG_OP, TG_RELNAME;
+                END;
+              $$
+              LANGUAGE plpgsql;
+            SQL
+            tables.each do |table_name|
+              database.query <<~SQL
+                CREATE TRIGGER deny_changes
+                  BEFORE INSERT
+                      OR UPDATE
+                      OR DELETE
+                      OR TRUNCATE
+                  ON #{schema}.#{table_name}
+                  FOR EACH STATEMENT
+                    EXECUTE PROCEDURE #{schema}.deny_changes()
+              SQL
+            end
+          end
+        end
+
+        private
+
+          # @return [Database::Factory]
+          attr_reader :database_factory
+          # @return [String]
+          attr_reader :database_url
+      end
+    end
+  end
+end

data/lib/lexicon/common/psql.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    class Psql
+      # @param [String] url
+      # @param [ShellExecutor] executor
+      def initialize(url:, executor:)
+        @url = url
+        @executor = executor
+      end
+
+      # @param [String] command
+      # @param [String, Array<String>] search_path
+      def execute(command, search_path:)
+        command = <<~SQL
+          SET search_path TO #{Array(search_path).join(', ')};
+          #{command}
+        SQL
+
+        execute_raw(command)
+      end
+
+      # @param [String] command
+      def execute_raw(command)
+        @executor.execute <<~BASH
+          psql '#{url}' --quiet -c #{Shellwords.escape(command)}
+        BASH
+      end
+
+      # @param [Pathname] file
+      # @param [String, Array<String>] search_path
+      def load_sql(file, search_path:)
+        @executor.execute <<~BASH
+          echo 'SET SEARCH_PATH TO #{Array(search_path).join(', ')};' | cat - #{file} | psql '#{url}'
+        BASH
+      end
+
+      private
+
+        # @return [ShellExecutor]
+        attr_reader :executor
+        # @return [String]
+        attr_reader :url
+    end
+  end
+end

data/lib/lexicon/common/remote/package_downloader.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+using Corindon::Result::Ext
+
 module Lexicon
   module Common
     module Remote
@@ -9,71 +11,87 @@ module Lexicon
         # @param [DirectoryPackageLoader] package_loader
         def initialize(s3:, out_dir:, package_loader:)
           super(s3: s3)
+
           @out_dir = out_dir
           @package_loader = package_loader
         end
 
         # @param [Semantic::Version] version
-        # @return [Boolean]
+        # @return [Corindon::Result::Result]
         def download(version)
-          bucket = version.to_s
+          rescue_failure do
+            bucket = version.to_s
 
-          if s3.bucket_exist?(bucket)
-            Dir.mktmpdir(nil, out_dir) do |tmp_dir|
-              tmp_dir = Pathname.new(tmp_dir)
+            if s3.bucket_exist?(bucket)
+              Dir.mktmpdir(nil, out_dir) do |tmp_dir|
+                tmp_dir = Pathname.new(tmp_dir)
 
-              s3.raw.get_object(
-                bucket: bucket,
-                key: Package::Package::SPEC_FILE_NAME,
-                response_target: tmp_dir.join(Package::Package::SPEC_FILE_NAME).to_s
-              )
-              s3.raw.get_object(
-                bucket: bucket,
-                key: Package::Package::CHECKSUM_FILE_NAME,
-                response_target: tmp_dir.join(Package::Package::CHECKSUM_FILE_NAME).to_s
-              )
+                download_spec_files(bucket, tmp_dir).unwrap!
 
-              package = package_loader.load_package(tmp_dir.basename.to_s)
-              if !package.nil?
-                puts "[  OK ] Found package with key #{version}, version is #{package.version}".green
+                package = package_loader.load_package(tmp_dir.basename.to_s)
+                if !package.nil?
+                  puts "[  OK ] Found package with key #{version}, version is #{package.version}".green
 
-                FileUtils.mkdir_p package.data_dir
+                  download_data_files(package, bucket).unwrap!
 
-                package.structure_files.map do |file|
-                  Thread.new do
-                    s3.raw.get_object(bucket: bucket, key: "data/#{file.basename.to_s}", response_target: file.to_s)
-                    puts "[  OK ] Downloaded #{file.basename}".green
-                  end
-                end.each(&:join)
+                  dest_dir = out_dir.join(version.to_s)
+                  FileUtils.mkdir_p(dest_dir)
 
-                package.file_sets.map do |fs|
-                  Thread.new do
-                    path = package.data_path(fs)
-                    s3.raw.get_object(bucket: bucket, key: "data/#{path.basename.to_s}", response_target: path.to_s)
-                    puts "[  OK ] Downloaded #{path.basename}".green
+                  tmp_dir.children.each do |child|
+                    FileUtils.mv(child.to_s, dest_dir.join(child.basename).to_s)
                   end
-                end.each(&:join)
-
-                dest_dir = out_dir.join(version.to_s)
-                FileUtils.mkdir_p(dest_dir)
-                tmp_dir.children.each do |child|
-                  FileUtils.mv(child.to_s, dest_dir.join(child.basename).to_s)
-                end
 
-                true
-              else
-                puts "[ NOK ] The remote contains a bucket '#{version}' but it does not contains a valid package.".red
+                  Success(package)
+                else
+                  puts "[ NOK ] The remote contains a bucket '#{version}' but it does not contains a valid package.".red
 
-                false
+                  Failure(StandardError.new("The folder #{bucket} on the server does not contain a valid package"))
+                end
               end
+            else
+              Failure(StandardError.new("The server does not have a directory named #{bucket}"))
             end
-          else
-            false
           end
         end
 
         private
 
+          def download_data_files(package, bucket)
+            rescue_failure do
+              threads = package.files.map do |file|
+                Thread.new do
+                  destination = package.dir.join(file.path)
+                  FileUtils.mkdir_p(destination.dirname)
+
+                  s3.raw.get_object(bucket: bucket, key: file.to_s, response_target: destination)
+
+                  puts "[  OK ] Downloaded #{file}".green
+                end
+              end
+
+              threads.each(&:join)
+
+              Success(nil)
+            end
+          end
+
+          def download_spec_files(bucket, tmp_dir)
+            rescue_failure do
+              s3.raw.get_object(
+                bucket: bucket,
+                key: Package::Package::SPEC_FILE_NAME,
+                response_target: tmp_dir.join(Package::Package::SPEC_FILE_NAME).to_s
+              )
+              s3.raw.get_object(
+                bucket: bucket,
+                key: Package::Package::CHECKSUM_FILE_NAME,
+                response_target: tmp_dir.join(Package::Package::CHECKSUM_FILE_NAME).to_s
+              )
+
+              Success(nil)
+            end
+          end
+
           # @return [DirectoryPackageLoader]
           attr_reader :package_loader
           # @return [Pathname]

data/lib/lexicon/common/remote/package_uploader.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+using Corindon::Result::Ext
+
 module Lexicon
   module Common
     module Remote
@@ -7,51 +9,58 @@ module Lexicon
         include Mixin::LoggerAware
 
         # @param [Package] package
-        # @return [Boolean]
+        # @return [Corindon::Result::Result]
         def upload(package)
-          bucket_name = package.version.to_s
-          if !s3.bucket_exist?(bucket_name)
-            s3.create_bucket(bucket: bucket_name)
-            puts 'Uploading structures...'
+          rescue_failure do
+            bucket_name = package.version.to_s
+
+            if s3.bucket_exist?(bucket_name)
+              Failure(StandardError.new("The server already has a folder named #{bucket_name}"))
+            else
+              upload_package(package, bucket_name)
+            end
+          end
+        end
 
-            upload_files(*package.structure_files, bucket: bucket_name, prefix: 'data')
-            puts '[  OK ] Structure uploaded.'.green
+        private
 
-            data_files = package.file_sets
-                                .select(&:data_path)
-                                .map { |fs| package.data_path(fs) }
+          # @return [Corindon::Result::Result]
+          def upload_package(package, bucket_name)
+            s3.raw.create_bucket(bucket: bucket_name)
 
-            upload_files(*data_files, bucket: bucket_name, prefix: 'data') do |path|
-              puts "[  OK ] #{path.basename}".green
-            end
+            relative_paths = [*base_files, *package.files.map(&:path)]
 
-            upload_files(package.checksum_file, package.spec_file, bucket: bucket_name) do |path|
+            upload_files(*relative_paths, from: package.dir, bucket: bucket_name) do |path|
               puts "[  OK ] #{path.basename}".green
             end
 
-            true
-          else
-            false
-          end
-        rescue StandardError => e
-          log_error(e)
-
-          false
-        end
+            Success(package)
+          rescue StandardError => e
+            s3.ensure_bucket_absent(bucket_name)
 
-        private
+            Failure(e)
+          end
 
           # @param [Array<Pathname>] files
-          #
+          # @param [Pathname] from
           # @yieldparam [Pathname] path
-          def upload_files(*files, bucket:, prefix: nil)
+          def upload_files(*files, bucket:, from:)
             files.each do |path|
-              path.open do |f|
-                s3.put_object(bucket: bucket, key: [prefix, path.basename.to_s].compact.join('/'), body: f)
+              from.join(path).open do |f|
+                s3.raw.put_object(bucket: bucket, key: path.to_s, body: f)
               end
+
               yield path if block_given?
             end
           end
+
+          # @return [Array<Pathname>]
+          def base_files
+            [
+              Pathname.new(Package::Package::CHECKSUM_FILE_NAME),
+              Pathname.new(Package::Package::SPEC_FILE_NAME),
+            ]
+          end
       end
     end
   end

data/lib/lexicon/common/remote/s3_client.rb

@@ -31,6 +31,12 @@ module Lexicon
           false
         end
 
+        # @param [String] name
+        def ensure_bucket_absent(name)
+          if bucket_exist?(name)
+            raw.delete_bucket(bucket: name)
+          end
+        end
       end
     end
   end

data/lib/lexicon/common/schema/validator_factory.rb

@@ -9,7 +9,7 @@ module Lexicon
           @schema_path = schema_path
         end
 
-        # @return [JSONSchemer::Schema]
+        # @return [JSONSchemer::Schema::Base]
         def build
           JSONSchemer.schema(schema_path)
         end

data/lib/lexicon/common/shell_executor.rb

@@ -4,7 +4,6 @@ module Lexicon
   module Common
     class ShellExecutor
       include Mixin::Finalizable
-      include Mixin::LoggerAware
 
       def initialize
         @command_dir = Dir.mktmpdir
@@ -13,8 +12,6 @@ module Lexicon
       # @param [String] command
       # @return [String]
       def execute(command)
-        log(command.cyan)
-
         cmd = Tempfile.new('command-', @command_dir)
         cmd.write <<~BASH
           #!/usr/bin/env bash

data/lib/lexicon/common/version.rb

@@ -1,5 +1,5 @@
 module Lexicon
   module Common
-    VERSION = '0.1.0'.freeze
+    VERSION = '0.2.1'.freeze
   end
 end

data/lib/lexicon/common.rb

@@ -3,5 +3,6 @@
 module Lexicon
   module Common
     LEXICON_SCHEMA_RELATIVE_PATH = 'resources/lexicon.schema.json'
+    LEXICON_SCHEMA_ABSOLUTE_PATH = Pathname.new(__dir__).join('..', '..', LEXICON_SCHEMA_RELATIVE_PATH).freeze
   end
 end

data/lib/lexicon-common.rb

@@ -3,10 +3,13 @@
 # require 'lexicon/common'
 require 'aws-sdk-s3'
 require 'colored'
+require 'concurrent-ruby'
+require 'corindon'
 require 'logger'
 require 'json_schemer'
 require 'pg'
 require 'semantic'
+require 'shellwords'
 require 'zeitwerk'
 
 # Require the common file as loading the version first through the gemspec prevents Zeitwerk to load it.