lexicon-common 0.1.0 → 0.2.0

data/lib/lexicon/common/package/v1/source_file_set.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V1
+        class SourceFileSet
+          include Mixin::Nameable
+
+          attr_reader :id, :name, :structure_path, :data_path, :tables
+
+          # @param [String] id
+          # @param [String] name
+          # @param [String] structure
+          # @param [String] data
+          # @param [Array<String>] tables
+          def initialize(id:, name:, structure:, data:, tables:)
+            @id = id
+            @name = name
+            @structure_path = structure
+            @data_path = data
+            @tables = tables
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/v2/package.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V2
+        class Package < Common::Package::Package
+          # @param [Pathname] dir
+          # @param [Pathname] checksum_file
+          # @param [Semantic::Version] version
+          def initialize(version:, spec_file:, checksum_file:, dir:, file_sets:)
+            super(
+              checksum_file: checksum_file,
+              dir: dir,
+              spec_file: spec_file,
+              schema_version: 2,
+              version: version,
+            )
+
+            @file_sets = file_sets
+          end
+
+          def valid?
+            super
+          end
+
+          def files
+            file_sets.flat_map { |fs| file_set_files(fs) }
+          end
+
+          # @return [SourceFileSet]
+          attr_reader :file_sets
+
+          def data_dir
+            dir.join('data')
+          end
+
+          private
+
+            # @param [SourceFileSet] file_set
+            # @return [Array<PackageFile>]
+            def file_set_files(file_set)
+              relative_data_dir = data_dir.basename
+
+              structure_file = PackageFile.new_structure(relative_data_dir.join(file_set.structure))
+              table_files = file_set.tables
+                                    .values.flatten(1)
+                                    .map { |table_file| PackageFile.new_data(relative_data_dir.join(table_file)) }
+
+              [structure_file, *table_files]
+            end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/v2/package_builder.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V2
+        class PackageBuilder < Package
+          def initialize(version:, dir:)
+            super(
+              file_sets: [],
+              version: version,
+              dir: dir,
+              checksum_file: dir.join(CHECKSUM_FILE_NAME),
+              spec_file: dir.join(SPEC_FILE_NAME),
+            )
+
+            FileUtils.mkdir_p(data_dir)
+          end
+
+          # @param [String] id
+          # @param [String] name
+          # @param [Pathname] structure
+          #   Takes ownership of the file (moves it to the correct folder)
+          # @param [Hash{String=>Array<Pathname>}] tables
+          #   Takes ownership of the files (moves them to the correct folder)
+          def add_file_set(id, name:, structure:, tables:)
+            # @type [Pathname] structure_file_path
+            structure_file_path = data_dir.join(structure_file_name(id))
+            FileUtils.mv(structure.to_s, structure_file_path.to_s)
+
+            table_data = tables.map do |table_name, files|
+              index = 0
+
+              file_names = files.map do |file|
+                file_name = "#{table_name}_#{index}.csv.gz"
+                FileUtils.mv(file.to_s, data_dir.join(file_name))
+                index += 1
+
+                file_name
+              end
+
+              [table_name, file_names]
+            end
+
+            file_sets << SourceFileSet.new(
+              id: id,
+              name: name,
+              structure: structure_file_name(id),
+              tables: table_data.to_h
+            )
+          end
+
+          def as_package
+            Package.new(
+              checksum_file: checksum_file,
+              dir: dir,
+              file_sets: file_sets,
+              spec_file: spec_file,
+              version: version,
+            )
+          end
+
+          private
+
+            def structure_file_name(id)
+              "#{id}__structure.sql"
+            end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/v2/source_file_set.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V2
+        class SourceFileSet
+          include Mixin::Nameable
+
+          attr_reader :id, :name, :structure, :tables
+
+          # @param [String] id
+          # @param [String] name
+          # @param [String] structure
+          # @param [Hash{String=>Array<String>}] tables
+          def initialize(id:, name:, structure:, tables:)
+            @id = id
+            @name = name
+            @structure = structure
+            @tables = tables.freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/production/datasource_loader.rb

@@ -1,61 +1,139 @@
 # frozen_string_literal: true
 
+using Corindon::Result::Ext
+
 module Lexicon
   module Common
     module Production
       class DatasourceLoader
+        include Mixin::LoggerAware
         include Mixin::SchemaNamer
+
         # @param [ShellExecutor] shell
         # @param [Database::Factory] database_factory
         # @param [FileLoader] file_loader
         # @param [String] database_url
-        def initialize(shell:, database_factory:, file_loader:, database_url:)
+        # @param [TableLocker] table_locker
+        # @param [Psql] psql
+        def initialize(shell:, database_factory:, file_loader:, database_url:, table_locker:, psql:)
           @shell = shell
           @database_factory = database_factory
           @file_loader = file_loader
           @database_url = database_url
+          @table_locker = table_locker
+          @psql = psql
         end
 
         # @param [Package::Package] package
         # @param [Array<String>, nil] only
-        # @param [Array<String>] without
         #   If nil, all datasets are loaded.
         #   If present, only listed datasets are loaded.
         #   Structures are ALWAYS loaded
+        # @param [Array<String>] without
         def load_package(package, only: nil, without: [])
-          file_sets = if only.nil?
-                        package.file_sets.select(&:data_path)
-                      else
-                        sets_by_name = package.file_sets.map { |fs| [fs.name, fs] }.to_h
+          case package.schema_version
+          when 1
+            load_v1(package, only: only, without: without)
+          when 2
+            load_v2(package, only: only, without: without)
+          else
+            log("Schema version #{package.schema_version} is not supported")
+          end
+        end
+
+        private
 
-                        missing, present = only.map { |name| [name, sets_by_name.fetch(name, nil)] }
-                                               .partition { |(_name, value)| value.nil? }
+          # @param [Package::V1::Package] package
+          def load_v1(package, only: nil, without: [])
+            file_sets = filter_file_sets(package.file_sets, only: only, without: without)
+                          .unwrap!
+                          .select(&:data_path)
 
-                        if missing.any?
-                          puts "[ NOK ] Datasources #{missing.map(&:first).join(', ')} don't exist!"
-                          return
-                        end
+            load_structure_files(
+              package.files.select(&:structure?).map(&:path),
+              schema: version_to_schema(package.version),
+              dir: package.dir
+            )
 
-                        present.map(&:second)
-                               .select(&:data_path)
-                      end
+            remaining = ::Concurrent::Set.new(file_sets.map(&:name))
 
-          file_sets = file_sets.reject { |fs| without.include?(fs.name) }
+            file_sets.map do |fs|
+              Thread.new do
+                file_loader.load_file(package.data_path(fs))
+                remaining.delete(fs.name)
+
+                puts '[  OK ] '.green + fs.name.yellow + ", #{remaining_message(remaining)}"
+              end
+            end.each(&:join)
 
-          load_structure_files(package.structure_files, schema: version_to_schema(package.version))
+            table_locker.lock_tables(package: package, tables: package.file_sets.flat_map(&:tables))
+          end
 
-          file_sets.map do |fs|
-            Thread.new do
-              puts "Loading #{fs.name}"
-              file_loader.load_file(package.data_path(fs))
-              puts '[  OK ] '.green + fs.name.yellow
+          def remaining_message(remaining)
+            if remaining.size.zero?
+              'All done!'
+            elsif remaining.size > 5
+              "#{remaining.size} remaining"
+            else
+              "Remaining: #{remaining.to_a.sort.join(', ')}"
             end
-          end.each(&:join)
+          end
 
-          lock_tables(package)
-        end
+          # @param [Package::V2::Package] package
+          # @param [Array<String>, nil] only
+          # @param [Array<String>] without
+          def load_v2(package, only: nil, without: [])
+            file_sets = filter_file_sets(package.file_sets, only: only, without: without)
+                          .unwrap!
+                          .select { |fs| fs.tables.any? }
 
-        private
+            schema = version_to_schema(package.version)
+
+            load_structure_files(package.files.select(&:structure?).map(&:path), schema: schema, dir: package.dir)
+
+            remaining = ::Concurrent::Set.new(file_sets.flat_map{|fs| fs.tables.values.flatten(1) })
+
+            threads = file_sets.flat_map do |fs|
+              fs.tables.flat_map do |name, files|
+                files.map do |file|
+                  Thread.new do
+                    load_csv(package.data_dir.join(file), into: name, schema: schema)
+                    remaining.delete(file)
+
+                    puts '[  OK ] '.green + file.to_s.yellow + ", #{remaining_message(remaining)}"
+                  end
+                end
+              end
+            end
+
+            threads.each(&:join)
+          end
+
+          # @param [Array<Package::Mixin::Nameable>] file_sets
+          # @param [Array<String>, nil] only
+          # @param [Array<String>] without
+          # @return [Corindon::Result::Result]
+          def filter_file_sets(file_sets, only:, without:)
+            sets = if only.nil?
+                     file_sets
+                   else
+                     sets_by_name = file_sets.map { |fs| [fs.name, fs] }.to_h
+
+                     missing, present = only.map { |name| [name, sets_by_name.fetch(name, nil)] }
+                                            .partition { |(_name, value)| value.nil? }
+
+                     if missing.any?
+                       puts "[ NOK ] Datasources #{missing.map(&:first).join(', ')} don't exist!"
+
+                       return Failure(StandardError.new("Datasources #{missing.map(&:first).join(', ')} don't exist!"))
+                     end
+
+                     present.map(&:second)
+                            .select(&:data_path)
+                   end
+
+            Success(sets.reject { |fs| without.include?(fs.name) })
+          end
 
           # @return [Database::Factory]
           attr_reader :database_factory
@@ -65,44 +143,25 @@ module Lexicon
           attr_reader :file_loader
           # @return [String]
           attr_reader :database_url
+          # @return [TableLocker]
+          attr_reader :table_locker
+          # @return [Psql]
+          attr_reader :psql
 
-          def load_structure_files(files, schema:)
-            database = database_factory.new_instance(url: database_url)
-            database.prepend_search_path(schema) do
-              files.each do |file|
-                database.query(file.read)
-              end
-            end
+          # @param [Pathname] file
+          # @param [String] into
+          # @param [String] schema
+          def load_csv(file, into:, schema:)
+            psql.execute_raw(<<~SQL)
+              \\copy "#{schema}"."#{into}" FROM PROGRAM 'zcat #{file}' WITH csv
+            SQL
           end
 
-          # @param [Package::Package] package
-          def lock_tables(package)
+          def load_structure_files(files, schema:, dir:)
             database = database_factory.new_instance(url: database_url)
-
-            schema = version_to_schema(package.version)
-
-            database.prepend_search_path schema do
-              database.query <<~SQL
-                CREATE OR REPLACE FUNCTION #{schema}.deny_changes()
-                  RETURNS TRIGGER
-                AS $$
-                  BEGIN
-                    RAISE EXCEPTION '% denied on % (master data)', TG_OP, TG_RELNAME;
-                  END;
-                $$
-                LANGUAGE plpgsql;
-              SQL
-              package.file_sets.flat_map(&:tables).each do |table_name|
-                database.query <<~SQL
-                  CREATE TRIGGER deny_changes
-                    BEFORE INSERT
-                        OR UPDATE
-                        OR DELETE
-                        OR TRUNCATE
-                    ON #{schema}.#{table_name}
-                    FOR EACH STATEMENT
-                      EXECUTE PROCEDURE #{schema}.deny_changes()
-                SQL
+            database.prepend_search_path(schema) do
+              files.each do |file|
+                database.query(dir.join(file).read)
               end
             end
           end

data/lib/lexicon/common/production/table_locker.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Production
+      class TableLocker
+        include Mixin::SchemaNamer
+
+        # @param [Database::Factory] database_factory
+        # @param [String] database_url
+        def initialize(database_factory:, database_url:)
+          @database_factory = database_factory
+          @database_url = database_url
+        end
+
+        # @param [Package::Package] package
+        # @param [Array<String>] tables
+        def lock_tables(package:, tables: [])
+          database = database_factory.new_instance(url: database_url)
+
+          schema = version_to_schema(package.version)
+
+          database.prepend_search_path schema do
+            database.query <<~SQL
+              CREATE OR REPLACE FUNCTION #{schema}.deny_changes()
+                RETURNS TRIGGER
+              AS $$
+                BEGIN
+                  RAISE EXCEPTION '% denied on % (master data)', TG_OP, TG_RELNAME;
+                END;
+              $$
+              LANGUAGE plpgsql;
+            SQL
+            tables.each do |table_name|
+              database.query <<~SQL
+                CREATE TRIGGER deny_changes
+                  BEFORE INSERT
+                      OR UPDATE
+                      OR DELETE
+                      OR TRUNCATE
+                  ON #{schema}.#{table_name}
+                  FOR EACH STATEMENT
+                    EXECUTE PROCEDURE #{schema}.deny_changes()
+              SQL
+            end
+          end
+        end
+
+        private
+
+          # @return [Database::Factory]
+          attr_reader :database_factory
+          # @return [String]
+          attr_reader :database_url
+      end
+    end
+  end
+end