lexicon-common 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bb97bec1598709b8fefb46c28055ac8291edd138177995e08616de3b06dfbdb8
+  data.tar.gz: 20db194da73907714cb53ae861bd650ded613f71ef99ae1c93db370c8547c830
+SHA512:
+  metadata.gz: aa5b4f83c9647382dabdaa4dd09250fd4e165476ca3984740763bd4b2cd8f54ac41e73379a5988391a3336d31e9928ca9332598216b102829dff1efe28ab14dd
+  data.tar.gz: 296628e759a78533b06160d653ab6e1299d0d1378964ac20d5350c7b83892d0b385b4d78f1569c1844591c014db2a0130f9249e423946e7d3e38095f28029f5a

data/lexicon-common.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'lib/lexicon/common/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'lexicon-common'
+  spec.version = Lexicon::Common::VERSION
+  spec.authors = ['Ekylibre developers']
+  spec.email = ['dev@ekylibre.com']
+
+  spec.summary = "Common classes and services for Ekylibre's Lexicon"
+  spec.required_ruby_version = '>= 2.6.0'
+  spec.homepage = 'https://www.ekylibre.com'
+  spec.license = 'AGPL-3.0-only'
+
+  # Specify which files should be added to the gem when it is released.
+  spec.files = Dir.glob(%w[lib/**/*.rb resources/**/* *.gemspec])
+
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'aws-sdk-s3', '~> 1.84'
+  spec.add_dependency 'colored', '~> 1.2'
+  spec.add_dependency 'json_schemer', '~> 0.2.16'
+  spec.add_dependency 'pg', '~> 1.2'
+  spec.add_dependency 'semantic', '~> 1.6'
+  spec.add_dependency 'zeitwerk', '~> 2.4'
+
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'minitest', '~> 5.14'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rubocop', '1.11.0'
+end

data/lib/lexicon-common.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# require 'lexicon/common'
+require 'aws-sdk-s3'
+require 'colored'
+require 'logger'
+require 'json_schemer'
+require 'pg'
+require 'semantic'
+require 'zeitwerk'
+
+# Require the common file as loading the version first through the gemspec prevents Zeitwerk to load it.
+require_relative 'lexicon/common'
+
+loader = Zeitwerk::Loader.for_gem
+loader.ignore(__FILE__)
+loader.setup

data/lib/lexicon/common.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    LEXICON_SCHEMA_RELATIVE_PATH = 'resources/lexicon.schema.json'
+  end
+end

data/lib/lexicon/common/database/database.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Database
+      class Database
+        class << self
+          def connect(url)
+            new(PG.connect(url))
+          end
+        end
+
+        attr_writer :verbose
+        # @return [Array<String>]
+        attr_reader :search_path
+
+        def initialize(connection, verbose: false)
+          @connection = connection
+          @search_path = []
+          @verbose = verbose
+
+          disable_notices unless verbose
+        end
+
+        def verbose?
+          @verbose
+        end
+
+        def transaction(&block)
+          connection.transaction(&block)
+        end
+
+        def prepend_search_path(*parts, &block)
+          return if block.nil?
+
+          parts.each { |part| ensure_schema(part) }
+
+          with_search_path(*parts, *search_path) do
+            block.call
+          end
+        end
+
+        def on_empty_schema(base_path: [], &block)
+          schema = make_random_schema_name
+
+          prepend_search_path(schema, *base_path) do
+            block.call(schema)
+          end
+        ensure
+          drop_schema(schema, cascade: true)
+        end
+
+        def drop_schema(name, cascade: false)
+          cascade = if cascade
+                      ' CASCADE'
+                    else
+                      ''
+                    end
+
+          query <<~SQL
+            DROP SCHEMA "#{name}"#{cascade};
+          SQL
+        end
+
+        def make_random_schema_name(prefix = 'lex')
+          "#{prefix}_#{rand(0x100000000).to_s(36)}"
+        end
+
+        def query(sql, *params, **_options)
+          pp sql if verbose?
+          if params.any?
+            @connection.exec_params(sql, params)
+          else
+            @connection.exec(sql)
+          end
+        end
+
+        # @param [#to_s] name
+        def ensure_schema(name)
+          query(<<~SQL)
+            CREATE SCHEMA IF NOT EXISTS "#{name}";
+          SQL
+        end
+
+        def ensure_schema_empty(name)
+          query(<<~SQL)
+            DROP SCHEMA IF EXISTS #{name} CASCADE;
+          SQL
+
+          ensure_schema(name)
+        end
+
+        def copy_data(sql, &block)
+          put_data = ->(d) { @connection.put_copy_data(d) }
+          @connection.copy_data(sql) { block.call(put_data) }
+        end
+
+        private
+
+          # @return [PG::Connection]
+          attr_reader :connection
+
+          def disable_notices
+            query <<~SQL
+              SET client_min_messages TO WARNING;
+            SQL
+          end
+
+          def with_search_path(*path, &block)
+            return if block.nil?
+
+            begin
+              saved_path = @search_path
+              @search_path = path
+
+              query <<~SQL
+                SET search_path TO #{path.map { |part| "\"#{part}\"" }.join(', ')};
+              SQL
+
+              result = block.call
+
+              result
+            ensure
+              path = if saved_path.any?
+                       saved_path.map { |part| "\"#{part}\"" }.join(', ')
+                     else
+                       '" "'
+                     end
+
+              query <<~SQL
+                SET search_path TO #{path};
+              SQL
+
+              @search_path = saved_path
+            end
+          end
+      end
+    end
+  end
+end

data/lib/lexicon/common/database/factory.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Database
+      class Factory
+        attr_reader :verbose
+
+        def initialize(verbose: false)
+          @verbose = verbose
+        end
+
+        def new_instance(url:)
+          Database.new(PG.connect(url), verbose: @verbose)
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/mixin/container_aware.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Mixin
+      module ContainerAware
+        attr_accessor :container
+
+        # @param [Object] service
+        def get(service)
+          container.get(service)
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/mixin/finalizable.rb

@@ -0,0 +1,57 @@
+module Lexicon
+  module Common
+    module Mixin
+      module Finalizable
+        def self.included(base)
+          class << base
+            alias_method :_new, :new
+
+            def new(*args, **options)
+              e = do_call(self, '_new', args, options)
+
+              ObjectSpace.define_finalizer(e, e.method(:_finalize))
+
+              e
+            end
+
+            # Empty Array and Hash splats are handled correctly starting ruby 2.7:
+            # if both args and kwargs are empty, no parameters are sent.
+            if ::Semantic::Version.new(RUBY_VERSION).satisfies?('>= 2.7.0')
+              private def do_call(obj, method, args, kwargs)
+                obj.send(method, *args, **kwargs)
+              end
+            else
+              private def do_call(obj, method, args, kwargs)
+                if args.empty? && kwargs.empty?
+                  obj.send(method)
+                elsif args.empty?
+                  obj.send(method, **kwargs)
+                elsif kwargs.empty?
+                  obj.send(method, *args)
+                else
+                  obj.send(method, *args, **kwargs)
+                end
+              end
+            end
+          end
+        end
+
+        private
+
+          def finalize
+            raise StandardError.new("Finalizer is not implemented in #{self.class.name}")
+          end
+
+          def _finalize(_id)
+            m = method(:finalize)
+
+            if !m.nil?
+              finalize
+            end
+          rescue StandardError => e
+            puts "Exception in finalizer: #{e.message}\n" + e.backtrace.join("\n")
+          end
+      end
+    end
+  end
+end

data/lib/lexicon/common/mixin/logger_aware.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Mixin
+      module LoggerAware
+        # @return [Logger]
+        attr_accessor :logger
+
+        def log(*args, **options)
+          if !logger.nil?
+            logger.debug(*args, **options)
+          end
+        end
+
+        def log_error(error)
+          if error.nil?
+            log('Error (nil)')
+          else
+            log([error.message, *error.backtrace].join("\n"))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/mixin/schema_namer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Mixin
+      module SchemaNamer
+        protected
+
+          # @param [Semantic::Version] version
+          # @return [String]
+          def version_to_schema(version)
+            "lexicon__#{version.to_s.gsub('.', '_')}"
+          end
+
+          # @param [String] schema
+          # @return [Semantic::Version, nil]
+          def schema_to_version(schema)
+            Semantic::Version.new(schema.sub(/\Alexicon__/, '').gsub('_', '.'))
+          end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/directory_package_loader.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      class DirectoryPackageLoader
+        include Mixin::LoggerAware
+
+        # @return [Pathname]
+        attr_reader :root_dir
+
+        # @param [Pathname] root_dir
+        # @param [JSONSchemer::Schema::Base] schema_validator
+        def initialize(root_dir, schema_validator:)
+          @root_dir = root_dir
+          @schema_validator = schema_validator
+        end
+
+        # @param [String] name
+        # @return [Package, nil]
+        def load_package(name)
+          package_dir = root_dir.join(name.to_s)
+
+          if package_dir.directory?
+            load_from_dir(package_dir)
+          else
+            nil
+          end
+        end
+
+        protected
+
+          def load_from_dir(dir)
+            # @type [Pathname]
+            spec_file = dir.join(Package::SPEC_FILE_NAME)
+            # @type [Pathname]
+            checksum_file = dir.join(Package::CHECKSUM_FILE_NAME)
+
+            if spec_file.exist? && checksum_file.exist?
+              json = JSON.parse(spec_file.read)
+
+              if @schema_validator.valid?(json)
+                version = Semantic::Version.new(json.fetch('version'))
+                file_sets = json.fetch('content').map do |id, values|
+                  SourceFileSet.new(
+                    id: id,
+                    name: values.fetch('name'),
+                    structure: values.fetch('structure'),
+                    data: values.fetch('data', nil),
+                    tables: values.fetch('tables', [])
+                  )
+                end
+
+                Package.new(file_sets: file_sets, version: version, dir: dir, checksum_file: checksum_file, spec_file: spec_file)
+              else
+                log("Package at path #{dir} has invalid manifest")
+
+                nil
+              end
+            else
+              nil
+            end
+          end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/package.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      class Package
+        SPEC_FILE_NAME = 'lexicon.json'
+        CHECKSUM_FILE_NAME = 'lexicon.sum'
+
+        # @return [Pathname]
+        attr_reader :checksum_file, :spec_file
+        # @return [Pathname]
+        attr_reader :dir
+        # @return [Array<SourceFileSet>]
+        attr_reader :file_sets
+        # @return [Semantic::Version]
+        attr_reader :version
+
+        # @param [Array<SourceFileSet>] file_sets
+        # @param [Pathname] dir
+        # @param [Pathname] checksum_file
+        # @param [Semantic::Version] version
+        def initialize(file_sets:, version:, dir:, checksum_file:, spec_file:)
+          @checksum_file = checksum_file
+          @dir = dir
+          @file_sets = file_sets
+          @spec_file = spec_file
+          @version = version
+        end
+
+        # @return [Boolean]
+        def valid?
+          checksum_file.exist? && dir.directory? && data_dir.directory? && all_sets_valid?
+        end
+
+        # @return [Array<Pathname>]
+        def structure_files
+          file_sets.map { |fs| dir.join(relative_structure_path(fs)) }
+        end
+
+        # @param [SourceFileSet] file_set
+        # @return [Pathname]
+        def data_path(file_set)
+          dir.join(relative_data_path(file_set))
+        end
+
+        # @return [Pathname]
+        def data_dir
+          dir.join('data')
+        end
+
+        # @return [Pathname, nil]
+        def relative_data_path(file_set)
+          if file_set.data_path.nil?
+            nil
+          else
+            data_dir.basename.join(file_set.data_path)
+          end
+        end
+
+        # @return [Pathname]
+        def relative_structure_path(file_set)
+          data_dir.basename.join(file_set.structure_path)
+        end
+
+        private
+
+          def all_sets_valid?
+            file_sets.all? do |set|
+              data_dir.join(set.structure_path).exist? && !set.data_path.nil? && data_dir.join(set.data_path).exist?
+            end
+          end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/package_builder.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      class PackageBuilder < Package
+        def initialize(version:, dir:)
+          super(
+            file_sets: [],
+            version: version,
+            dir: dir,
+            checksum_file: dir.join(CHECKSUM_FILE_NAME),
+            spec_file: dir.join(SPEC_FILE_NAME)
+          )
+
+          FileUtils.mkdir_p(data_dir)
+        end
+
+        # @param [String] id
+        # @param [String] name
+        # @param [Pathname] structure
+        #   Takes ownership of the file (moves it to the correct folder)
+        # @param [Array<String>] tables
+        # @param [Pathname] data
+        #   Takes ownership of the file (moves it to the correct folder)
+        # @param [String] data_ext
+        def add_file_set(id, name:, structure:, tables:, data: nil, data_ext: '.sql')
+          # @type [Pathname] structure_file_path
+          structure_file_path = data_dir.join(structure_file_name(id))
+          FileUtils.mv(structure.to_s, structure_file_path.to_s)
+
+          # @type [Pathname] data_file_path
+          data_name = if data.nil?
+                        nil
+                      else
+                        dname = data_file_name(id, data_ext)
+                        path = data_dir.join(dname)
+                        FileUtils.mv(data, path)
+
+                        dname
+                      end
+
+          file_sets << SourceFileSet.new(
+            id: id,
+            name: name,
+            structure: structure_file_name(id),
+            data: data.nil? ? nil : data_name,
+            tables: tables
+          )
+        end
+
+        def as_package
+          Package.new(version: version, dir: dir, file_sets: file_sets, checksum_file: checksum_file, spec_file: spec_file)
+        end
+
+        private
+
+          def data_file_name(id, ext)
+            "#{id}#{ext}"
+          end
+
+          def structure_file_name(id)
+            "#{id}__structure.sql"
+          end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/package_integrity_validator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      class PackageIntegrityValidator
+        # @param [ShellExecutor] shell
+        def initialize(shell:)
+          @shell = shell
+        end
+
+        # @param [Package] package
+        # @return [Boolean]
+        def valid?(package)
+          integrity_states(package).values.all? { |v| v == true }
+        end
+
+        # @param [Package] package
+        # @return [Hash{String => Boolean}]
+        def integrity_states(package)
+          sumstr = shell.execute <<~BASH
+            (cd "#{package.dir}" && sha256sum -c #{package.checksum_file.basename} 2>/dev/null)
+          BASH
+
+          sumstr.scan(/(.*?): (.*?)\n/)
+                .to_h
+                .transform_values { |value| value == 'OK' }
+        end
+
+        protected
+
+          # @return [ShellExecutor]
+          attr_reader :shell
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/source_file_set.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      class SourceFileSet
+        attr_reader :id, :name, :structure_path, :data_path, :tables
+
+        # @param [String] id
+        # @param [String] name
+        # @param [String] structure
+        # @param [String] data
+        # @param [Array<String>] tables
+        def initialize(id:, name:, structure:, data:, tables:)
+          @id = id
+          @name = name
+          @structure_path = structure
+          @data_path = data
+          @tables = tables
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/production/datasource_loader.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Production
+      class DatasourceLoader
+        include Mixin::SchemaNamer
+        # @param [ShellExecutor] shell
+        # @param [Database::Factory] database_factory
+        # @param [FileLoader] file_loader
+        # @param [String] database_url
+        def initialize(shell:, database_factory:, file_loader:, database_url:)
+          @shell = shell
+          @database_factory = database_factory
+          @file_loader = file_loader
+          @database_url = database_url
+        end
+
+        # @param [Package::Package] package
+        # @param [Array<String>, nil] only
+        # @param [Array<String>] without
+        #   If nil, all datasets are loaded.
+        #   If present, only listed datasets are loaded.
+        #   Structures are ALWAYS loaded
+        def load_package(package, only: nil, without: [])
+          file_sets = if only.nil?
+                        package.file_sets.select(&:data_path)
+                      else
+                        sets_by_name = package.file_sets.map { |fs| [fs.name, fs] }.to_h
+
+                        missing, present = only.map { |name| [name, sets_by_name.fetch(name, nil)] }
+                                               .partition { |(_name, value)| value.nil? }
+
+                        if missing.any?
+                          puts "[ NOK ] Datasources #{missing.map(&:first).join(', ')} don't exist!"
+                          return
+                        end
+
+                        present.map(&:second)
+                               .select(&:data_path)
+                      end
+
+          file_sets = file_sets.reject { |fs| without.include?(fs.name) }
+
+          load_structure_files(package.structure_files, schema: version_to_schema(package.version))
+
+          file_sets.map do |fs|
+            Thread.new do
+              puts "Loading #{fs.name}"
+              file_loader.load_file(package.data_path(fs))
+              puts '[  OK ] '.green + fs.name.yellow
+            end
+          end.each(&:join)
+
+          lock_tables(package)
+        end
+
+        private
+
+          # @return [Database::Factory]
+          attr_reader :database_factory
+          # @return [ShellExecutor]
+          attr_reader :shell
+          # @return [FileLoader]
+          attr_reader :file_loader
+          # @return [String]
+          attr_reader :database_url
+
+          def load_structure_files(files, schema:)
+            database = database_factory.new_instance(url: database_url)
+            database.prepend_search_path(schema) do
+              files.each do |file|
+                database.query(file.read)
+              end
+            end
+          end
+
+          # @param [Package::Package] package
+          def lock_tables(package)
+            database = database_factory.new_instance(url: database_url)
+
+            schema = version_to_schema(package.version)
+
+            database.prepend_search_path schema do
+              database.query <<~SQL
+                CREATE OR REPLACE FUNCTION #{schema}.deny_changes()
+                  RETURNS TRIGGER
+                AS $$
+                  BEGIN
+                    RAISE EXCEPTION '% denied on % (master data)', TG_OP, TG_RELNAME;
+                  END;
+                $$
+                LANGUAGE plpgsql;
+              SQL
+              package.file_sets.flat_map(&:tables).each do |table_name|
+                database.query <<~SQL
+                  CREATE TRIGGER deny_changes
+                    BEFORE INSERT
+                        OR UPDATE
+                        OR DELETE
+                        OR TRUNCATE
+                    ON #{schema}.#{table_name}
+                    FOR EACH STATEMENT
+                      EXECUTE PROCEDURE #{schema}.deny_changes()
+                SQL
+              end
+            end
+          end
+      end
+    end
+  end
+end

data/lib/lexicon/common/production/file_loader.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Production
+      class FileLoader
+        # @param [ShellExecutor] shell
+        # @param [String] database_url
+        def initialize(shell:, database_url:)
+          @shell = shell
+          @database_url = database_url
+        end
+
+        # @param [Pathname] data_file
+        # @return [Boolean]
+        def load_file(data_file)
+          if data_file.basename.to_s =~ /\.sql\z/
+            load_sql(data_file)
+          elsif data_file.basename.to_s =~ /\.sql\.gz\z/
+            load_archive(data_file)
+          else
+            raise StandardError.new("Unknown file type: #{data_file.basename}")
+          end
+        end
+
+        # @param [Pathname] archive
+        # @return [Boolean]
+        def load_archive(archive)
+          shell.execute <<~BASH
+            cat '#{archive}' | gzip -d | psql '#{database_url}'
+          BASH
+
+          true
+        end
+
+        # @param [Pathname] file
+        # @return [Boolean]
+        def load_sql(file)
+          shell.execute <<~BASH
+            echo psql '#{database_url}' < '#{file}'
+          BASH
+
+          true
+        end
+
+        private
+
+          # @return [String]
+          attr_reader :database_url
+          # @return [ShellExecutor]
+          attr_reader :shell
+      end
+    end
+  end
+end

data/lib/lexicon/common/remote/package_downloader.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Remote
+      class PackageDownloader < RemoteBase
+        # @param [S3Client] s3
+        # @param [Pathname] out_dir
+        # @param [DirectoryPackageLoader] package_loader
+        def initialize(s3:, out_dir:, package_loader:)
+          super(s3: s3)
+          @out_dir = out_dir
+          @package_loader = package_loader
+        end
+
+        # @param [Semantic::Version] version
+        # @return [Boolean]
+        def download(version)
+          bucket = version.to_s
+
+          if s3.bucket_exist?(bucket)
+            Dir.mktmpdir(nil, out_dir) do |tmp_dir|
+              tmp_dir = Pathname.new(tmp_dir)
+
+              s3.raw.get_object(
+                bucket: bucket,
+                key: Package::Package::SPEC_FILE_NAME,
+                response_target: tmp_dir.join(Package::Package::SPEC_FILE_NAME).to_s
+              )
+              s3.raw.get_object(
+                bucket: bucket,
+                key: Package::Package::CHECKSUM_FILE_NAME,
+                response_target: tmp_dir.join(Package::Package::CHECKSUM_FILE_NAME).to_s
+              )
+
+              package = package_loader.load_package(tmp_dir.basename.to_s)
+              if !package.nil?
+                puts "[  OK ] Found package with key #{version}, version is #{package.version}".green
+
+                FileUtils.mkdir_p package.data_dir
+
+                package.structure_files.map do |file|
+                  Thread.new do
+                    s3.raw.get_object(bucket: bucket, key: "data/#{file.basename.to_s}", response_target: file.to_s)
+                    puts "[  OK ] Downloaded #{file.basename}".green
+                  end
+                end.each(&:join)
+
+                package.file_sets.map do |fs|
+                  Thread.new do
+                    path = package.data_path(fs)
+                    s3.raw.get_object(bucket: bucket, key: "data/#{path.basename.to_s}", response_target: path.to_s)
+                    puts "[  OK ] Downloaded #{path.basename}".green
+                  end
+                end.each(&:join)
+
+                dest_dir = out_dir.join(version.to_s)
+                FileUtils.mkdir_p(dest_dir)
+                tmp_dir.children.each do |child|
+                  FileUtils.mv(child.to_s, dest_dir.join(child.basename).to_s)
+                end
+
+                true
+              else
+                puts "[ NOK ] The remote contains a bucket '#{version}' but it does not contains a valid package.".red
+
+                false
+              end
+            end
+          else
+            false
+          end
+        end
+
+        private
+
+          # @return [DirectoryPackageLoader]
+          attr_reader :package_loader
+          # @return [Pathname]
+          attr_reader :out_dir
+      end
+    end
+  end
+end

data/lib/lexicon/common/remote/package_uploader.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Remote
+      class PackageUploader < RemoteBase
+        include Mixin::LoggerAware
+
+        # @param [Package] package
+        # @return [Boolean]
+        def upload(package)
+          bucket_name = package.version.to_s
+          if !s3.bucket_exist?(bucket_name)
+            s3.create_bucket(bucket: bucket_name)
+            puts 'Uploading structures...'
+
+            upload_files(*package.structure_files, bucket: bucket_name, prefix: 'data')
+            puts '[  OK ] Structure uploaded.'.green
+
+            data_files = package.file_sets
+                                .select(&:data_path)
+                                .map { |fs| package.data_path(fs) }
+
+            upload_files(*data_files, bucket: bucket_name, prefix: 'data') do |path|
+              puts "[  OK ] #{path.basename}".green
+            end
+
+            upload_files(package.checksum_file, package.spec_file, bucket: bucket_name) do |path|
+              puts "[  OK ] #{path.basename}".green
+            end
+
+            true
+          else
+            false
+          end
+        rescue StandardError => e
+          log_error(e)
+
+          false
+        end
+
+        private
+
+          # @param [Array<Pathname>] files
+          #
+          # @yieldparam [Pathname] path
+          def upload_files(*files, bucket:, prefix: nil)
+            files.each do |path|
+              path.open do |f|
+                s3.put_object(bucket: bucket, key: [prefix, path.basename.to_s].compact.join('/'), body: f)
+              end
+              yield path if block_given?
+            end
+          end
+      end
+    end
+  end
+end

data/lib/lexicon/common/remote/remote_base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Remote
+      class RemoteBase
+        # @param [S3Client] s3
+        def initialize(s3:)
+          @s3 = s3
+        end
+
+        private
+
+          # @return [S3Client]
+          attr_reader :s3
+      end
+    end
+  end
+end

data/lib/lexicon/common/remote/s3_client.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Remote
+      class S3Client
+        # @return [Aws::S3::Client]
+        attr_reader :raw
+
+        # @param [Aws::S3::Client] raw
+        def initialize(raw:)
+          @raw = raw
+        end
+
+        # @return [Array<Object>]
+        def ls(bucket)
+          raw.list_objects_v2(bucket: bucket)
+            .to_h
+            .fetch(:contents, [])
+        end
+
+        # @param [String] name
+        # @return [Boolean]
+        def bucket_exist?(name)
+          if raw.head_bucket(bucket: name)
+            true
+          else
+            false
+          end
+        rescue StandardError
+          false
+        end
+
+      end
+    end
+  end
+end

data/lib/lexicon/common/schema/validator_factory.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Schema
+      class ValidatorFactory
+        # @param [Pathname] schema_path
+        def initialize(schema_path)
+          @schema_path = schema_path
+        end
+
+        # @return [JSONSchemer::Schema]
+        def build
+          JSONSchemer.schema(schema_path)
+        end
+
+        private
+
+          # @return [Pathname]
+          attr_reader :schema_path
+      end
+    end
+  end
+end

data/lib/lexicon/common/shell_executor.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    class ShellExecutor
+      include Mixin::Finalizable
+      include Mixin::LoggerAware
+
+      def initialize
+        @command_dir = Dir.mktmpdir
+      end
+
+      # @param [String] command
+      # @return [String]
+      def execute(command)
+        log(command.cyan)
+
+        cmd = Tempfile.new('command-', @command_dir)
+        cmd.write <<~BASH
+          #!/usr/bin/env bash
+          set -e
+
+          #{command}
+        BASH
+        cmd.close
+
+        `bash #{cmd.path}`
+      ensure
+        cmd.close
+        cmd.unlink
+      end
+
+      def finalize
+        if !@command_dir.nil?
+          FileUtils.rm_rf(@command_dir)
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/version.rb

@@ -0,0 +1,5 @@
+module Lexicon
+  module Common
+    VERSION = '0.1.0'.freeze
+  end
+end

data/resources/lexicon.schema.json

@@ -0,0 +1,51 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://lexicon.ekylibre.dev/lexicon.schema.json",
+  "title": "Lexicon Package JSON Schema",
+  "description": "",
+  "type": "object",
+  "properties": {
+    "version": {
+      "description": "The version of the packaged version",
+      "type": "string",
+      "$comment": "Regex is from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string",
+      "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"
+    },
+    "content": {
+      "type": "object",
+      "patternProperties": {
+        "^[a-z][a-z_]*$": {
+          "type": "object",
+          "properties": {
+            "name": {
+              "type": "string"
+            },
+            "structure": {
+              "type": "string"
+            },
+            "data": {
+              "type": "string"
+            },
+            "tables": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "additionalItems": false
+            }
+          },
+          "required": [
+            "name",
+            "structure",
+            "tables"
+          ]
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "required": [
+    "version",
+    "content"
+  ]
+}

metadata

@@ -0,0 +1,207 @@
+--- !ruby/object:Gem::Specification
+name: lexicon-common
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ekylibre developers
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-03-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.84'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.84'
+- !ruby/object:Gem::Dependency
+  name: colored
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: json_schemer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.16
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.16
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: semantic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+description:
+email:
+- dev@ekylibre.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lexicon-common.gemspec
+- lib/lexicon-common.rb
+- lib/lexicon/common.rb
+- lib/lexicon/common/database/database.rb
+- lib/lexicon/common/database/factory.rb
+- lib/lexicon/common/mixin/container_aware.rb
+- lib/lexicon/common/mixin/finalizable.rb
+- lib/lexicon/common/mixin/logger_aware.rb
+- lib/lexicon/common/mixin/schema_namer.rb
+- lib/lexicon/common/package/directory_package_loader.rb
+- lib/lexicon/common/package/package.rb
+- lib/lexicon/common/package/package_builder.rb
+- lib/lexicon/common/package/package_integrity_validator.rb
+- lib/lexicon/common/package/source_file_set.rb
+- lib/lexicon/common/production/datasource_loader.rb
+- lib/lexicon/common/production/file_loader.rb
+- lib/lexicon/common/remote/package_downloader.rb
+- lib/lexicon/common/remote/package_uploader.rb
+- lib/lexicon/common/remote/remote_base.rb
+- lib/lexicon/common/remote/s3_client.rb
+- lib/lexicon/common/schema/validator_factory.rb
+- lib/lexicon/common/shell_executor.rb
+- lib/lexicon/common/version.rb
+- resources/lexicon.schema.json
+homepage: https://www.ekylibre.com
+licenses:
+- AGPL-3.0-only
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: Common classes and services for Ekylibre's Lexicon
+test_files: []