lexicon-common 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb97bec1598709b8fefb46c28055ac8291edd138177995e08616de3b06dfbdb8
-  data.tar.gz: 20db194da73907714cb53ae861bd650ded613f71ef99ae1c93db370c8547c830
+  metadata.gz: 5b24914e5f9dbd3d4261ec1b6c04741958a5d9fcfccb9dde8fd14b92b5c19b57
+  data.tar.gz: 7b2a40b0e4e033dbc111804a5c6a6a7105fdd2ac991d72baeb7022e66f585913
 SHA512:
-  metadata.gz: aa5b4f83c9647382dabdaa4dd09250fd4e165476ca3984740763bd4b2cd8f54ac41e73379a5988391a3336d31e9928ca9332598216b102829dff1efe28ab14dd
-  data.tar.gz: 296628e759a78533b06160d653ab6e1299d0d1378964ac20d5350c7b83892d0b385b4d78f1569c1844591c014db2a0130f9249e423946e7d3e38095f28029f5a
+  metadata.gz: 0b7e024f856d40571826aa8237cfbd0b1ea55be19bf31f997ce5f65cf04c561ffaa43d4b5c23b5fcdda05ae7f3b48629ae0ca68c0c91903fa362e4674e3cdbd4
+  data.tar.gz: 7c225eb0b67789074fa76e5a50bcc61439503b5fb579d811077732b7f9763efc822f7f4c12d2aa314d5b0719ef3820d0eff097e08aaa3c167d442f48665dbe64

data/lexicon-common.gemspec

@@ -20,6 +20,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'aws-sdk-s3', '~> 1.84'
   spec.add_dependency 'colored', '~> 1.2'
+  spec.add_dependency 'concurrent-ruby', '~> 1.1'
+  spec.add_dependency 'corindon', '~> 0.8.0'
   spec.add_dependency 'json_schemer', '~> 0.2.16'
   spec.add_dependency 'pg', '~> 1.2'
   spec.add_dependency 'semantic', '~> 1.6'

data/lib/lexicon/common/database/database.rb

@@ -50,7 +50,7 @@ module Lexicon
           drop_schema(schema, cascade: true)
         end
 
-        def drop_schema(name, cascade: false)
+        def drop_schema(name, cascade: false, if_exists: false)
           cascade = if cascade
                       ' CASCADE'
                     else
@@ -58,7 +58,7 @@ module Lexicon
                     end
 
           query <<~SQL
-            DROP SCHEMA "#{name}"#{cascade};
+            DROP SCHEMA #{if_exists ? 'IF EXISTS ' : ''}"#{name}"#{cascade};
           SQL
         end
 
@@ -95,6 +95,28 @@ module Lexicon
           @connection.copy_data(sql) { block.call(put_data) }
         end
 
+        # @param [#to_s] table
+        # @param [#to_s | nil] schema
+        # @return [Boolean]
+        def table_exists?(table, schema: nil)
+          schema = search_path.first if schema.nil?
+
+          query(<<~SQL, table, schema).any?
+            SELECT table_name FROM information_schema.tables
+            WHERE table_name = $1 AND table_schema = $2
+          SQL
+        end
+
+        # @param [#to_s] schema_name
+        # @return [Boolean]
+        def schema_exists?(schema_name)
+          query(<<~SQL, schema_name).count > 0
+            SELECT "schema_name"
+            FROM "information_schema"."schemata"
+            WHERE "schema_name" = $1
+          SQL
+        end
+
         private
 
           # @return [PG::Connection]

data/lib/lexicon/common/mixin/logger_aware.rb

@@ -16,8 +16,8 @@ module Lexicon
         def log_error(error)
           if error.nil?
             log('Error (nil)')
-          else
-            log([error.message, *error.backtrace].join("\n"))
+          elsif !logger.nil?
+            logger.error([error.message, *error.backtrace].join("\n"))
           end
         end
       end

data/lib/lexicon/common/mixin/nameable.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+using Corindon::Guards::Ext
+
+module Lexicon
+  module Common
+    module Mixin
+      module Nameable
+        # @return [String]
+        def name
+          unimplemented!
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/directory_package_loader.rb

@@ -17,7 +17,7 @@ module Lexicon
         end
 
         # @param [String] name
-        # @return [Package, nil]
+        # @return [Package::Package, nil]
         def load_package(name)
           package_dir = root_dir.join(name.to_s)
 
@@ -40,18 +40,17 @@ module Lexicon
               json = JSON.parse(spec_file.read)
 
               if @schema_validator.valid?(json)
-                version = Semantic::Version.new(json.fetch('version'))
-                file_sets = json.fetch('content').map do |id, values|
-                  SourceFileSet.new(
-                    id: id,
-                    name: values.fetch('name'),
-                    structure: values.fetch('structure'),
-                    data: values.fetch('data', nil),
-                    tables: values.fetch('tables', [])
-                  )
-                end
+                package_version = json.fetch('schema_version', 1)
+                case package_version
+                when 1
+                  load_v1(dir: dir, spec_file: spec_file, checksum_file: checksum_file, json: json)
+                when 2
+                  load_v2(dir: dir, spec_file: spec_file, checksum_file: checksum_file, json: json)
+                else
+                  log("Package version #{package_version} is not supported")
 
-                Package.new(file_sets: file_sets, version: version, dir: dir, checksum_file: checksum_file, spec_file: spec_file)
+                  nil
+                end
               else
                 log("Package at path #{dir} has invalid manifest")
 
@@ -61,6 +60,51 @@ module Lexicon
               nil
             end
           end
+
+          # @param [Pathname] checksum_file
+          # @param [Pathname] dir
+          # @param [Hash] json
+          # @param [Pathname] spec_file
+          # @return [V1::Package]
+          def load_v1(dir:, spec_file:, checksum_file:, json:)
+            version = Semantic::Version.new(json.fetch('version'))
+            file_sets = json.fetch('content').map do |id, values|
+              V1::SourceFileSet.new(
+                id: id,
+                name: values.fetch('name'),
+                structure: values.fetch('structure'),
+                data: values.fetch('data', nil),
+                tables: values.fetch('tables', [])
+              )
+            end
+
+            V1::Package.new(file_sets: file_sets, version: version, dir: dir, checksum_file: checksum_file, spec_file: spec_file)
+          end
+
+          # @param [Pathname] checksum_file
+          # @param [Pathname] dir
+          # @param [Hash] json
+          # @param [Pathname] spec_file
+          # @return [V2::Package]
+          def load_v2(dir:, spec_file:, checksum_file:, json:)
+            version = Semantic::Version.new(json.fetch('version'))
+            file_sets = json.fetch('content').map do |id, values|
+              V2::SourceFileSet.new(
+                id: id,
+                name: values.fetch('name'),
+                structure: values.fetch('structure'),
+                tables: values.fetch('tables', {})
+              )
+            end
+
+            V2::Package.new(
+              file_sets: file_sets,
+              version: version,
+              dir: dir,
+              checksum_file: checksum_file,
+              spec_file: spec_file
+            )
+          end
       end
     end
   end

data/lib/lexicon/common/package/package.rb

@@ -11,65 +11,33 @@ module Lexicon
         attr_reader :checksum_file, :spec_file
         # @return [Pathname]
         attr_reader :dir
-        # @return [Array<SourceFileSet>]
-        attr_reader :file_sets
         # @return [Semantic::Version]
         attr_reader :version
+        # @return [Integer]
+        attr_reader :schema_version
 
-        # @param [Array<SourceFileSet>] file_sets
-        # @param [Pathname] dir
         # @param [Pathname] checksum_file
+        # @param [Pathname] dir
+        # @param [Integer] schema_version
+        # @param [Pathname] spec_file
         # @param [Semantic::Version] version
-        def initialize(file_sets:, version:, dir:, checksum_file:, spec_file:)
+        def initialize(checksum_file:, dir:, schema_version:, spec_file:, version:)
           @checksum_file = checksum_file
           @dir = dir
-          @file_sets = file_sets
+          @schema_version = schema_version
           @spec_file = spec_file
           @version = version
         end
 
         # @return [Boolean]
         def valid?
-          checksum_file.exist? && dir.directory? && data_dir.directory? && all_sets_valid?
-        end
-
-        # @return [Array<Pathname>]
-        def structure_files
-          file_sets.map { |fs| dir.join(relative_structure_path(fs)) }
+          checksum_file.exist? && dir.directory? && files.all? { |f| f.path.exist? rescue false }
         end
 
-        # @param [SourceFileSet] file_set
-        # @return [Pathname]
-        def data_path(file_set)
-          dir.join(relative_data_path(file_set))
-        end
-
-        # @return [Pathname]
-        def data_dir
-          dir.join('data')
+        # @return [Array<PackageFile>] Array of File of the package
+        def files
+          []
         end
-
-        # @return [Pathname, nil]
-        def relative_data_path(file_set)
-          if file_set.data_path.nil?
-            nil
-          else
-            data_dir.basename.join(file_set.data_path)
-          end
-        end
-
-        # @return [Pathname]
-        def relative_structure_path(file_set)
-          data_dir.basename.join(file_set.structure_path)
-        end
-
-        private
-
-          def all_sets_valid?
-            file_sets.all? do |set|
-              data_dir.join(set.structure_path).exist? && !set.data_path.nil? && data_dir.join(set.data_path).exist?
-            end
-          end
       end
     end
   end

data/lib/lexicon/common/package/package_file.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      class PackageFile
+        class << self
+          def new_structure(path)
+            new(path, type: STRUCTURE)
+          end
+
+          def new_data(path)
+            new(path, type: DATA)
+          end
+        end
+
+        # @return [Pathname]
+        attr_reader :path
+
+        # @return [String]
+        def to_s
+          path.to_s
+        end
+
+        # @return [Boolean]
+        def data?
+          type == DATA
+        end
+
+        # @return [Boolean]
+        def structure?
+          type == STRUCTURE
+        end
+
+        private
+
+          attr_reader :type
+
+          DATA = 'data'
+          STRUCTURE = 'structure'
+
+          def initialize(path, type:)
+            @path = path
+            @type = type
+          end
+
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/v1/package.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V1
+        class Package < Common::Package::Package
+          # @return [Array<SourceFileSet>]
+          attr_reader :file_sets
+
+          # @param [Array<SourceFileSet>] file_sets
+          # @param [Pathname] dir
+          # @param [Pathname] checksum_file
+          # @param [Semantic::Version] version
+          def initialize(version:, spec_file:, checksum_file:, dir:, file_sets:)
+            super(
+              checksum_file: checksum_file,
+              dir: dir,
+              spec_file: spec_file,
+              schema_version: 1,
+              version: version,
+            )
+
+            @file_sets = file_sets
+          end
+
+          # @return [Boolean]
+          def valid?
+            super && data_dir.directory? && all_sets_valid?
+          end
+
+          def files
+            structures = file_sets.map { |fs| PackageFile.new_structure(relative_structure_path(fs)) }
+            data = file_sets.flat_map do |fs|
+              data_path = relative_data_path(fs)
+
+              if data_path.nil?
+                []
+              else
+                [PackageFile.new_data(data_path)]
+              end
+            end
+
+            [*structures, *data]
+          end
+
+          # @param [SourceFileSet] file_set
+          # @return [Pathname]
+          def data_path(file_set)
+            dir.join(relative_data_path(file_set))
+          end
+
+          # @return [Pathname]
+          def data_dir
+            dir.join('data')
+          end
+
+          # @return [Pathname, nil]
+          def relative_data_path(file_set)
+            if file_set.data_path.nil?
+              nil
+            else
+              data_dir.basename.join(file_set.data_path)
+            end
+          end
+
+          # @return [Pathname]
+          def relative_structure_path(file_set)
+            data_dir.basename.join(file_set.structure_path)
+          end
+
+          private
+
+            def all_sets_valid?
+              file_sets.all? do |set|
+                data_dir.join(set.structure_path).exist? && !set.data_path.nil? && data_dir.join(set.data_path).exist?
+              end
+            end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/v1/package_builder.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V1
+        class PackageBuilder < Package
+          def initialize(version:, dir:)
+            super(
+              file_sets: [],
+              version: version,
+              dir: dir,
+              checksum_file: dir.join(CHECKSUM_FILE_NAME),
+              spec_file: dir.join(SPEC_FILE_NAME)
+            )
+
+            FileUtils.mkdir_p(data_dir)
+          end
+
+          # @param [String] id
+          # @param [String] name
+          # @param [Pathname] structure
+          #   Takes ownership of the file (moves it to the correct folder)
+          # @param [Array<String>] tables
+          # @param [Pathname] data
+          #   Takes ownership of the file (moves it to the correct folder)
+          # @param [String] data_ext
+          def add_file_set(id, name:, structure:, tables:, data: nil, data_ext: '.sql')
+            # @type [Pathname] structure_file_path
+            structure_file_path = data_dir.join(structure_file_name(id))
+            FileUtils.mv(structure.to_s, structure_file_path.to_s)
+
+            # @type [Pathname] data_file_path
+            data_name = if data.nil?
+                          nil
+                        else
+                          dname = data_file_name(id, data_ext)
+                          path = data_dir.join(dname)
+                          FileUtils.mv(data, path)
+
+                          dname
+                        end
+
+            file_sets << SourceFileSet.new(
+              id: id,
+              name: name,
+              structure: structure_file_name(id),
+              data: data.nil? ? nil : data_name,
+              tables: tables
+            )
+          end
+
+          def as_package
+            Package.new(version: version, dir: dir, file_sets: file_sets, checksum_file: checksum_file, spec_file: spec_file)
+          end
+
+          private
+
+            def data_file_name(id, ext)
+              "#{id}#{ext}"
+            end
+
+            def structure_file_name(id)
+              "#{id}__structure.sql"
+            end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/v1/source_file_set.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V1
+        class SourceFileSet
+          include Mixin::Nameable
+
+          attr_reader :id, :name, :structure_path, :data_path, :tables
+
+          # @param [String] id
+          # @param [String] name
+          # @param [String] structure
+          # @param [String] data
+          # @param [Array<String>] tables
+          def initialize(id:, name:, structure:, data:, tables:)
+            @id = id
+            @name = name
+            @structure_path = structure
+            @data_path = data
+            @tables = tables
+          end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/v2/package.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V2
+        class Package < Common::Package::Package
+          # @param [Pathname] dir
+          # @param [Pathname] checksum_file
+          # @param [Semantic::Version] version
+          def initialize(version:, spec_file:, checksum_file:, dir:, file_sets:)
+            super(
+              checksum_file: checksum_file,
+              dir: dir,
+              spec_file: spec_file,
+              schema_version: 2,
+              version: version,
+            )
+
+            @file_sets = file_sets
+          end
+
+          def valid?
+            super
+          end
+
+          def files
+            file_sets.flat_map { |fs| file_set_files(fs) }
+          end
+
+          # @return [SourceFileSet]
+          attr_reader :file_sets
+
+          def data_dir
+            dir.join('data')
+          end
+
+          private
+
+            # @param [SourceFileSet] file_set
+            # @return [Array<PackageFile>]
+            def file_set_files(file_set)
+              relative_data_dir = data_dir.basename
+
+              structure_file = PackageFile.new_structure(relative_data_dir.join(file_set.structure))
+              table_files = file_set.tables
+                                    .values.flatten(1)
+                                    .map { |table_file| PackageFile.new_data(relative_data_dir.join(table_file)) }
+
+              [structure_file, *table_files]
+            end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/v2/package_builder.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V2
+        class PackageBuilder < Package
+          def initialize(version:, dir:)
+            super(
+              file_sets: [],
+              version: version,
+              dir: dir,
+              checksum_file: dir.join(CHECKSUM_FILE_NAME),
+              spec_file: dir.join(SPEC_FILE_NAME),
+            )
+
+            FileUtils.mkdir_p(data_dir)
+          end
+
+          # @param [String] id
+          # @param [String] name
+          # @param [Pathname] structure
+          #   Takes ownership of the file (moves it to the correct folder)
+          # @param [Hash{String=>Array<Pathname>}] tables
+          #   Takes ownership of the files (moves them to the correct folder)
+          def add_file_set(id, name:, structure:, tables:)
+            # @type [Pathname] structure_file_path
+            structure_file_path = data_dir.join(structure_file_name(id))
+            FileUtils.mv(structure.to_s, structure_file_path.to_s)
+
+            table_data = tables.map do |table_name, files|
+              index = 0
+
+              file_names = files.map do |file|
+                file_name = "#{table_name}_#{index}.csv.gz"
+                FileUtils.mv(file.to_s, data_dir.join(file_name))
+                index += 1
+
+                file_name
+              end
+
+              [table_name, file_names]
+            end
+
+            file_sets << SourceFileSet.new(
+              id: id,
+              name: name,
+              structure: structure_file_name(id),
+              tables: table_data.to_h
+            )
+          end
+
+          def as_package
+            Package.new(
+              checksum_file: checksum_file,
+              dir: dir,
+              file_sets: file_sets,
+              spec_file: spec_file,
+              version: version,
+            )
+          end
+
+          private
+
+            def structure_file_name(id)
+              "#{id}__structure.sql"
+            end
+        end
+      end
+    end
+  end
+end

data/lib/lexicon/common/package/v2/source_file_set.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Lexicon
+  module Common
+    module Package
+      module V2
+        class SourceFileSet
+          include Mixin::Nameable
+
+          attr_reader :id, :name, :structure, :tables
+
+          # @param [String] id
+          # @param [String] name
+          # @param [String] structure
+          # @param [Hash{String=>Array<String>}] tables
+          def initialize(id:, name:, structure:, tables:)
+            @id = id
+            @name = name
+            @structure = structure
+            @tables = tables.freeze
+          end
+        end
+      end
+    end
+  end
+end