sample_data_dump 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9cc7e8b7228910401bba4a6183eae46a28c4a3b2
-  data.tar.gz: 2170275aa22f92f616dc221695fdd291bad88ef8
+  metadata.gz: 42b460018084e41b910908bbafb017a58f4cd918
+  data.tar.gz: e09b53a900e7859880eb521d30b459e33d7b33c3
 SHA512:
-  metadata.gz: be2af199d1d6d3ff13e4b9c49a181cc96aa8fcc573fcc12a64e2d199e5b69548acd1fa8a98d35b29448e9cb1dfabb0687af5b8f86127bb83c60a14ee7e516566
-  data.tar.gz: 43044756c988a517ec8b76474ed4963b70d33e4e13b8d731d7e2ef02f0cb392b45c1c5ffe9321350f301c7f8407bd2514499d2b08e9fe54b40c70e196d212a4a
+  metadata.gz: 04f4391e631cb33f8fe5e9c7d443d8ad65ab2d9e89a274477cb8bcb9af178de192e970872a6f1f7c03337453894fde2e56b0b82811c588ad7295060f5afb4f50
+  data.tar.gz: 2ec63119a0593f6285da5569762a4011cd3742d8add99f0b0070d64131d60caddd752fcb28ddc85f083326fe69f314cd55a4cc7d07e73d0b180a4e03f4de0265

data/.gitignore

@@ -1 +1,3 @@
+compacted_dump/
 coverage/
+.rspec_status

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.0.2
+
+  * Split all I/O across multiple gateways based on data type
+  * Changed use cases to commands; use cases imply an actor
+
 0.0.1
 
    * First release

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sample_data_dump (0.0.1)
+    sample_data_dump (0.0.2)
       dry-matcher
       dry-monads
       dry-struct

data/README.md

@@ -6,12 +6,70 @@ This gem provides:
 
 ## Usage
 
-1. Create a class that implements the persistence interface.
-2. Pass it to the appropriate use case.
-3. Use the result to take your next action.
+1. Create a class that implements the CompressedDumpStorageGateway interface.
+2. Create a class that implements the DataStoreGateway interface.
+3. Initialize your gateway classes and a Settings class.
+4. Pass the gateways and settings to the appropriate command.
+5. Use the result to take your next action.
 
 ```
-SampleDataDump::UseCases::ActorDownloadsSampleDataFiles.new(persistence).result
+settings = SampleDataDump::Settings.new(
+  compacted_dump_directory: '/tmp/compacted_dumps',
+  config_file_path: 'config/sample_data_dump.yml'
+)
+local_file_system = SampleDataDump::Gateways::LocalFileSystem.new(settings)
+s3_storage = AmazonS3CompressedDumpStorageGateway.new(my_amazon_settings)
+postgres_db = PostgresDataStoreGateway.new(my_postgres_settings)
+
+SampleDataDump::Commands::DownloadCompressedDataDumps.new(local_file_system, s3_storage, settings).result.bind do
+  SampleDataDump::Commands::DecompressAndLoadSampleDataDumps.new(local_file_system, postgres_db, settings).result
+end
+```
+
+Some useful source code for creating rake tasks:
+
+```
+namespace :sample_data do
+  desc 'dumps sample data to files'
+  task dump: :environment do
+    settings = SampleDataDump::Entities::Settings.new
+    local_file_system = SampleDataDump::Gateways::LocalFileSystem.new(settings)
+    data_store = MySampleDataDumpDataStore::Gateway.new(ActiveRecord::Base.connection, settings)
+    command = SampleDataDump::Commands::DumpSampleDataToFilesAndCompress.new(local_file_system, data_store)
+    result = command.result
+    puts result.failure if result.failure?
+  end
+
+  desc 'uploads compressed sample data files to storage'
+  task upload: :environment do
+    settings = SampleDataDump::Entities::Settings.new
+    local_file_system = SampleDataDump::Gateways::LocalFileSystem.new(settings)
+    compressed_dump_storage = MySampleDataRemoteStorage::Gateway.new(ActiveRecord::Base.connection, settings)
+    command = SampleDataDump::Commands::UploadCompressedSampleDataDumps.new(local_file_system, compressed_dump_storage)
+    result = command.result
+    puts result.failure if result.failure?
+  end
+
+  desc 'downloads sample data files from storage'
+  task download: :environment do
+    settings = SampleDataDump::Entities::Settings.new
+    local_file_system = SampleDataDump::Gateways::LocalFileSystem.new(settings)
+    compressed_dump_storage = MySampleDataRemoteStorage::Gateway.new(ActiveRecord::Base.connection, settings)
+    command = SampleDataDump::Commands::DownloadCompressedDataDumps.new(local_file_system, compressed_dump_storage)
+    result = command.result
+    puts result.failure if result.failure?
+  end
+
+  desc 'populates data store from downloaded sample data files'
+  task load: :environment do
+    settings = SampleDataDump::Entities::Settings.new
+    local_file_system = SampleDataDump::Gateways::LocalFileSystem.new(settings)
+    data_store = MySampleDataDumpDataStore::Gateway.new(ActiveRecord::Base.connection, settings)
+    command = SampleDataDump::Commands::DecompressAndLoadSampleDataDumps.new(local_file_system, data_store)
+    result = command.result
+    puts result.failure if result.failure?
+  end
+end
 ```
 
 ## Installation

data/lib/sample_data_dump/commands/decompress_and_load_sample_data_dumps.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SampleDataDump
+  module Commands
+    class DecompressAndLoadSampleDataDumps
+      def initialize(local_file_system_gateway, data_store_gateway)
+        @local_file_system_gateway = local_file_system_gateway
+        @data_store_gateway = data_store_gateway
+      end
+
+      def result
+        @local_file_system_gateway.load_table_configurations.fmap do |table_configs|
+          result = process_table_configs_list(table_configs)
+          return result if result.failure?
+        end
+      end
+
+      private
+
+      def process_table_configs_list(table_configs)
+        table_configs.reverse_each do |table_config|
+          @data_store_gateway.wipe_table(table_config)
+        end
+
+        results = table_configs.map do |table_config|
+          result = process_table_config(table_config)
+          return result if result.failure?
+        end
+        results.last || Dry::Monads::Success(true)
+      end
+
+      def process_table_config(table_config)
+        @data_store_gateway.valid?(table_config).bind do
+          @local_file_system_gateway.decompress_compressed_dump_file(table_config).bind do
+            @data_store_gateway.load_dump_file(table_config).bind do
+              @data_store_gateway.reset_sequence(table_config)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sample_data_dump/commands/download_compressed_sample_data_dumps.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SampleDataDump
+  module Commands
+    class DownloadCompressedSampleDataDumps
+      def initialize(local_file_system_gateway, compressed_dump_storage_gateway)
+        @local_file_system_gateway = local_file_system_gateway
+        @compressed_dump_storage_gateway = compressed_dump_storage_gateway
+      end
+
+      def result
+        @local_file_system_gateway.clean_dump_directory.bind do
+          @local_file_system_gateway.load_table_configurations.fmap do |table_configs|
+            result = process_table_configs_list(table_configs)
+            return result if result.failure?
+          end
+        end
+      end
+
+      private
+
+      def process_table_configs_list(table_configs)
+        results = table_configs.map do |table_config|
+          result = @compressed_dump_storage_gateway.retrieve_compressed_dump_file(table_config)
+          return result if result.failure?
+        end
+        results.last || Dry::Monads::Success(true)
+      end
+    end
+  end
+end

data/lib/sample_data_dump/commands/dump_sample_data_to_files_and_compress.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module SampleDataDump
+  module Commands
+    class DumpSampleDataToFilesAndCompress
+      def initialize(local_file_system_gateway, data_store_gateway)
+        @local_file_system_gateway = local_file_system_gateway
+        @data_store_gateway = data_store_gateway
+      end
+
+      def result
+        @local_file_system_gateway.clean_dump_directory.bind do
+          @local_file_system_gateway.load_table_configurations.fmap do |table_configs|
+            result = process_table_configs_list(table_configs)
+            return result if result.failure?
+          end
+        end
+      end
+
+      private
+
+      def process_table_configs_list(table_configs)
+        results = table_configs.map do |table_config|
+          result = process_table_config(table_config)
+          return result if result.failure?
+        end
+        results.last || Dry::Monads::Success(true)
+      end
+
+      def process_table_config(table_config)
+        @data_store_gateway.valid?(table_config).bind do
+          @data_store_gateway.dump_to_local_file(table_config).bind do
+            @local_file_system_gateway.compress_dump_file(table_config)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sample_data_dump/commands/upload_compressed_sample_data_dumps.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module SampleDataDump
+  module Commands
+    class UploadCompressedSampleDataDumps
+      def initialize(local_file_system_gateway, compressed_dump_storage_gateway)
+        @local_file_system_gateway = local_file_system_gateway
+        @compressed_dump_storage_gateway = compressed_dump_storage_gateway
+      end
+
+      def result
+        @local_file_system_gateway.load_table_configurations.fmap do |table_configs|
+          result = process_table_configs_list(table_configs)
+          return result if result.failure?
+        end
+      end
+
+      private
+
+      def process_table_configs_list(table_configs)
+        results = table_configs.map do |config|
+          result = @compressed_dump_storage_gateway.store_compressed_dump_file(config)
+          return result if result.failure?
+        end
+        results.last || Dry::Monads::Success(true)
+      end
+    end
+  end
+end

data/lib/sample_data_dump/entities/settings.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'dry/struct'
+require 'sample_data_dump/types'
+
+module SampleDataDump
+  module Entities
+    class Settings < Dry::Struct
+      attribute :compacted_dump_directory, Types::Strict::String.default('compacted_dump')
+      attribute :config_file_path, Types::Strict::String.default('config/sample_data_dump.yml')
+    end
+  end
+end

data/lib/sample_data_dump/gateways/local_file_system.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'dry/monads/all'
+require 'sample_data_dump/entities/table_configuration'
+require 'sample_data_dump/helpers/dump_file'
+require 'sample_data_dump/interfaces/local_file_system_gateway'
+require 'yaml'
+
+module SampleDataDump
+  module Gateways
+    class LocalFileSystem
+      extend Forwardable
+      implements_interface Interfaces::LocalFileSystemGateway
+
+      def initialize(settings)
+        @settings = settings
+        @sample_data_dump_config = YAML.load_file(config_file_path)
+
+        `mkdir -p #{compacted_dump_directory}`
+      end
+
+      def clean_dump_directory
+        `rm -rf #{compacted_dump_directory}/*.sql`
+        `rm -rf #{compacted_dump_directory}/*.sql.tar.gz`
+        Dry::Monads::Success(true)
+      end
+
+      def compress_dump_file(table_configuration)
+        dump_file_helper = Helpers::DumpFile.new(table_configuration, @settings)
+        uncompressed_file_path = dump_file_helper.local_dump_file_path
+        uncompressed_file_name = dump_file_helper.local_dump_file_name
+        compressed_file_name = dump_file_helper.local_compressed_dump_file_name
+        compressed_file_path = dump_file_helper.local_compressed_dump_file_path
+
+        unless File.exists?(uncompressed_file_path)
+          return Dry::Monads::Failure("No file found at #{uncompressed_file_path}")
+        end
+
+        `cd #{compacted_dump_directory}; tar -zcf #{compressed_file_name} #{uncompressed_file_name}`
+
+        if File.exist?(compressed_file_path)
+          Dry::Monads::Success(compressed_file_path)
+        else
+          Dry::Monads::Failure("No file found at #{compressed_file_path} after compressing")
+        end
+      end
+
+      def decompress_compressed_dump_file(table_configuration)
+        dump_file_helper = Helpers::DumpFile.new(table_configuration, @settings)
+        compressed_file_name = dump_file_helper.local_compressed_dump_file_name
+        compressed_file_path = dump_file_helper.local_compressed_dump_file_path
+        unless File.exist?(compressed_file_path)
+          return Dry::Monads::Failure("#{compressed_file_path} does not exist for decompressing!")
+        end
+
+        `cd #{compacted_dump_directory}; tar -zxf #{compressed_file_name}`
+        Dry::Monads::Success(true)
+      end
+
+      def load_table_configurations
+        result = []
+        @sample_data_dump_config.each_pair do |schema_name, schema_config_hash|
+          schema_config_hash.each_pair do |table_name, table_config_hash|
+            dump_where = table_config_hash&.[]('dump_where') || '1 = 1'
+            obfuscate_columns = table_config_hash&.[]('obfuscate_columns') || []
+            result << Entities::TableConfiguration.new(
+              schema_name: schema_name,
+              table_name: table_name,
+              dump_where: dump_where,
+              obfuscate_columns: obfuscate_columns
+            )
+          end
+        end
+        Dry::Monads::Success(result)
+      end
+
+      private
+
+      attr_reader :settings
+      def_delegators :settings, :compacted_dump_directory, :config_file_path
+    end
+  end
+end

data/lib/sample_data_dump/helpers/dump_file.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SampleDataDump
+  module Helpers
+    class DumpFile
+      extend Forwardable
+
+      def initialize(table_configuration, settings)
+        @table_configuration = table_configuration
+        @settings = settings
+      end
+
+      def local_compressed_dump_file_name
+        "#{@table_configuration.schema_name}.#{@table_configuration.table_name}.sql.tar.gz"
+      end
+
+      def local_compressed_dump_file_path
+        "#{compacted_dump_directory}/#{local_compressed_dump_file_name}"
+      end
+
+      def local_dump_file_name
+        "#{@table_configuration.schema_name}.#{@table_configuration.table_name}.sql"
+      end
+
+      def local_dump_file_path
+        "#{compacted_dump_directory}/#{local_dump_file_name}"
+      end
+
+      private
+
+      attr_reader :settings
+      def_delegators :settings, :compacted_dump_directory
+    end
+  end
+end

data/lib/sample_data_dump/interfaces/compressed_dump_storage_gateway.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'duckface'
+
+module SampleDataDump
+  module Interfaces
+    module CompressedDumpStorageGateway
+      extend Duckface::ActsAsInterface
+
+      def retrieve_compressed_dump_file(_table_configuration)
+        raise NotImplementedError
+      end
+
+      def store_compressed_dump_file(_table_configuration)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/sample_data_dump/interfaces/{persistence.rb → data_store_gateway.rb}

@@ -4,14 +4,10 @@ require 'duckface'
 
 module SampleDataDump
   module Interfaces
-    module Persistence
+    module DataStoreGateway
       extend Duckface::ActsAsInterface
 
-      def decompress_compressed_dump_file(_table_configuration)
-        raise NotImplementedError
-      end
-
-      def dump_and_compress_data_to_local_file(_table_configuration)
+      def dump_to_local_file(_table_configuration)
         raise NotImplementedError
       end
 
@@ -19,22 +15,10 @@ module SampleDataDump
         raise NotImplementedError
       end
 
-      def load_table_configurations
-        raise NotImplementedError
-      end
-
-      def move_compressed_dump_file_to_central_location(_table_configuration)
-        raise NotImplementedError
-      end
-
       def reset_sequence(_table_configuration)
         raise NotImplementedError
       end
 
-      def retrieve_compressed_dump_file_from_central_location(_table_configuration)
-        raise NotImplementedError
-      end
-
       def valid?(_table_configuration)
         raise NotImplementedError
       end

data/lib/sample_data_dump/interfaces/local_file_system_gateway.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'duckface'
+
+module SampleDataDump
+  module Interfaces
+    module LocalFileSystemGateway
+      extend Duckface::ActsAsInterface
+
+      def clean_dump_directory
+        raise NotImplementedError
+      end
+
+      def compress_dump_file(_table_configuration)
+        raise NotImplementedError
+      end
+
+      def decompress_compressed_dump_file(_table_configuration)
+        raise NotImplementedError
+      end
+
+      def load_table_configurations
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/sample_data_dump/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SampleDataDump
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sample_data_dump
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Bellroy Dev Team
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-30 00:00:00.000000000 Z
+date: 2018-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dry-matcher
@@ -132,7 +132,6 @@ files:
 - ".gitignore"
 - ".reek.yml"
 - ".rspec"
-- ".rspec_status"
 - ".rubocop.yml"
 - ".ruby-version"
 - CHANGELOG.md
@@ -141,13 +140,18 @@ files:
 - Guardfile
 - README.md
 - lib/sample_data_dump.rb
+- lib/sample_data_dump/commands/decompress_and_load_sample_data_dumps.rb
+- lib/sample_data_dump/commands/download_compressed_sample_data_dumps.rb
+- lib/sample_data_dump/commands/dump_sample_data_to_files_and_compress.rb
+- lib/sample_data_dump/commands/upload_compressed_sample_data_dumps.rb
+- lib/sample_data_dump/entities/settings.rb
 - lib/sample_data_dump/entities/table_configuration.rb
-- lib/sample_data_dump/interfaces/persistence.rb
+- lib/sample_data_dump/gateways/local_file_system.rb
+- lib/sample_data_dump/helpers/dump_file.rb
+- lib/sample_data_dump/interfaces/compressed_dump_storage_gateway.rb
+- lib/sample_data_dump/interfaces/data_store_gateway.rb
+- lib/sample_data_dump/interfaces/local_file_system_gateway.rb
 - lib/sample_data_dump/types.rb
-- lib/sample_data_dump/use_cases/actor_downloads_sample_data_files.rb
-- lib/sample_data_dump/use_cases/actor_dumps_sample_data_to_files.rb
-- lib/sample_data_dump/use_cases/actor_loads_sample_data_files.rb
-- lib/sample_data_dump/use_cases/actor_uploads_sample_data_files.rb
 - lib/sample_data_dump/version.rb
 - sample_data_dump.gemspec
 homepage: http://www.bellroy.com

data/.rspec_status

@@ -1,9 +0,0 @@
-example_id                                                                             | status | run_time        |
--------------------------------------------------------------------------------------- | ------ | --------------- |
-./spec/duckface/acts_as_interface_spec.rb[1:1:1:1:1]                                   | passed | 0.00232 seconds |
-./spec/duckface/acts_as_interface_spec.rb[1:1:1:1:2]                                   | passed | 0.00333 seconds |
-./spec/lib/sample_data_dump/entities/table_configuration_spec.rb[1:1:1]                | passed | 0.00021 seconds |
-./spec/lib/sample_data_dump/use_cases/actor_downloads_sample_data_files_spec.rb[1:1:1] | passed | 0.01413 seconds |
-./spec/lib/sample_data_dump/use_cases/actor_dumps_sample_data_to_files_spec.rb[1:1:1]  | passed | 0.00108 seconds |
-./spec/lib/sample_data_dump/use_cases/actor_loads_sample_data_files_spec.rb[1:1:1]     | passed | 0.00174 seconds |
-./spec/lib/sample_data_dump/use_cases/actor_uploads_sample_data_files_spec.rb[1:1:1]   | passed | 0.001 seconds   |

data/lib/sample_data_dump/use_cases/actor_downloads_sample_data_files.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module SampleDataDump
-  module UseCases
-    class ActorDownloadsSampleDataFiles
-      def initialize(persistence)
-        @persistence = persistence
-      end
-
-      def result
-        @persistence.load_table_configurations.fmap do |table_configurations|
-          result = process_table_configurations_list(table_configurations)
-          return result if result.failure?
-        end
-      end
-
-      private
-
-      def process_table_configurations_list(table_configurations)
-        results = table_configurations.map do |table_configuration|
-          result = process_table_configuration(table_configuration)
-          return result if result.failure?
-        end
-        results.last || Dry::Monads::Success(true)
-      end
-
-      def process_table_configuration(table_configuration)
-        @persistence.valid?(table_configuration).bind do
-          @persistence.retrieve_compressed_dump_file_from_central_location(table_configuration)
-        end
-      end
-    end
-  end
-end

data/lib/sample_data_dump/use_cases/actor_dumps_sample_data_to_files.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module SampleDataDump
-  module UseCases
-    class ActorDumpsSampleDataToFiles
-      def initialize(persistence)
-        @persistence = persistence
-      end
-
-      def result
-        @persistence.load_table_configurations.fmap do |table_configurations|
-          result = process_table_configurations_list(table_configurations)
-          return result if result.failure?
-        end
-      end
-
-      private
-
-      def process_table_configurations_list(table_configurations)
-        results = table_configurations.map do |table_configuration|
-          result = process_table_configuration(table_configuration)
-          return result if result.failure?
-        end
-        results.last || Dry::Monads::Success(true)
-      end
-
-      def process_table_configuration(table_configuration)
-        @persistence.valid?(table_configuration).bind do
-          @persistence.dump_and_compress_data_to_local_file(table_configuration)
-        end
-      end
-    end
-  end
-end

data/lib/sample_data_dump/use_cases/actor_loads_sample_data_files.rb

@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module SampleDataDump
-  module UseCases
-    class ActorLoadsSampleDataFiles
-      def initialize(persistence)
-        @persistence = persistence
-      end
-
-      def result
-        @persistence.load_table_configurations.fmap do |table_configurations|
-          result = process_table_configurations_list(table_configurations)
-          return result if result.failure?
-        end
-      end
-
-      private
-
-      def process_table_configurations_list(table_configurations)
-        table_configurations.reverse_each do |table_configuration|
-          @persistence.wipe_table(table_configuration)
-        end
-
-        results = table_configurations.map do |table_configuration|
-          result = process_table_configuration(table_configuration)
-          return result if result.failure?
-        end
-        results.last || Dry::Monads::Success(true)
-      end
-
-      def process_table_configuration(table_configuration)
-        @persistence.valid?(table_configuration).bind do
-          @persistence.decompress_compressed_dump_file(table_configuration).bind do
-            @persistence.load_dump_file(table_configuration).bind do
-              @persistence.reset_sequence(table_configuration)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/sample_data_dump/use_cases/actor_uploads_sample_data_files.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module SampleDataDump
-  module UseCases
-    class ActorUploadsSampleDataFiles
-      def initialize(persistence)
-        @persistence = persistence
-      end
-
-      def result
-        @persistence.load_table_configurations.fmap do |table_configurations|
-          result = process_table_configurations_list(table_configurations)
-          return result if result.failure?
-        end
-      end
-
-      private
-
-      def process_table_configurations_list(table_configurations)
-        results = table_configurations.map do |table_configuration|
-          result = process_table_configuration(table_configuration)
-          return result if result.failure?
-        end
-        results.last || Dry::Monads::Success(true)
-      end
-
-      def process_table_configuration(table_configuration)
-        @persistence.valid?(table_configuration).bind do
-          @persistence.move_compressed_dump_file_to_central_location(table_configuration)
-        end
-      end
-    end
-  end
-end