tartarus-rb 0.2.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ffb5cafa941ec43d2c07a6745ef9b75b91c31784649bbf04187e90f6960812e0
-  data.tar.gz: 9615766c2064a4e127c836ca4cf6ad9b0f2925d08246e4486b9f9fd382884e4f
+  metadata.gz: f7669ce816882e4f9607e41afde316cbe50027291a0dd89008d3baaa9ca6b308
+  data.tar.gz: d03d89b04491a2dce92820674e94c895cac9cad706b790927aab9129f8c15b26
 SHA512:
-  metadata.gz: a97481429cc2c5faa977d75baae80c78e17f56cfa0323b76e613a91a57586bbfb6890442e340b7d9aff88eb7a155f694174109a92317d77e7b8c4c367224ae50
-  data.tar.gz: 143dc43864c453873d6154f52d1504eff11f0f88e406e69893fd62937ea8b59899e4c1fdd344adc65fab4e4d6f15c8dcee4cd5c09c14e86a19d47be2e17d403d
+  metadata.gz: e3906591489650a6e4f9ce9f3b0eaa17ab2a95797096abcf8bba9f50855a7463f0a2f1354017f3df512519775765e807ece44b4c75cbc2b3b3bd326f67a6ffcf
+  data.tar.gz: 48d503bc4bbea2be0196778d5dca18b9d9770cf069685128860f4567fdc1727f46042e009b43a86dc9697b217c8a6b5816a45eae1ed935f362ce63b42c2301e2

data/.env.sample

@@ -0,0 +1,4 @@
+AWS_REGION=""
+AWS_SECRET=""
+AWS_KEY=""
+VAULT_NAME=""

data/.gitignore

@@ -7,6 +7,8 @@
 /spec/reports/
 /tmp/
 /spec/test.db
+/spec/tmp/
+.env
 
 # rspec failure tracking
 .rspec_status

data/.travis.yml

@@ -3,6 +3,13 @@ language: ruby
 cache: bundler
 rvm:
   - 2.7.2
+env:
+  global:
+    - AWS_KEY=AWS_KEY
+    - AWS_SECRET=AWS_SECRET
+    - AWS_REGION="us-east-2"
+    - VAULT_NAME=VAULT_NAME
 before_install: gem install bundler -v 2.1.4
 services:
   - redis-server
+  - postgresql

data/Changelog.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## Master
+
+## 0.5.0
+
+- Provide ability to explicitly set the name of archivable item to have multiple ways of archiving the same model
+
+## 0.4.1
+
+- Do not make Glacier a required dependency if not used
+
+## 0.4.0
+
+- Add Glacier remote storage support to upload data before deleting it
+
+## 0.3.0
+
+- Add `delete_all_using_limit_in_batches` strategy
+
 ## 0.2.0
 - Add support for deleting and destroying in batches
 - Add integration tests with a real database and ActiveRecord

data/Gemfile.lock

@@ -1,13 +1,26 @@
 PATH
   remote: .
   specs:
-    tartarus-rb (0.2.0)
+    tartarus-rb (0.5.0)
       sidekiq (>= 5)
       sidekiq-cron (~> 1)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    actionpack (6.1.0.rc1)
+      actionview (= 6.1.0.rc1)
+      activesupport (= 6.1.0.rc1)
+      rack (~> 2.0, >= 2.0.9)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actionview (6.1.0.rc1)
+      activesupport (= 6.1.0.rc1)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
     activemodel (6.1.0.rc1)
       activesupport (= 6.1.0.rc1)
     activerecord (6.1.0.rc1)
@@ -19,21 +32,74 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    aws-eventstream (1.1.0)
+    aws-partitions (1.424.0)
+    aws-sdk-core (3.112.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.239.0)
+      aws-sigv4 (~> 1.1)
+      jmespath (~> 1.0)
+    aws-sdk-glacier (1.36.0)
+      aws-sdk-core (~> 3, >= 3.112.0)
+      aws-sigv4 (~> 1.1)
+    aws-sigv4 (1.2.2)
+      aws-eventstream (~> 1, >= 1.0.2)
+    builder (3.2.4)
     concurrent-ruby (1.1.7)
     connection_pool (2.2.3)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    crass (1.0.6)
     diff-lcs (1.4.4)
+    dotenv (2.7.6)
+    erubi (1.10.0)
     et-orbi (1.2.4)
       tzinfo
-    fugit (1.4.0)
+    fugit (1.4.5)
       et-orbi (~> 1.1, >= 1.1.8)
       raabro (~> 1.4)
+    hashdiff (1.0.0)
     i18n (1.8.5)
       concurrent-ruby (~> 1.0)
+    jmespath (1.4.0)
+    loofah (2.9.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    method_source (1.0.0)
+    mini_portile2 (2.5.0)
     minitest (5.14.2)
+    nokogiri (1.11.1)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
+    pg (1.2.3)
+    postgres-copy (1.5.0)
+      activerecord (>= 5.1)
+      pg (>= 0.17)
+      responders
+    public_suffix (4.0.4)
     raabro (1.4.0)
+    racc (1.5.2)
     rack (2.2.3)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    railties (6.1.0.rc1)
+      actionpack (= 6.1.0.rc1)
+      activesupport (= 6.1.0.rc1)
+      method_source
+      rake (>= 0.8.7)
+      thor (~> 1.0)
     rake (13.0.1)
     redis (4.2.2)
+    responders (3.0.1)
+      actionpack (>= 5.0)
+      railties (>= 5.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -50,6 +116,7 @@ GEM
       rspec-core (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
     rspec-support (3.9.3)
+    safe_yaml (1.0.5)
     sidekiq (6.1.2)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
@@ -57,9 +124,15 @@ GEM
     sidekiq-cron (1.2.0)
       fugit (~> 1.1)
       sidekiq (>= 4.2.1)
-    sqlite3 (1.4.2)
+    thor (1.1.0)
+    timecop (0.9.2)
     tzinfo (2.0.2)
       concurrent-ruby (~> 1.0)
+    vcr (6.0.0)
+    webmock (3.7.6)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
     zeitwerk (2.4.1)
 
 PLATFORMS
@@ -67,11 +140,17 @@ PLATFORMS
 
 DEPENDENCIES
   activerecord (~> 6)
+  aws-sdk-glacier
+  dotenv
+  pg
+  postgres-copy
   rake (~> 13.0)
   rspec (~> 3.0)
   rspec-sidekiq
-  sqlite3
   tartarus-rb!
+  timecop
+  vcr
+  webmock
 
 BUNDLED WITH
    2.1.4

data/README.md

@@ -56,12 +56,23 @@ if File.exist?(schedule_file) && Sidekiq.server?
     item.timestamp_field = :created_at
   end
 
+  glacier_configuration = Tartarus::RemoteStorage::Glacier::Configuration.build(
+    aws_key: ENV.fetch("AWS_KEY"),
+    aws_secret: ENV.fetch("AWS_SECRET"),
+    aws_region: ENV.fetch("AWS_REGION"),
+    vault_name: ENV.fetch("GLACIER_VAULT_NAME"),
+    root_path: Rails.root.to_s,
+    archive_registry_factory: ArchiveRegistry,
+  )
+  # don't forget about installing `aws-sdk-glacier` gem
+
   tartarus.register do |item|
     item.model = YetAnotherModel
     item.cron = "5 6 * * *"
     item.queue = "default"
     item.timestamp_field = :created_at
     item.archive_items_older_than = -> { 1.week.ago }
+    item.remote_storage = Tartarus::RemoteStorage::Glacier.new(glacier_configuration)
   end
 
   tartarus.schedule #  this method must be called to create jobs for sidekiq-cron!
@@ -71,6 +82,31 @@ end
 
 You can use the following config params:
 - `model` - a name of the ActiveReord model you want to archive, required
+- `name` - name of your strategy, optional. It fallbacks `model.to_s`. It's important to set in in cases when you have several strategies for the same model:
+```rb
+  tartarus.register do |item|
+    item.model = InternalEvent
+    item.name = "archive_account_and_user_internal_events"
+    item.cron = "5 5 * * *"
+    item.queue = "default"
+    item.tenants_range = -> { ["Account", "User"] }
+    item.tenant_id_field = :model_type
+    item.archive_items_older_than = -> { 30.days.ago }
+    item.timestamp_field = :created_at
+  end
+
+  tartarus.register do |item|
+    item.model = InternalEvent
+    item.name = "archive_post_and_comment_internal_events"
+    item.cron = "5 15 * * *"
+    item.queue = "default"
+    item.tenants_range = -> { ["Post", "Comment"] }
+    item.tenant_id_field = :model_type
+    item.archive_items_older_than = -> { 10.days.ago }
+    item.timestamp_field = :created_at
+  end
+```
+
 - `cron` - cron syntax, required
 - `queue` - name of the sidekiq queue you want to use for execution of the jobs, required
 - `tenants_range` - optional, use if you want to scope items by a tenant (or any field that can be used for partitioning). It doesn't have to be ActiveRecord collection, could be just an array. Must be a proc/lambda/object responding to `call` method. For ActvieRecord collection, `find_each` loop will be used for optimization.
@@ -78,7 +114,92 @@ You can use the following config params:
 - `tenant_id_field` - required when using tenant_value_source/tenant_value_source. It's a DB column that will be used for scoping records by a tenant. For example, here it would be: `ModelThatYouWantToArchive.where(account_uuid: value_of_uuid_from_some_active_account)`
 - `archive_items_older_than` - required, for defining retention policy
 - `timestamp_field` - required, used for performing a query using the value from `archive_items_older_than`
-- `archive_with` - optional (defaults to `delete_all`). Could be `delete_all`, `destroy_all`, `delete_all_without_batches`, `destroy_all_without_batches`
+- `archive_with` - optional (defaults to `delete_all`). Could be `delete_all`, `destroy_all`, `delete_all_without_batches`, `destroy_all_without_batches`, `delete_all_using_limit_in_batches`
+- `batch_size` - optional (defaults to `10_000`, used with `delete_all_using_limit_in_batches` strategy)
+- `remote_storage` - optional (defaults to `Tartarus::RemoteStorage::Null` which does nothing). Use this option if you want store the data somewhere before deleting it.
+
+### Remote Storage
+
+Currently, only `Glacier` (for AWS Glacier) is supported. Also, it works only with Postgres database and requires [postgres-copy](https://github.com/diogob/postgres-copy).
+
+To take advantage of this feature you will need a couple of things:
+1. Apply `acts_as_copy_target` to the archivable model (from `postgres-copy` gem).
+2. Create a model that will be used as a registry for all uploads that happened.
+3. Install `aws-sdk-glacier` gem.
+
+If you want to make `Version` model archivable and use `ArchiveRegistry` as the registry, you will need the following models and tables:
+
+``` rb
+database.create_table(:archive_registries) do |t|
+  t.string :glacier_location, null: false
+  t.string :glacier_checksum, null: false
+  t.string :glacier_archive_id, null: false
+  t.string :archivable_model, null: false
+  t.string :tenant_id_field
+  t.string :tenant_id
+  t.datetime :completed_at, null: false
+end
+
+database.create_table(:versions) do |t|
+end
+
+class Version < ApplicationRecord
+  acts_as_copy_target
+end
+
+class ArchiveRegistry < ApplicationRecord
+end
+```
+
+You can use the above schema for the registry model as it contains all needed fields.
+
+To initialize the service:
+
+``` rb
+glacier_configuration = Tartarus::RemoteStorage::Glacier::Configuration.build(
+  aws_key: ENV.fetch("AWS_KEY"),
+  aws_secret: ENV.fetch("AWS_SECRET"),
+  aws_region: ENV.fetch("AWS_REGION"),
+  vault_name: ENV.fetch("GLACIER_VAULT_NAME"),
+  root_path: Rails.root.to_s,
+  archive_registry_factory: ArchiveRegistry,
+)
+Tartarus::RemoteStorage::Glacier.new(glacier_configuration)
+```
+
+You can also pass `account_id` (by default "-" string will be used):
+
+``` rb
+glacier_configuration = Tartarus::RemoteStorage::Glacier::Configuration.build(
+  aws_key: ENV.fetch("AWS_KEY"),
+  aws_secret: ENV.fetch("AWS_SECRET"),
+  aws_region: ENV.fetch("AWS_REGION"),
+  vault_name: ENV.fetch("GLACIER_VAULT_NAME"),
+  root_path: Rails.root.to_s,
+  archive_registry_factory: ArchiveRegistry,
+  account_id: "some_account_id"
+)
+Tartarus::RemoteStorage::Glacier.new(glacier_configuration)
+```
+
+**Important** - do not use Glacier Storage for large batches (> 4 GB) as multipart uploads are not supported yet.
+
+
+If you know what you are doing, you can add your own storage, as long as it complies with the following interface:
+
+``` rb
+class Glacier
+  attr_reader :configuration
+  private     :configuration
+
+  def initialize(configuration)
+    @configuration = configuration
+  end
+
+  def store(collection, archivable_model, tenant_id: nil, tenant_id_field: nil)
+  end
+end
+```
 
 ### Testing before actually using it
 
@@ -87,7 +208,6 @@ You might want to verify that the gem works in the way you expect it to work. Fo
 1. scheduling/enqueueing: use `Tartarus::ScheduleArchivingModel#schedule` - for example, `Tartarus::ScheduleArchivingModel.new.schedule("PaperTrailVersion")`, it's going to enqueue either `Tartarus::Sidekiq::ArchiveModelWithTenantJob` or `Tartarus::Sidekiq::ArchiveModelWithoutTenantJob`, depending on the config.
 2. execution of the archiving logic: use `Tartarus::ArchiveModelWithTenant#archive` (for example, `Tartarus::ArchiveModelWithTenant.new.archive("PaperTrailVersion", "User")`) or `Tartarus::ArchiveModelWithoutTenant#archive` (for example, `Tartarus::ArchiveModelWithoutTenant.new.archive("PaperTrailVersion")`)
 
-
 You might also want to check `spec/integration` to get an idea how the integration tests were written.
 
 ## Development
@@ -96,10 +216,6 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
-## TODO
-
-- add support for uploading archives to AWS Glacier before deleting items
-
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/tartarus-rb.

data/lib/tartarus.rb

@@ -4,6 +4,7 @@ require "tartarus/archivable_item/sidekiq_cron_job_serializer"
 require "tartarus/archive_strategy"
 require "tartarus/archive_strategy/delete_all"
 require "tartarus/archive_strategy/delete_all_without_batches"
+require "tartarus/archive_strategy/delete_all_using_limit_in_batches"
 require "tartarus/archive_strategy/destroy_all"
 require "tartarus/archive_strategy/destroy_all_without_batches"
 require "tartarus/archive_strategy/extract_batch"
@@ -17,6 +18,7 @@ require "tartarus/rb/version"
 require "tartarus/registry"
 require "tartarus/repository"
 require "tartarus/schedule_archiving_model"
+require "tartarus/remote_storage"
 require "sidekiq/cron/job"
 require "sidekiq"
 

data/lib/tartarus/archivable_collection_repository.rb

@@ -13,6 +13,7 @@ class Tartarus
       ensure_column_exists(collection, model_name, tenant_id_field)
 
       collection.where("#{timestamp_field} < ?", timestamp).where(tenant_id_field => tenant_id)
+                .order(tenant_id_field, timestamp_field)
     end
 
     def items_older_than(model_name, timestamp_field, timestamp)

data/lib/tartarus/archivable_item.rb

@@ -1,7 +1,7 @@
 class Tartarus::ArchivableItem
   REQUIRED_ATTRIBUTES_NAMES = %i(model cron queue archive_items_older_than timestamp_field active_job
     archive_with tenant_value_source).freeze
-  OPTIONAL_ATTRIBUTES_NAMES = %i(tenants_range tenant_id_field).freeze
+  OPTIONAL_ATTRIBUTES_NAMES = %i(tenants_range tenant_id_field batch_size remote_storage name).freeze
 
   attr_accessor *(REQUIRED_ATTRIBUTES_NAMES + OPTIONAL_ATTRIBUTES_NAMES)
 
@@ -45,6 +45,16 @@ class Tartarus::ArchivableItem
     @archive_with ||= :delete_all
   end
 
+  def batch_size
+    return @batch_size if defined?(@batch_size)
+
+    @batch_size ||= 10_000
+  end
+
+  def name
+    @name || @model.to_s
+  end
+
   def validate!
     validate_presence
   end
@@ -54,11 +64,11 @@ class Tartarus::ArchivableItem
   end
 
   def archive_strategy(factory: Tartarus::ArchiveStrategy.new)
-    factory.for(archive_with)
+    factory.for(archive_with, batch_size: batch_size)
   end
 
-  def for_model?(provided_model_name)
-    model.to_s == provided_model_name.to_s
+  def remote_storage
+    @remote_storage || Tartarus::RemoteStorage::Null
   end
 
   private

data/lib/tartarus/archivable_item/sidekiq_cron_job_serializer.rb

@@ -7,7 +7,7 @@ class Tartarus
           description: description_for_item(archivable_item),
           cron: archivable_item.cron,
           class: Tartarus::Sidekiq::ScheduleArchivingModelJob,
-          args: [archivable_item.model],
+          args: [archivable_item.name],
           queue: archivable_item.queue,
           active_job: archivable_item.active_job
         }
@@ -16,7 +16,7 @@ class Tartarus
       private
 
       def name_for_item(archivable_item)
-        "TARTARUS_#{archivable_item.model}"
+        "TARTARUS_#{archivable_item.name}"
       end
 
       def description_for_item(archivable_item)

data/lib/tartarus/archive_model_with_tenant.rb

@@ -7,18 +7,20 @@ class Tartarus::ArchiveModelWithTenant
     @repository = repository
   end
 
-  def archive(model_name, tenant_id)
-    archivable_item = registry.find_by_model(model_name)
-
-    archivable_item.archive_strategy.call(collection_to_archive(model_name, archivable_item, tenant_id))
+  def archive(archivable_item_name, tenant_id)
+    archivable_item = registry.find_by_name(archivable_item_name)
+    collection = collection_to_archive(archivable_item, tenant_id)
+    archivable_item.remote_storage.store(collection, archivable_item.name, tenant_id: tenant_id,
+      tenant_id_field: archivable_item.tenant_id_field)
+    archivable_item.archive_strategy.call(collection)
   end
 
   private
 
-  def collection_to_archive(model_name, archivable_item, tenant_id)
+  def collection_to_archive(archivable_item, tenant_id)
     repository
       .items_older_than_for_tenant(
-        model_name,
+        archivable_item.model,
         archivable_item.timestamp_field, archivable_item.archive_items_older_than.call,
         archivable_item.tenant_id_field, tenant_id
       )

data/lib/tartarus/archive_model_without_tenant.rb

@@ -7,18 +7,19 @@ class Tartarus::ArchiveModelWithoutTenant
     @repository = repository
   end
 
-  def archive(model_name)
-    archivable_item = registry.find_by_model(model_name)
-
-    archivable_item.archive_strategy.call(collection_to_archive(model_name, archivable_item))
+  def archive(archivable_item_name)
+    archivable_item = registry.find_by_name(archivable_item_name)
+    collection = collection_to_archive(archivable_item)
+    archivable_item.remote_storage.store(collection, archivable_item.name)
+    archivable_item.archive_strategy.call(collection)
   end
 
   private
 
-  def collection_to_archive(model_name, archivable_item)
+  def collection_to_archive(archivable_item)
     repository
       .items_older_than(
-        model_name,
+        archivable_item.model,
         archivable_item.timestamp_field, archivable_item.archive_items_older_than.call
       )
   end

data/lib/tartarus/archive_strategy.rb

@@ -1,5 +1,5 @@
 class Tartarus::ArchiveStrategy
-  def for(strategy_name)
+  def for(strategy_name, batch_size: 0)
     case strategy_name.to_sym
     when :delete_all
       Tartarus::ArchiveStrategy::DeleteAll.new
@@ -9,6 +9,8 @@ class Tartarus::ArchiveStrategy
       Tartarus::ArchiveStrategy::DeleteAllWithoutBatches.new
     when :destroy_all_without_batches
       Tartarus::ArchiveStrategy::DestroyAllWithoutBatches.new
+    when :delete_all_using_limit_in_batches
+      Tartarus::ArchiveStrategy::DeleteAllUsingLimitInBatches.new(batch_size: batch_size)
     else
       raise "unknown strategy: #{strategy_name}"
     end

data/lib/tartarus/archive_strategy/delete_all_using_limit_in_batches.rb

@@ -0,0 +1,20 @@
+class Tartarus
+  class ArchiveStrategy
+    class DeleteAllUsingLimitInBatches
+      attr_reader :batch_size
+      private     :batch_size
+
+      def initialize(batch_size:)
+        @batch_size = batch_size
+      end
+
+      def call(collection)
+        num = 1
+
+        while num > 0
+          num = collection.limit(batch_size).delete_all
+        end
+      end
+    end
+  end
+end

data/lib/tartarus/rb/version.rb

@@ -1,5 +1,5 @@
 class Tartarus
   module Rb
-    VERSION = "0.2.0"
+    VERSION = "0.5.0"
   end
 end

data/lib/tartarus/registry.rb

@@ -16,8 +16,8 @@ class Tartarus::Registry
     @storage << item
   end
 
-  def find_by_model(model)
-    storage.find(->{ raise "#{model} not found in registry" }) { |item| item.for_model?(model) }
+  def find_by_name(name)
+    storage.find(->{ raise "#{name} not found in registry" }) { |item| item.name == name }
   end
 
   def reset

data/lib/tartarus/remote_storage.rb

@@ -0,0 +1,4 @@
+module Tartarus::RemoteStorage
+  autoload :Glacier, "tartarus/remote_storage/glacier"
+  autoload :Null, "tartarus/remote_storage/null"
+end

data/lib/tartarus/remote_storage/glacier.rb

@@ -0,0 +1,67 @@
+require "aws-sdk-glacier"
+require "tartarus/remote_storage/glacier/client"
+require "tartarus/remote_storage/glacier/file"
+require "tartarus/remote_storage/glacier/csv_export"
+require "tartarus/remote_storage/glacier/register_upload"
+require "tartarus/remote_storage/glacier/configuration"
+
+class Tartarus
+  module RemoteStorage
+    class Glacier
+      attr_reader :configuration, :clock
+      private     :configuration, :clock
+
+      def initialize(configuration, clock: Time)
+        @configuration = configuration
+        @clock = clock
+      end
+
+      def store(collection, archivable_model, tenant_id: nil, tenant_id_field: nil)
+        path_to_file = path_to_file_for(archivable_model, tenant_id_field, tenant_id)
+        export_to_csv(collection, path_to_file)
+        glacier_file = Tartarus::RemoteStorage::Glacier::File.new(::File.new(path_to_file))
+        glacier_response = upload(glacier_file)
+        register_upload(glacier_response, archivable_model, tenant_id_field, tenant_id)
+      ensure
+        glacier_file.delete_from_local_storage if glacier_file
+      end
+
+      private
+
+      def upload(file)
+        client.upload_archive(configuration.vault_name, file)
+      end
+
+      def client
+        @client ||= begin
+          Tartarus::RemoteStorage::Glacier::Client.new(
+            key: configuration.aws_key,
+            secret: configuration.aws_secret,
+            region: configuration.aws_region,
+            account_id: configuration.account_id,
+          )
+        end
+      end
+
+      def export_to_csv(collection, path_to_file)
+        Tartarus::RemoteStorage::Glacier::CsvExport
+          .new(configuration.storage_directory)
+          .export(collection, path_to_file)
+      end
+
+
+      def register_upload(glacier_response, archivable_model, tenant_id_field, tenant_id)
+        Tartarus::RemoteStorage::Glacier::RegisterUpload.new(configuration.archive_registry_factory).register(
+          glacier_response,
+          archivable_model,
+          tenant_id_field,
+          tenant_id
+        )
+      end
+
+      def path_to_file_for(archivable_model, tenant_id_field, tenant_id)
+        "#{configuration.storage_directory}/#{archivable_model}_#{tenant_id_field}_#{tenant_id}_#{clock.now.to_i}.csv"
+      end
+    end
+  end
+end

data/lib/tartarus/remote_storage/glacier/client.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class Tartarus
+  module RemoteStorage
+    class Glacier
+      class Client
+        attr_reader :client, :account_id
+        private     :client, :account_id
+
+        def initialize(key:, secret:, region:, account_id:)
+          @client = Aws::Glacier::Client.new(credentials: Aws::Credentials.new(key, secret), region: region)
+          @account_id = account_id
+        end
+
+        def upload_archive(vault_name, file)
+          client.upload_archive(
+            account_id: account_id,
+            archive_description: file.description,
+            body: file.body,
+            vault_name: vault_name
+          )
+        end
+      end
+    end
+  end
+end

data/lib/tartarus/remote_storage/glacier/configuration.rb

@@ -0,0 +1,43 @@
+class Tartarus
+  module RemoteStorage
+    class Glacier
+      class Configuration
+        DEFAULT_ACCOUNT_ID = "-"
+        private_constant :DEFAULT_ACCOUNT_ID
+
+        REQUIRED_ATTRIBUTES_NAMES = %i(aws_key aws_secret aws_region account_id vault_name root_path
+          archive_registry_factory).freeze
+        attr_accessor *REQUIRED_ATTRIBUTES_NAMES
+
+        def self.build(aws_key:, aws_secret:, aws_region:, account_id: DEFAULT_ACCOUNT_ID, vault_name:, root_path:, archive_registry_factory:)
+          new.tap do |config|
+            config.aws_key = aws_key
+            config.aws_secret = aws_secret
+            config.aws_region = aws_region
+            config.account_id = account_id
+            config.vault_name = vault_name
+            config.root_path = root_path
+            config.archive_registry_factory = archive_registry_factory
+            config.validate!
+          end
+        end
+
+        def validate!
+          validate_presence
+        end
+
+        def storage_directory
+          "#{root_path}/tmp/tartarus/#{archive_registry_factory}"
+        end
+
+        private
+
+        def validate_presence
+          REQUIRED_ATTRIBUTES_NAMES.each do |attribute|
+            raise ":#{attribute} must be present" if public_send(attribute).nil?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tartarus/remote_storage/glacier/csv_export.rb

@@ -0,0 +1,38 @@
+class Tartarus
+  module RemoteStorage
+    class Glacier
+      class CsvExport
+        FILE_MODE = "w"
+        DELIMITER = ";"
+        NO_PATH_FOR_EXPORT = nil
+        ENCODING = "UTF-8"
+        private_constant :FILE_MODE, :DELIMITER, :NO_PATH_FOR_EXPORT, :ENCODING
+
+        attr_reader :storage_directory, :file_service, :file_utils
+        private     :storage_directory, :file_service, :file_utils
+
+        def initialize(storage_directory, file_service: ::File, file_utils: FileUtils)
+          @storage_directory = storage_directory
+          @file_service = file_service
+          @file_utils = file_utils
+        end
+
+        def export(collection, path_to_file)
+          with_csv_export_file(path_to_file) do |file|
+            collection.copy_to(NO_PATH_FOR_EXPORT, delimiter: DELIMITER) do |line|
+              file.write(line.force_encoding(ENCODING))
+            end
+          end
+        end
+
+        private
+
+        def with_csv_export_file(path_to_file, &block)
+          file_utils.mkdir_p(storage_directory) if !file_service.exist?(storage_directory)
+
+          file_service.open(path_to_file, FILE_MODE, &block)
+        end
+      end
+    end
+  end
+end

data/lib/tartarus/remote_storage/glacier/file.rb

@@ -0,0 +1,31 @@
+require "delegate"
+
+class Tartarus
+  module RemoteStorage
+    class Glacier
+      class File < SimpleDelegator
+        def description
+          file_service.basename(self, ".*")
+        end
+
+        def body
+          self
+        end
+
+        def checksum
+          Digest::SHA256.file(path)
+        end
+
+        def delete_from_local_storage
+          file_service.delete(path)
+        end
+
+        private
+
+        def file_service
+          ::File
+        end
+      end
+    end
+  end
+end

data/lib/tartarus/remote_storage/glacier/register_upload.rb

@@ -0,0 +1,28 @@
+class Tartarus
+  module RemoteStorage
+    class Glacier
+      class RegisterUpload
+        attr_reader :archive_registry_factory, :clock
+        private     :archive_registry_factory, :clock
+
+        def initialize(archive_registry_factory, clock: Time)
+          @archive_registry_factory = archive_registry_factory
+          @clock = clock
+        end
+
+        def register(glacier_response, archivable_model, tenant_id_field, tenant_id)
+          archive_registry_factory.new.tap do |archive_registry|
+            archive_registry.glacier_location = glacier_response.location
+            archive_registry.glacier_checksum = glacier_response.checksum
+            archive_registry.glacier_archive_id = glacier_response.archive_id
+            archive_registry.archivable_model = archivable_model
+            archive_registry.tenant_id_field = tenant_id_field
+            archive_registry.tenant_id = tenant_id
+            archive_registry.completed_at = clock.now
+            archive_registry.save!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tartarus/remote_storage/null.rb

@@ -0,0 +1,8 @@
+class Tartarus
+  module RemoteStorage
+    class Null
+      def self.store(*)
+      end
+    end
+  end
+end

data/lib/tartarus/schedule_archiving_model.rb

@@ -6,15 +6,15 @@ class Tartarus::ScheduleArchivingModel
     @registry = registry
   end
 
-  def schedule(model_name)
-    archivable_item = registry.find_by_model(model_name)
+  def schedule(archivable_item_name)
+    archivable_item = registry.find_by_name(archivable_item_name)
 
     if archivable_item.scope_by_tenant?
       each_tenant(archivable_item) do |tenant|
-        enqueue(Tartarus::Sidekiq::ArchiveModelWithTenantJob, archivable_item.queue, model_name, tenant)
+        enqueue(Tartarus::Sidekiq::ArchiveModelWithTenantJob, archivable_item.queue, archivable_item.name, tenant)
       end
     else
-      enqueue(Tartarus::Sidekiq::ArchiveModelWithoutTenantJob, archivable_item.queue, model_name)
+      enqueue(Tartarus::Sidekiq::ArchiveModelWithoutTenantJob, archivable_item.queue, archivable_item.name)
     end
   end
 

data/lib/tartarus/sidekiq/archive_model_with_tenant_job.rb

@@ -4,8 +4,8 @@ class Tartarus
   class Sidekiq::ArchiveModelWithTenantJob
     include ::Sidekiq::Worker
 
-    def perform(model_name, tenant_id)
-      Tartarus::ArchiveModelWithTenant.new.archive(model_name, tenant_id)
+    def perform(archivable_item_name, tenant_id)
+      Tartarus::ArchiveModelWithTenant.new.archive(archivable_item_name, tenant_id)
     end
   end
 end

data/lib/tartarus/sidekiq/archive_model_without_tenant_job.rb

@@ -4,8 +4,8 @@ class Tartarus
   class Sidekiq::ArchiveModelWithoutTenantJob
     include ::Sidekiq::Worker
 
-    def perform(model_name)
-      Tartarus::ArchiveModelWithoutTenant.new.archive(model_name)
+    def perform(archivable_item_name)
+      Tartarus::ArchiveModelWithoutTenant.new.archive(archivable_item_name)
     end
   end
 end

data/lib/tartarus/sidekiq/schedule_archiving_model_job.rb

@@ -4,8 +4,8 @@ class Tartarus
   class Sidekiq::ScheduleArchivingModelJob
     include ::Sidekiq::Worker
 
-    def perform(model_name)
-      Tartarus::ScheduleArchivingModel.new.schedule(model_name)
+    def perform(archivable_item_name)
+      Tartarus::ScheduleArchivingModel.new.schedule(archivable_item_name)
     end
   end
 end

data/tartarus-rb.gemspec

@@ -30,10 +30,17 @@ Gem::Specification.new do |spec|
   spec.add_dependency "sidekiq", ">= 5"
   spec.add_dependency "sidekiq-cron", "~> 1"
 
+
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-sidekiq"
+  spec.add_development_dependency "aws-sdk-glacier"
 
   spec.add_development_dependency "activerecord", "~> 6"
-  spec.add_development_dependency "sqlite3"
+  spec.add_development_dependency "pg"
+  spec.add_development_dependency "vcr"
+  spec.add_development_dependency "webmock"
+  spec.add_development_dependency "dotenv"
+  spec.add_development_dependency "postgres-copy"
+  spec.add_development_dependency "timecop"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tartarus-rb
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-04 00:00:00.000000000 Z
+date: 2021-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-glacier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -95,7 +109,77 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '6'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: postgres-copy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -116,6 +200,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".env.sample"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -136,6 +221,7 @@ files:
 - lib/tartarus/archive_model_without_tenant.rb
 - lib/tartarus/archive_strategy.rb
 - lib/tartarus/archive_strategy/delete_all.rb
+- lib/tartarus/archive_strategy/delete_all_using_limit_in_batches.rb
 - lib/tartarus/archive_strategy/delete_all_without_batches.rb
 - lib/tartarus/archive_strategy/destroy_all.rb
 - lib/tartarus/archive_strategy/destroy_all_without_batches.rb
@@ -143,6 +229,14 @@ files:
 - lib/tartarus/rb.rb
 - lib/tartarus/rb/version.rb
 - lib/tartarus/registry.rb
+- lib/tartarus/remote_storage.rb
+- lib/tartarus/remote_storage/glacier.rb
+- lib/tartarus/remote_storage/glacier/client.rb
+- lib/tartarus/remote_storage/glacier/configuration.rb
+- lib/tartarus/remote_storage/glacier/csv_export.rb
+- lib/tartarus/remote_storage/glacier/file.rb
+- lib/tartarus/remote_storage/glacier/register_upload.rb
+- lib/tartarus/remote_storage/null.rb
 - lib/tartarus/repository.rb
 - lib/tartarus/schedule_archiving_model.rb
 - lib/tartarus/sidekiq.rb