tartarus-rb 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee9118a9365d2a74dd802218b0b0b36286c6494ecffe4f81cbf9bfed35597057
-  data.tar.gz: f85ffe0e1f182667b5b9b522746a85b45cc9cd3452270f583549a0616d06337c
+  metadata.gz: d964cd9b85733405512f08d58cea33e9f12f5b6d3fa05d8fb2d1c979463ffead
+  data.tar.gz: 955301decdf77128432413afb6dd40612bb5c279e2cb65129f740cf1c81c1373
 SHA512:
-  metadata.gz: '08e1ab969f1681fca948bdfb4ae98fb919e7caef4e95a1cf5a3a2dffc6fa79a59c73655927be759408f951ad0846ff5c1dea6b7396220801f31c179730e9d6e3'
-  data.tar.gz: 72f485499631b46912293650672cd79e8df364ebbc7b18144731c1077e7fa8157be8b20e4078212b04dfb539a3de19af99abcca720163785c3132f51bf574968
+  metadata.gz: d455a12a6503b078c1b49e3902d840a017bfa50f99c6eee2ff17367d3398eb62b376503f84b260a90663e9c3f465ac59ba4509a1b3f9767835548853a24f3469
+  data.tar.gz: 2b4502ce62eb6bd831413d237e9b69535fbbe113c09e25e35f0ee680c902599bd0c52a210c0eb365ccde044c6a200da32c4b12798a6137e996a228f1192d8b10

data/Changelog.md

@@ -2,6 +2,18 @@
 
 ## Master
 
+## 0.6.0
+
+- Optimize query from `Tartarus::ArchivableCollectionRepository#items_older_than` by adding explicit ordering
+
+## 0.5.0
+
+- Provide ability to explicitly set the name of archivable item to have multiple ways of archiving the same model
+
+## 0.4.1
+
+- Do not make Glacier a required dependency if not used
+
 ## 0.4.0
 
 - Add Glacier remote storage support to upload data before deleting it

data/Gemfile.lock

@@ -1,8 +1,7 @@
 PATH
   remote: .
   specs:
-    tartarus-rb (0.4.0)
-      aws-sdk-glacier
+    tartarus-rb (0.6.0)
       sidekiq (>= 5)
       sidekiq-cron (~> 1)
 
@@ -58,7 +57,7 @@ GEM
     erubi (1.10.0)
     et-orbi (1.2.4)
       tzinfo
-    fugit (1.4.2)
+    fugit (1.4.5)
       et-orbi (~> 1.1, >= 1.1.8)
       raabro (~> 1.4)
     hashdiff (1.0.0)
@@ -141,6 +140,7 @@ PLATFORMS
 
 DEPENDENCIES
   activerecord (~> 6)
+  aws-sdk-glacier
   dotenv
   pg
   postgres-copy

data/README.md

@@ -64,6 +64,7 @@ if File.exist?(schedule_file) && Sidekiq.server?
     root_path: Rails.root.to_s,
     archive_registry_factory: ArchiveRegistry,
   )
+  # don't forget about installing `aws-sdk-glacier` gem
 
   tartarus.register do |item|
     item.model = YetAnotherModel
@@ -81,6 +82,31 @@ end
 
 You can use the following config params:
 - `model` - a name of the ActiveReord model you want to archive, required
+- `name` - name of your strategy, optional. It fallbacks `model.to_s`. It's important to set in in cases when you have several strategies for the same model:
+```rb
+  tartarus.register do |item|
+    item.model = InternalEvent
+    item.name = "archive_account_and_user_internal_events"
+    item.cron = "5 5 * * *"
+    item.queue = "default"
+    item.tenants_range = -> { ["Account", "User"] }
+    item.tenant_id_field = :model_type
+    item.archive_items_older_than = -> { 30.days.ago }
+    item.timestamp_field = :created_at
+  end
+
+  tartarus.register do |item|
+    item.model = InternalEvent
+    item.name = "archive_post_and_comment_internal_events"
+    item.cron = "5 15 * * *"
+    item.queue = "default"
+    item.tenants_range = -> { ["Post", "Comment"] }
+    item.tenant_id_field = :model_type
+    item.archive_items_older_than = -> { 10.days.ago }
+    item.timestamp_field = :created_at
+  end
+```
+
 - `cron` - cron syntax, required
 - `queue` - name of the sidekiq queue you want to use for execution of the jobs, required
 - `tenants_range` - optional, use if you want to scope items by a tenant (or any field that can be used for partitioning). It doesn't have to be ActiveRecord collection, could be just an array. Must be a proc/lambda/object responding to `call` method. For ActvieRecord collection, `find_each` loop will be used for optimization.
@@ -90,7 +116,7 @@ You can use the following config params:
 - `timestamp_field` - required, used for performing a query using the value from `archive_items_older_than`
 - `archive_with` - optional (defaults to `delete_all`). Could be `delete_all`, `destroy_all`, `delete_all_without_batches`, `destroy_all_without_batches`, `delete_all_using_limit_in_batches`
 - `batch_size` - optional (defaults to `10_000`, used with `delete_all_using_limit_in_batches` strategy)
-- `remote_storage` - optional (defaults to `Tartarus::RemoteStorage::Null` which does nothing). Use this option if you want store the data somewhere before deleting it. 
+- `remote_storage` - optional (defaults to `Tartarus::RemoteStorage::Null` which does nothing). Use this option if you want store the data somewhere before deleting it.
 
 ### Remote Storage
 
@@ -99,6 +125,7 @@ Currently, only `Glacier` (for AWS Glacier) is supported. Also, it works only wi
 To take advantage of this feature you will need a couple of things:
 1. Apply `acts_as_copy_target` to the archivable model (from `postgres-copy` gem).
 2. Create a model that will be used as a registry for all uploads that happened.
+3. Install `aws-sdk-glacier` gem.
 
 If you want to make `Version` model archivable and use `ArchiveRegistry` as the registry, you will need the following models and tables:
 
@@ -155,6 +182,9 @@ glacier_configuration = Tartarus::RemoteStorage::Glacier::Configuration.build(
 Tartarus::RemoteStorage::Glacier.new(glacier_configuration)
 ```
 
+**Important** - do not use Glacier Storage for large batches (> 4 GB) as multipart uploads are not supported yet.
+
+
 If you know what you are doing, you can add your own storage, as long as it complies with the following interface:
 
 ``` rb
@@ -166,7 +196,7 @@ class Glacier
     @configuration = configuration
   end
 
-  def store(collection, archivable_model, tenant_id: nil, tenant_id_field: nil) 
+  def store(collection, archivable_model, tenant_id: nil, tenant_id_field: nil)
   end
 end
 ```

data/lib/tartarus/archivable_collection_repository.rb

@@ -20,7 +20,7 @@ class Tartarus
       collection = collection_for(model_name)
       ensure_column_exists(collection, model_name, timestamp_field)
 
-      collection.where("#{timestamp_field} < ?", timestamp)
+      collection.where("#{timestamp_field} < ?", timestamp).order(timestamp_field)
     end
 
     private

data/lib/tartarus/archivable_item/sidekiq_cron_job_serializer.rb

@@ -7,7 +7,7 @@ class Tartarus
           description: description_for_item(archivable_item),
           cron: archivable_item.cron,
           class: Tartarus::Sidekiq::ScheduleArchivingModelJob,
-          args: [archivable_item.model],
+          args: [archivable_item.name],
           queue: archivable_item.queue,
           active_job: archivable_item.active_job
         }
@@ -16,7 +16,7 @@ class Tartarus
       private
 
       def name_for_item(archivable_item)
-        "TARTARUS_#{archivable_item.model}"
+        "TARTARUS_#{archivable_item.name}"
       end
 
       def description_for_item(archivable_item)

data/lib/tartarus/archivable_item.rb

@@ -1,7 +1,7 @@
 class Tartarus::ArchivableItem
   REQUIRED_ATTRIBUTES_NAMES = %i(model cron queue archive_items_older_than timestamp_field active_job
     archive_with tenant_value_source).freeze
-  OPTIONAL_ATTRIBUTES_NAMES = %i(tenants_range tenant_id_field batch_size remote_storage).freeze
+  OPTIONAL_ATTRIBUTES_NAMES = %i(tenants_range tenant_id_field batch_size remote_storage name).freeze
 
   attr_accessor *(REQUIRED_ATTRIBUTES_NAMES + OPTIONAL_ATTRIBUTES_NAMES)
 
@@ -51,6 +51,10 @@ class Tartarus::ArchivableItem
     @batch_size ||= 10_000
   end
 
+  def name
+    @name || @model.to_s
+  end
+
   def validate!
     validate_presence
   end
@@ -63,10 +67,6 @@ class Tartarus::ArchivableItem
     factory.for(archive_with, batch_size: batch_size)
   end
 
-  def for_model?(provided_model_name)
-    model.to_s == provided_model_name.to_s
-  end
-
   def remote_storage
     @remote_storage || Tartarus::RemoteStorage::Null
   end

data/lib/tartarus/archive_model_with_tenant.rb

@@ -7,20 +7,20 @@ class Tartarus::ArchiveModelWithTenant
     @repository = repository
   end
 
-  def archive(model_name, tenant_id)
-    archivable_item = registry.find_by_model(model_name)
-    collection = collection_to_archive(model_name, archivable_item, tenant_id)
-    archivable_item.remote_storage.store(collection, model_name, tenant_id: tenant_id,
+  def archive(archivable_item_name, tenant_id)
+    archivable_item = registry.find_by_name(archivable_item_name)
+    collection = collection_to_archive(archivable_item, tenant_id)
+    archivable_item.remote_storage.store(collection, archivable_item.name, tenant_id: tenant_id,
       tenant_id_field: archivable_item.tenant_id_field)
     archivable_item.archive_strategy.call(collection)
   end
 
   private
 
-  def collection_to_archive(model_name, archivable_item, tenant_id)
+  def collection_to_archive(archivable_item, tenant_id)
     repository
       .items_older_than_for_tenant(
-        model_name,
+        archivable_item.model,
         archivable_item.timestamp_field, archivable_item.archive_items_older_than.call,
         archivable_item.tenant_id_field, tenant_id
       )

data/lib/tartarus/archive_model_without_tenant.rb

@@ -7,19 +7,19 @@ class Tartarus::ArchiveModelWithoutTenant
     @repository = repository
   end
 
-  def archive(model_name)
-    archivable_item = registry.find_by_model(model_name)
-    collection = collection_to_archive(model_name, archivable_item)
-    archivable_item.remote_storage.store(collection, model_name)
+  def archive(archivable_item_name)
+    archivable_item = registry.find_by_name(archivable_item_name)
+    collection = collection_to_archive(archivable_item)
+    archivable_item.remote_storage.store(collection, archivable_item.name)
     archivable_item.archive_strategy.call(collection)
   end
 
   private
 
-  def collection_to_archive(model_name, archivable_item)
+  def collection_to_archive(archivable_item)
     repository
       .items_older_than(
-        model_name,
+        archivable_item.model,
         archivable_item.timestamp_field, archivable_item.archive_items_older_than.call
       )
   end

data/lib/tartarus/rb/version.rb

@@ -1,5 +1,5 @@
 class Tartarus
   module Rb
-    VERSION = "0.4.0"
+    VERSION = "0.6.0"
   end
 end

data/lib/tartarus/registry.rb

@@ -16,8 +16,8 @@ class Tartarus::Registry
     @storage << item
   end
 
-  def find_by_model(model)
-    storage.find(->{ raise "#{model} not found in registry" }) { |item| item.for_model?(model) }
+  def find_by_name(name)
+    storage.find(->{ raise "#{name} not found in registry" }) { |item| item.name == name }
   end
 
   def reset

data/lib/tartarus/remote_storage/glacier.rb

@@ -1,3 +1,10 @@
+require "aws-sdk-glacier"
+require "tartarus/remote_storage/glacier/client"
+require "tartarus/remote_storage/glacier/file"
+require "tartarus/remote_storage/glacier/csv_export"
+require "tartarus/remote_storage/glacier/register_upload"
+require "tartarus/remote_storage/glacier/configuration"
+
 class Tartarus
   module RemoteStorage
     class Glacier

data/lib/tartarus/remote_storage.rb

@@ -1,2 +1,4 @@
 module Tartarus::RemoteStorage
+  autoload :Glacier, "tartarus/remote_storage/glacier"
+  autoload :Null, "tartarus/remote_storage/null"
 end

data/lib/tartarus/schedule_archiving_model.rb

@@ -6,15 +6,15 @@ class Tartarus::ScheduleArchivingModel
     @registry = registry
   end
 
-  def schedule(model_name)
-    archivable_item = registry.find_by_model(model_name)
+  def schedule(archivable_item_name)
+    archivable_item = registry.find_by_name(archivable_item_name)
 
     if archivable_item.scope_by_tenant?
       each_tenant(archivable_item) do |tenant|
-        enqueue(Tartarus::Sidekiq::ArchiveModelWithTenantJob, archivable_item.queue, model_name, tenant)
+        enqueue(Tartarus::Sidekiq::ArchiveModelWithTenantJob, archivable_item.queue, archivable_item.name, tenant)
       end
     else
-      enqueue(Tartarus::Sidekiq::ArchiveModelWithoutTenantJob, archivable_item.queue, model_name)
+      enqueue(Tartarus::Sidekiq::ArchiveModelWithoutTenantJob, archivable_item.queue, archivable_item.name)
     end
   end
 

data/lib/tartarus/sidekiq/archive_model_with_tenant_job.rb

@@ -4,8 +4,8 @@ class Tartarus
   class Sidekiq::ArchiveModelWithTenantJob
     include ::Sidekiq::Worker
 
-    def perform(model_name, tenant_id)
-      Tartarus::ArchiveModelWithTenant.new.archive(model_name, tenant_id)
+    def perform(archivable_item_name, tenant_id)
+      Tartarus::ArchiveModelWithTenant.new.archive(archivable_item_name, tenant_id)
     end
   end
 end

data/lib/tartarus/sidekiq/archive_model_without_tenant_job.rb

@@ -4,8 +4,8 @@ class Tartarus
   class Sidekiq::ArchiveModelWithoutTenantJob
     include ::Sidekiq::Worker
 
-    def perform(model_name)
-      Tartarus::ArchiveModelWithoutTenant.new.archive(model_name)
+    def perform(archivable_item_name)
+      Tartarus::ArchiveModelWithoutTenant.new.archive(archivable_item_name)
     end
   end
 end

data/lib/tartarus/sidekiq/schedule_archiving_model_job.rb

@@ -4,8 +4,8 @@ class Tartarus
   class Sidekiq::ScheduleArchivingModelJob
     include ::Sidekiq::Worker
 
-    def perform(model_name)
-      Tartarus::ScheduleArchivingModel.new.schedule(model_name)
+    def perform(archivable_item_name)
+      Tartarus::ScheduleArchivingModel.new.schedule(archivable_item_name)
     end
   end
 end

data/lib/tartarus.rb

@@ -19,15 +19,7 @@ require "tartarus/registry"
 require "tartarus/repository"
 require "tartarus/schedule_archiving_model"
 require "tartarus/remote_storage"
-require "tartarus/remote_storage/null"
-require "tartarus/remote_storage/glacier"
-require "tartarus/remote_storage/glacier/client"
-require "tartarus/remote_storage/glacier/file"
-require "tartarus/remote_storage/glacier/csv_export"
-require "tartarus/remote_storage/glacier/register_upload"
-require "tartarus/remote_storage/glacier/configuration"
 require "sidekiq/cron/job"
-require "aws-sdk-glacier"
 require "sidekiq"
 
 class Tartarus

data/tartarus-rb.gemspec

@@ -29,11 +29,12 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "sidekiq", ">= 5"
   spec.add_dependency "sidekiq-cron", "~> 1"
-  spec.add_dependency "aws-sdk-glacier"
+
 
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-sidekiq"
+  spec.add_development_dependency "aws-sdk-glacier"
 
   spec.add_development_dependency "activerecord", "~> 6"
   spec.add_development_dependency "pg"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tartarus-rb
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Karol Galanciak
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-02-08 00:00:00.000000000 Z
+date: 2022-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -38,20 +38,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
-- !ruby/object:Gem::Dependency
-  name: aws-sdk-glacier
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-glacier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -264,7 +264,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: A gem for archving (deleting) old records you no longer need. Send them straight