activerecord_hoarder 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 476f76a46fae44bba761080f33f0c5073733de0e
+  data.tar.gz: 5040b1ce780046408e8ab065d18452e9ff6803b1
+SHA512:
+  metadata.gz: 195f314b03bf2f5f0e2f82da4a9c409b03ca4896d39e1bb055a8d3090d956b5ad929a69e0c1ab9015f6c5726233318062f364e2c392ccc0b58fcf662f7740370
+  data.tar.gz: ebd4a813bb3de31d7e61f962b4ff1b5098f71857dda2d016782c91227d22725de4c31dc45c99579de60b5e2f7f374cbde7fb2c6556f6fd6e5adf4bdc7abca75d

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+config/*.yml
+*.sqlite3

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.15.4

data/Gemfile

@@ -0,0 +1,18 @@
+source "https://rubygems.org"
+
+repo_name = "gem_batch_archiving"
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+gem 'aws-sdk-s3', '~> 1'
+
+group :test do
+  gem 'factory_girl_rails'
+  gem 'sqlite3'
+  gem 'timecop'
+end
+
+group :development do
+  gem 'sqlite3'
+end
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 ACCESS MARKETING & COMMUNICATION LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+# Activerecord Hoarder
+
+hoard records
+
+## 1 Use
+
+### 1.1 make model a hoarder
+```
+class ExampleModel < ActiveRecord::Base
+  acts_as_hoarder
+end
+```
+
+### 1.2  hoarding records
+from console:
+```
+ExampleModel.hoard
+```
+will create S3 entries with keys: `<bucket_sub_dir>/<table_name = example_models>/<year>/<month>/<year>-<month>-<day>.json` and json formatted content
+
+### 1.3 restoring records
+from console:
+```
+ExampleModel.restore_date(Date.new(<Y>,<m>,<d>))
+```
+
+## 2 Development
+
+### 2.0 initial setup
+
+Make a clone. Make a branch. Install dependencies.
+
+### 2.1 playing around
+
+#### Configure database
+
+Create config file from template (`cp config/dbspec.yml.template config/dbspec.yml`). Change database from `postgresql` to `sqlite3` and database name from `activerecord_hoarder` to `<as_desired>.sqlite3`.
+
+#### Configure archive
+
+Create config file from template (`cp config/activerecord_hoarder.yml.template config/activerecord_hoarder.yml`). Add your S3 credentials `access_key_id` and `secret_access_key` for target bucket `bucket`. Change `region` if necessary. If you want, change `acl` and add `bucket_sub_dir`.
+
+#### Hop into sandbox
+```
+bundler exec bin/console
+```
+
+#### bin/example
+Convenience functionality
+- `require_relative "bin/example/schema"` for creating an example table `examples`
+- `require_relative "bin/example/example"` for an example archivable model `Example`
+- `require_relative "bin/example/fixture"` for a factory method `create_examples(count, start: 0, deleted: true)` for creating examples
+
+### 2.2 testing it
+
+#### Configure test database
+Create config file from template (`cp config/dbspec_rspec.yml.template config/dbspec_rspec.yml`). Modify settings if you want.
+
+#### Run tests
+```
+bundler exec rspec spec
+```

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/activerecord_hoarder.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "activerecord_hoarder/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "activerecord_hoarder"
+  spec.version = ActiverecordHoarder::VERSION
+  spec.authors = ["Matthias Engh"]
+  spec.email = ["matthias@wescrimmage.com"]
+
+  spec.summary = %q{hoards records}
+  spec.description = %q{extend active records with simple archiving mechanics}
+  spec.homepage = "https://github.com/Scrimmage/gem_batch_archiving"
+  spec.license = "MIT"
+
+  spec.files = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activerecord", [">= 4.2", "< 6.0"]
+
+  spec.add_development_dependency "bundler", "~> 1.15"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/bin/console

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+
+# require dependencies from gemspec
+require "active_record"
+require "aws-sdk-s3"
+require "pp"
+require "yaml"
+
+dbspec_location = "config/dbspec.yml"
+
+if File.exist?(dbspec_location)
+  dbspec = YAML.load_file(dbspec_location)
+  ActiveRecord::Base.establish_connection(dbspec)
+  ActiveRecord::Base.connection
+else
+  raise "database configuration file #{dbspec_location} not found"
+end
+
+require "activerecord_hoarder"
+
+storage_config_path = "config/activerecord_hoarder.yml"
+raise "storage configuration file #{storage_config_path}` not found" if !File.exists?(storage_config_path)
+storage_options = YAML.load_file(storage_config_path)
+::ActiverecordHoarder::Storage.configure(storage: :aws_s3, storage_options: storage_options)
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config/activerecord_hoarder.yml.template

@@ -0,0 +1,6 @@
+access_key_id: 
+acl: private
+bucket: 
+bucket_sub_dir: 
+region: us-east-1
+secret_access_key: 

data/config/dbspec.yml.template

@@ -0,0 +1,2 @@
+adapter: postgresql
+database: activerecord_hoarder

data/config/dbspec_rspec.yml.template

@@ -0,0 +1,2 @@
+adapter: sqlite3
+database: activerecord_hoarder_rspec

data/example/example.rb

@@ -0,0 +1,3 @@
+class Example < ActiveRecord::Base
+  acts_as_hoarder
+end

data/example/fixture.rb

@@ -0,0 +1,6 @@
+def create_examples(count, start: 0, deleted: true)
+  (1..count).each do |n|
+    Example.create(content: "example: #{n}", created_at: (n+start).hours.ago, deleted_at: Time.now)
+  end
+  nil
+end

data/example/schema.rb

@@ -0,0 +1,8 @@
+ActiveRecord::Schema.define do
+  create_table "examples", force: true do |t|
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.datetime "deleted_at"
+    t.string "content"
+  end
+end

data/lib/activerecord_hoarder/aws_s3_storage.rb

@@ -0,0 +1,64 @@
+module ActiverecordHoarder
+  class AwsS3
+    DEFAULT_ACL = "private"
+    OPTION_CONTENT_ACCESS = "acl"
+    OPTION_SUB_DIR = "bucket_sub_dir"
+    OPTION_BUCKET = "bucket"
+    OPTION_ACCESS_KEY_ID = "access_key_id"
+    OPTIONS_SECRET_ACCESS_KEY = "secret_access_key"
+    OPTION_REGION = "region"
+
+    attr_reader :storage_options
+
+    def initialize(table_name, storage_options)
+      @storage_options = storage_options
+
+      if storage_options[OPTION_SUB_DIR].blank?
+        @key_prefix = table_name
+      else
+        @key_prefix = File.join(storage_options[OPTION_SUB_DIR], table_name)
+      end
+    end
+
+    def fetch_data(key)
+      full_key = key_with_prefix(key)
+      begin
+        response = s3_client.get_object(bucket: s3_bucket, key: full_key)
+      rescue Aws::S3::Errors::NoSuchKey => e
+        raise ::ActiverecordHoarder::StorageError.new("fetch_data erred with '#{e.class}':'#{e.message}'' trying to access '#{full_key}'' in bucket: '#{s3_bucket}'")
+      end
+      response.body
+    end
+
+    def store_data(batch)
+      full_key = key_with_prefix(batch.key.to_s)
+
+      s3_client.put_object(bucket: s3_bucket, body: batch.content_string, key: full_key, acl: s3_acl)
+      true
+    end
+
+    private
+
+    def key_with_prefix(key)
+      File.join(@key_prefix, key.to_s)
+    end
+
+    def s3_acl
+      storage_options[OPTION_CONTENT_ACCESS] || DEFAULT_ACL
+    end
+
+    def s3_bucket
+      storage_options[OPTION_BUCKET]
+    end
+
+    def s3_client
+      @s3_client ||= begin
+        access_key_id = storage_options[OPTION_ACCESS_KEY_ID] or raise StorageError.new("access_key_id missing")
+        secret_access_key = storage_options[OPTIONS_SECRET_ACCESS_KEY] or raise StorageError.new("secret_access_key missing")
+        credentials = Aws::Credentials.new(access_key_id, secret_access_key)
+        region = storage_options[OPTION_REGION] or raise StorageError.new("region missing")
+        Aws::S3::Client.new(credentials: credentials, region: region)
+      end
+    end
+  end
+end

data/lib/activerecord_hoarder/batch.rb

@@ -0,0 +1,26 @@
+module ::ActiverecordHoarder
+  class Batch
+    RECORD_DATE_FIELD = "created_at"
+
+    def self.from_records(record_data)
+      record_data.present? ? new(record_data) : nil
+    end
+
+    def initialize(record_data)
+      @record_data = record_data
+      @serializer = ::ActiverecordHoarder::Serializer
+    end
+
+    def date
+      @date ||= @record_data.first[RECORD_DATE_FIELD].to_date
+    end
+
+    def key
+      @key ||= ::ActiverecordHoarder::StorageKey.from_date(date, @serializer.extension)
+    end
+
+    def content_string
+      @serializer.create_archive(@record_data)
+    end
+  end
+end

data/lib/activerecord_hoarder/batch_archiver.rb

@@ -0,0 +1,16 @@
+class ::ActiverecordHoarder::BatchArchiver
+  def initialize(model_class, storage = nil)
+    @record_collector = ::ActiverecordHoarder::RecordCollector.new(model_class)
+    @archive_storage = storage || default_storage_for_records(model_class.table_name)
+  end
+
+  def archive_batch
+    @record_collector.in_batches(delete_on_success: true) do |batch|
+      @archive_storage.store_data(batch)
+    end
+  end
+
+  def default_storage_for_records(table_name)
+    ::ActiverecordHoarder::Storage.new(table_name)
+  end
+end

data/lib/activerecord_hoarder/core.rb

@@ -0,0 +1,13 @@
+module ActiverecordHoarder::Core
+  def self.included(base)
+    raise 'created_at accessor needed' if !base.column_names.include?("created_at")
+    raise 'deleted_at accessor needed' if !base.column_names.include?("deleted_at")
+    base.extend ClassMethods
+  end
+
+  module ClassMethods
+    def hoard
+      ::ActiverecordHoarder::BatchArchiver.new(self).archive_batch
+    end
+  end
+end

data/lib/activerecord_hoarder/record_collector.rb

@@ -0,0 +1,67 @@
+class ::ActiverecordHoarder::RecordCollector
+  attr_reader :relative_limit
+
+  def initialize(model_class)
+    @model_class = model_class
+  end
+
+  def in_batches(delete_on_success: false)
+    while collect_batch
+      success = yield @batch
+      return if !success
+      next if !delete_on_success
+      destroy_current_records!
+    end
+  end
+
+  private
+
+  def collect_batch
+    activate_limit if batch_data_cached? && !limit_toggled?
+    if limit_toggled?
+      @batch = ensuring_new_records do
+        retrieve_next_batch
+      end
+    else
+      @batch = retrieve_first_batch
+    end
+    batch_data_cached?
+  end
+
+  def activate_limit
+    @relative_limit = [@batch.date.end_of_week + 1, archive_timeframe_upper_limit].min
+  end
+
+  def archive_timeframe_upper_limit
+    Time.now.getutc.beginning_of_day
+  end
+
+  def batch_data_cached?
+    @batch.present?
+  end
+
+  def destroy_current_records!
+    @model_class.connection.execute(@batch_query.delete(@batch.date))
+  end
+
+  def ensuring_new_records
+    record_batch = yield
+    @batch.date == record_batch.try(:date) ? nil : record_batch
+  end
+
+  def limit_toggled?
+    @relative_limit.present?
+  end
+
+  def retrieve_first_batch
+    @batch_query = ::ActiverecordHoarder::BatchQuery.new(archive_timeframe_upper_limit, @model_class)
+    batch_data = @model_class.connection.exec_query(@batch_query.fetch)
+    ::ActiverecordHoarder::Batch.from_records(batch_data)
+  end
+
+  def retrieve_next_batch
+    @batch_query = ::ActiverecordHoarder::BatchQuery.new(relative_limit, @model_class)
+    batch_data = @model_class.connection.exec_query(@batch_query.fetch)
+    ::ActiverecordHoarder::Batch.from_records(batch_data)
+  end
+end

data/lib/activerecord_hoarder/record_query.rb

@@ -0,0 +1,85 @@
+module ActiverecordHoarder
+  class BatchQuery
+    SUBQUERY_DELETED_RECORDS = <<~SQL.strip_heredoc
+      SELECT
+        %{fields}
+      FROM
+        %{table_name}
+      WHERE
+        deleted_at IS NOT NULL
+    SQL
+
+    SUBQUERY_NON_DELETED_RECORDS = <<~SQL.strip_heredoc
+      SELECT
+        %{fields}
+      FROM
+        %{table_name}
+      WHERE
+        deleted_at IS NULL
+    SQL
+
+    QUERY_TEMPLATE_FOR_DATE_DELETION = <<~SQL.strip_heredoc
+      DELETE FROM %{table_name} WHERE date(created_at) = '%{date}';
+    SQL
+
+    QUERY_TEMPLATE_FOR_RECORD_WITH_LIMIT = <<~SQL.strip_heredoc
+      SELECT
+        %{fields}
+      FROM
+        %{table_name}
+      WHERE
+        date(created_at) = (
+          SELECT
+            min(dates_with_deleted.creation_date)
+          FROM
+            (
+              #{SUBQUERY_DELETED_RECORDS % {
+                  fields: "date(created_at) as creation_date",
+                  table_name: "%{table_name}"
+              }}
+            ) as dates_with_deleted
+            LEFT OUTER JOIN
+            (
+              #{SUBQUERY_NON_DELETED_RECORDS % {
+                fields: "date(created_at) as creation_date",
+                table_name: "%{table_name}"
+              }}
+            ) as dates_with_non_deleted
+            ON
+              dates_with_deleted.creation_date = dates_with_non_deleted.creation_date
+            WHERE
+              dates_with_non_deleted.creation_date IS NULL
+            AND
+              created_at < '%{limit}'
+        )
+      ;
+    SQL
+
+    def initialize(limit, model_class)
+      @limit = limit
+      @model_class = model_class
+    end
+
+    def delete(date)
+      QUERY_TEMPLATE_FOR_DATE_DELETION % {
+        fields: @model_class.column_names.join(", "),
+        date: date,
+        table_name: table_name
+      }
+    end
+
+    def fetch
+      QUERY_TEMPLATE_FOR_RECORD_WITH_LIMIT % {
+        fields: @model_class.column_names.join(", "),
+        limit: @limit,
+        table_name: table_name
+      }
+    end
+
+    private
+
+    def table_name
+      @model_class.table_name
+    end
+  end
+end

data/lib/activerecord_hoarder/restore.rb

@@ -0,0 +1,16 @@
+module ActiverecordHoarder
+  module Restore
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def restore_archive_records(date)
+        storage = ::ActiverecordHoarder::Storage.new(self.table_name)
+        key = ::ActiverecordHoarder::StorageKey.from_date(date, :json)
+        dataIO = storage.fetch_data(key)
+        create(JSON.parse(dataIO.read))
+      end
+    end
+  end
+end

data/lib/activerecord_hoarder/serializer.rb

@@ -0,0 +1,9 @@
+class ::ActiverecordHoarder::Serializer
+  def self.create_archive(batch_data)
+    batch_data.to_json
+  end
+
+  def self.extension
+    :json
+  end
+end

data/lib/activerecord_hoarder/storage.rb

@@ -0,0 +1,27 @@
+class ::ActiverecordHoarder::Storage
+  class_attribute :storage, :storage_options
+
+  def self.configure(storage:, storage_options:)
+    ::ActiverecordHoarder::Storages.is_valid_storage?(storage)
+
+    self.storage_options = storage_options
+    self.storage = storage
+    self
+  end
+
+  private
+
+  def self.new(table_name, storage_override: nil, storage_options_override: {})
+    self.check_configured
+    storage_class = ::ActiverecordHoarder::Storages.retrieve(storage_override || storage)
+    storage_class.new(table_name, storage_options.merge(storage_options_override))
+  end
+
+  def self.check_configured
+    raise ::ActiverecordHoarder::StorageError.new("storage needs to be configured") unless is_configured?
+  end
+
+  def self.is_configured?
+    storage.present? && storage_options.is_a?(Hash)
+  end
+end

data/lib/activerecord_hoarder/storage_error.rb

@@ -0,0 +1,4 @@
+module ::ActiverecordHoarder
+  class StorageError < StandardError
+  end
+end

data/lib/activerecord_hoarder/storage_key.rb

@@ -0,0 +1,18 @@
+module ActiverecordHoarder
+  class StorageKey
+    def self.from_date(date, file_extension = nil)
+      key_parts = [date.year.to_s, date.month.to_s, date.iso8601]
+      new(key_parts, file_extension)
+    end
+
+    def initialize(key_parts, file_extension)
+      @key_parts = key_parts
+      @file_extension = file_extension
+    end
+
+    def to_s
+      key_without_extension = File.join(@key_parts)
+      @file_extension.present? ? key_without_extension + '.' + @file_extension.to_s : key_without_extension
+    end
+  end
+end

data/lib/activerecord_hoarder/storages.rb

@@ -0,0 +1,20 @@
+module ::ActiverecordHoarder
+  class Storages
+    STORAGE_DICT = {
+      aws_s3: ::ActiverecordHoarder::AwsS3
+    }
+
+    def self.check_storage(storage_key)
+      raise ::ActiverecordHoarder::StorageError.new("unknown storage (#{storage_key}), known keys are #{STORAGE_DICT.keys}") if !is_valid_storage?(storage_key)
+    end
+
+    def self.is_valid_storage?(storage_key)
+      STORAGE_DICT.keys.include?(storage_key)
+    end
+
+    def self.retrieve(storage_key)
+      check_storage(storage_key)
+      STORAGE_DICT[storage_key]
+    end
+  end
+end

data/lib/activerecord_hoarder/version.rb

@@ -0,0 +1,3 @@
+module ActiverecordHoarder
+  VERSION = "0.0.1"
+end

data/lib/activerecord_hoarder.rb

@@ -0,0 +1,21 @@
+require 'activerecord_hoarder/aws_s3_storage'
+require 'activerecord_hoarder/batch'
+require 'activerecord_hoarder/batch_archiver'
+require 'activerecord_hoarder/core'
+require 'activerecord_hoarder/record_collector'
+require 'activerecord_hoarder/record_query'
+require 'activerecord_hoarder/restore'
+require 'activerecord_hoarder/serializer'
+require 'activerecord_hoarder/storage'
+require 'activerecord_hoarder/storage_error'
+require 'activerecord_hoarder/storage_key'
+require 'activerecord_hoarder/storages'
+
+module ActiverecordHoarder
+  def acts_as_hoarder
+    include ActiverecordHoarder::Core
+    include ActiverecordHoarder::Restore
+  end
+end
+
+::ActiveRecord::Base.send :extend, ActiverecordHoarder

data/spec/activerecord_hoarder/aws_s3_storage_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+DATA_TEMPLATE = "data for key %{key}"
+
+RSpec.describe ::ActiverecordHoarder::AwsS3 do
+  describe ".fetch_data" do
+    let(:date_data) { DATA_TEMPLATE % { key: full_key } }
+    let(:full_key) { File.join(table_name, key_string) }
+    let(:key) { double(key_string, to_s: key_string) }
+    let(:key_string) { "key" }
+    let(:storage) { ::ActiverecordHoarder::AwsS3.new(table_name, storage_options) }
+    let(:storage_options) { {} }
+    let(:table_name) { "records" }
+
+    before do
+      allow(storage).to receive(:s3_client).and_return(Aws::S3::Client.new(stub_responses: true))
+      allow_any_instance_of(Aws::S3::Client).to receive(:get_object) do |object, args|
+        double(body: DATA_TEMPLATE % { key: args[:key] })
+      end
+    end
+
+    it "returns the archive entry for given date" do
+      expect(storage.fetch_data(key)).to eq(date_data)
+    end
+  end
+end

data/spec/activerecord_hoarder/batch_archiver_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+RSpec.describe ::ActiverecordHoarder::BatchArchiver do
+  let(:model_class) { double("Record", table_name: "records") }
+
+  it "creates instance that can archive_batch" do
+    allow(::ActiverecordHoarder::Storage).to receive(:new).and_return(double)
+    expect(described_class.new(model_class)).to respond_to(:archive_batch)
+  end
+end

data/spec/activerecord_hoarder/core_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+RSpec.describe 'ActiverecordHoarder::Core' do
+  context 'included and instantiated' do
+    describe 'hoard' do
+      it "it works, no questions asked" do
+        expect(::ActiverecordHoarder::Storage).to receive(:new).and_return(double)
+        expect_any_instance_of(::ActiverecordHoarder::BatchArchiver).to receive(:archive_batch)
+        expect{ ExampleHoarder.hoard }.not_to raise_error
+      end
+    end
+  end
+end

data/spec/activerecord_hoarder/serializer_spec.rb

@@ -0,0 +1,7 @@
+require 'spec_helper'
+
+RSpec.describe ::ActiverecordHoarder::Serializer do
+  it "has class method for serialization" do
+    expect(described_class).to respond_to(:create_archive)
+  end
+end

data/spec/activerecord_hoarder/storage_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+RSpec.describe ::ActiverecordHoarder::Storage do
+  let(:stub_model) { double }
+
+  before do
+    allow(stub_model).to receive(:table_name)
+  end
+
+  describe "new" do
+    let(:storage) { ::ActiverecordHoarder::Storage.new(stub_model) }
+
+    context "not configured" do
+      it "fails and complains about configuration" do
+        expect{ ::ActiverecordHoarder::Storage.new(stub_model) }.to raise_error(::ActiverecordHoarder::StorageError)
+      end
+    end
+
+    context "aws_s3" do
+      before do
+        ::ActiverecordHoarder::Storage.configure(storage: :aws_s3, storage_options: {})
+      end
+
+      it "returns an aws storage" do
+        expect(storage.class).to be(::ActiverecordHoarder::AwsS3)
+      end
+    end
+  end
+end

data/spec/activerecord_hoarder_spec.rb

@@ -0,0 +1,168 @@
+require "spec_helper"
+
+RSpec.describe ActiverecordHoarder do
+
+  around(:all) do |example|
+    current_zone = Time.zone
+    Time.zone = "America/Chicago"
+    Timecop.freeze((Date.today.beginning_of_week - 3).to_time(:utc).end_of_day) do
+      example.run
+    end
+    Time.zone = current_zone
+  end
+
+  it "has a version number" do
+    expect(ActiverecordHoarder::VERSION).not_to be nil
+  end
+
+  it "extends ::ActiveRecord::Base with acts_as_hoarder" do
+    expect(::ActiveRecord::Base.methods).to include(:acts_as_hoarder)
+  end
+
+  describe "acts_as_hoarder" do
+    context "successfully included in ActiveRecord model" do
+      it "extends with public class method .hoard" do
+        expect(ExampleHoarder.methods).to include(:hoard)
+      end
+    end
+  end
+
+  describe "record archiving" do
+    let(:archive_data) {
+      @archivable_records.group_by { |item|
+        item.created_at.getutc.to_date
+      } .collect { |date, group|
+        JSON.pretty_generate(group.collect(&:serializable_hash))
+      }
+    }
+    let(:storage) { double }
+
+    before :each do
+      allow(::ActiverecordHoarder::Storage).to receive(:new).and_return(storage)
+      allow(storage).to receive(:store_data).and_return(true)
+    end
+
+    context "with records only in current week" do
+      before do
+        @archivable_records = create_list(
+          :examples_in_range,
+          20,
+          deleted: true,
+          start_time: Time.now.getutc.beginning_of_week,
+          end_time: (Date.today - 1).to_time(:utc).end_of_day
+        )
+        @non_archivable_records = create_list(
+          :examples_on_date,
+          4,
+          deleted: true,
+          records_date: Date.today
+        )
+        ExampleHoarder.hoard
+      end
+
+      it "ignores current day" do
+        expect(ExampleHoarder.unscoped.to_a).to include(*@non_archivable_records)
+      end
+
+      it "archives previous days" do
+        expect(ExampleHoarder.unscoped.to_a).not_to include(*@archivable_records)
+      end
+    end
+
+    context "with records in multiple weeks, non-deleted records mixed in and trailing" do
+      before do
+        @archivable_records = create_list(
+          :examples_in_range,
+          20,
+          deleted: true,
+          end_time: (1.week.ago.getutc.beginning_of_week + 2.days).end_of_day,
+          start_time: 1.week.ago.getutc.beginning_of_week
+        ) + create_list(
+          :examples_in_range,
+          20,
+          deleted: true,
+          end_time: 1.week.ago.getutc.end_of_week,
+          start_time: 1.week.ago.getutc.beginning_of_week + 4.days
+        )
+        @non_archivable_records = create_list(
+          :examples_on_date,
+          2,
+          deleted: true,
+          records_date: (1.week.ago.getutc.beginning_of_week - 1.day).to_date
+        ) + create_list(
+          :examples_on_date,
+          1,
+          deleted: false,
+          records_date: (1.week.ago.getutc.beginning_of_week - 1.day).to_date
+        ) + create_list(
+          :examples_on_date,
+          4,
+          deleted: true,
+          records_date: (1.week.ago.getutc.beginning_of_week + 3.days).to_date,
+        ) + create_list(
+          :examples_on_date,
+          2,
+          deleted: false,
+          records_date: (1.week.ago.getutc.beginning_of_week + 3.days).to_date
+        )
+        @out_of_range_records = create_list(
+          :examples_in_range,
+          2,
+          deleted: true,
+          end_time: Time.now,
+          start_time: Time.now.getutc.beginning_of_week
+        ) + create_list(
+          :examples_in_range,
+          2,
+          deleted: false,
+          end_time: Time.now,
+          start_time: Time.now.getutc.beginning_of_week
+        )
+        ExampleHoarder.hoard
+      end
+
+      it "skips records from days with active records" do
+        expect(ExampleHoarder.unscoped.to_a).to include(*@non_archivable_records)
+      end
+
+      it "archives one week of fully deleted records" do
+        expect(ExampleHoarder.unscoped.to_a).not_to include(*@archivable_records)
+      end
+
+      it "stops after one week" do
+        expect(ExampleHoarder.unscoped.to_a).to include(*@out_of_range_records)
+      end
+    end
+  end
+
+  describe "workflow" do
+    let(:batch1) { double }
+    let(:collector) { ::ActiverecordHoarder::RecordCollector.new(ExampleHoarder) }
+    let(:storage) { double }
+
+    before do
+      allow(::ActiverecordHoarder::Storage).to receive(:new).and_return(storage)
+      allow(::ActiverecordHoarder::RecordCollector).to receive(:new).and_return(collector)
+      allow(collector).to receive(:collect_batch).and_return(true, false)
+      allow(collector).to receive(:batch_data_cached?).and_return(true)
+      allow_any_instance_of(::ActiverecordHoarder::BatchArchiver).to receive(:compose_key).and_return("key")
+      allow(collector).to receive(:destroy_current_records!)
+    end
+
+    after do
+      ExampleHoarder.hoard
+    end
+
+    it "fully processes one record batch before moving on to the next" do
+      expect(collector).to receive(:collect_batch)
+      expect(storage).to receive(:store_data).and_return(true)
+      expect(collector).to receive(:destroy_current_records!)
+      expect(collector).to receive(:collect_batch)
+    end
+
+    it "does not delete a record that wasn't successfully archived" do
+      expect(storage).to receive(:store_data).and_return(false)
+      expect(collector).not_to receive(:destroy_current_records!)
+    end
+  end
+end

data/spec/factories/examples.rb

@@ -0,0 +1,40 @@
+def datetimes_in_range(n, step: 5, start:, stop:)
+  seconds_in_range = stop - start
+  seconds_ago = Time.now.getutc - stop
+  ((n*(step.hours+1.minute + 1.second)) % seconds_in_range + seconds_ago).seconds.ago
+end
+
+FactoryGirl.define do
+  factory :example_hoarder do
+    transient do
+      deleted false
+    end
+
+    after :build do |record, evaluator|
+      if evaluator.deleted
+        record.deleted_at = record.created_at
+      end
+    end
+  end
+
+  factory :examples_in_range, parent: :example_hoarder do
+    transient do
+      end_time Time.now
+      start_time 2.weeks.ago
+      step 5
+    end
+
+    sequence(:created_at, 0) { |n| datetimes_in_range(n, step: step, start: start_time, stop: end_time) }
+  end
+
+  factory :examples_on_date, parent: :example_hoarder do
+    transient do
+      records_date Date.today - 1
+      step 5
+    end
+
+    sequence(:created_at, 0) { |n|
+      created_at_value = datetimes_in_range(n, step: step, start: records_date.to_time(:utc), stop: records_date.to_time(:utc).end_of_day)
+    }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+require "bundler/setup"
+require "active_record"
+require "aws-sdk-s3"
+
+require "activerecord_hoarder"
+
+require "factory_girl_rails"
+require "pp"
+require "timecop"
+
+Dir.glob("spec/support/*.rb").each do |file| require File.expand_path(file) end
+
+class ExampleHoarder < ActiveRecord::Base
+  acts_as_hoarder
+end

data/spec/support/0_rspec_configuration.rb

@@ -0,0 +1,14 @@
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+
+  config.include FactoryGirl::Syntax::Methods
+  FactoryGirl.find_definitions
+end

data/spec/support/1_active_record_configuration.rb

@@ -0,0 +1 @@
+ActiveRecord::Base.establish_connection(YAML.load_file("config/dbspec_rspec.yml"))

data/spec/support/2_database_content.rb

@@ -0,0 +1,14 @@
+ActiveRecord::Schema.define do
+  create_table "example_hoarders", force: true do |t|
+    t.datetime "created_at"
+    t.datetime "deleted_at"
+    t.datetime "updated_at"
+    t.string "content"
+  end
+end
+
+RSpec.configure do |config|
+  config.after(:each) do
+    ActiveRecord::Base.connection.execute("DELETE FROM example_hoarders;")
+  end
+end

data/spec/support/3_test_storage.rb

@@ -0,0 +1,7 @@
+
+RSpec.configure do |config|
+  config.after(:each) do
+    ActiverecordHoarder.send(:remove_const, 'Storage')
+    load 'lib/activerecord_hoarder/storage.rb'
+  end
+end

metadata

@@ -0,0 +1,162 @@
+--- !ruby/object:Gem::Specification
+name: activerecord_hoarder
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Matthias Engh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-11-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: extend active records with simple archiving mechanics
+email:
+- matthias@wescrimmage.com
+executables:
+- console
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- activerecord_hoarder.gemspec
+- bin/console
+- bin/setup
+- config/activerecord_hoarder.yml.template
+- config/dbspec.yml.template
+- config/dbspec_rspec.yml.template
+- example/example.rb
+- example/fixture.rb
+- example/schema.rb
+- lib/activerecord_hoarder.rb
+- lib/activerecord_hoarder/aws_s3_storage.rb
+- lib/activerecord_hoarder/batch.rb
+- lib/activerecord_hoarder/batch_archiver.rb
+- lib/activerecord_hoarder/core.rb
+- lib/activerecord_hoarder/record_collector.rb
+- lib/activerecord_hoarder/record_query.rb
+- lib/activerecord_hoarder/restore.rb
+- lib/activerecord_hoarder/serializer.rb
+- lib/activerecord_hoarder/storage.rb
+- lib/activerecord_hoarder/storage_error.rb
+- lib/activerecord_hoarder/storage_key.rb
+- lib/activerecord_hoarder/storages.rb
+- lib/activerecord_hoarder/version.rb
+- spec/activerecord_hoarder/aws_s3_storage_spec.rb
+- spec/activerecord_hoarder/batch_archiver_spec.rb
+- spec/activerecord_hoarder/core_spec.rb
+- spec/activerecord_hoarder/serializer_spec.rb
+- spec/activerecord_hoarder/storage_spec.rb
+- spec/activerecord_hoarder_spec.rb
+- spec/factories/examples.rb
+- spec/spec_helper.rb
+- spec/support/0_rspec_configuration.rb
+- spec/support/1_active_record_configuration.rb
+- spec/support/2_database_content.rb
+- spec/support/3_test_storage.rb
+homepage: https://github.com/Scrimmage/gem_batch_archiving
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: hoards records
+test_files:
+- spec/activerecord_hoarder/aws_s3_storage_spec.rb
+- spec/activerecord_hoarder/batch_archiver_spec.rb
+- spec/activerecord_hoarder/core_spec.rb
+- spec/activerecord_hoarder/serializer_spec.rb
+- spec/activerecord_hoarder/storage_spec.rb
+- spec/activerecord_hoarder_spec.rb
+- spec/factories/examples.rb
+- spec/spec_helper.rb
+- spec/support/0_rspec_configuration.rb
+- spec/support/1_active_record_configuration.rb
+- spec/support/2_database_content.rb
+- spec/support/3_test_storage.rb