bard-backup 0.3.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/lib/bard/backup/deleter.rb +6 -6
 - data/lib/bard/backup/{controller.rb → local_backhoe.rb} +5 -3
 - data/lib/bard/backup/s3_dir.rb +20 -7
 - data/lib/bard/backup/version.rb +1 -1
 - data/lib/bard/backup.rb +4 -6
 - metadata +3 -3
 
    
        checksums.yaml
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            ---
         
     | 
| 
       2 
2 
     | 
    
         
             
            SHA256:
         
     | 
| 
       3 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       4 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: c646d0f6f82c6bc07b6fdfc94256094e60ec681bde22b295aa7ad7f6766a0a23
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: 05f47f4952af321ac6a7183ed0643ecf134e9f847b42b3a14a05c0b30399d26f
         
     | 
| 
       5 
5 
     | 
    
         
             
            SHA512:
         
     | 
| 
       6 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       7 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: 8eea608f2e85b5bb968985e57a6ca0a87f127516e6e0638ab0657dc829f68f46c9a467d18e53c9b0fdb128c83aa88b25c6fa814d079777b506e592ce21cf34f2
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: 7aaac7dad88c8ca3c1f50d7a95157adb5c096a41347accd498c46f08905c3f573cd934f6aa7f3c364250628af3126b0e45fd65ec03dc26b70d0209c82ef04586
         
     | 
    
        data/lib/bard/backup/deleter.rb
    CHANGED
    
    | 
         @@ -6,24 +6,24 @@ module Bard 
     | 
|
| 
       6 
6 
     | 
    
         
             
              module Backup
         
     | 
| 
       7 
7 
     | 
    
         
             
                class Deleter < Struct.new(:s3_dir, :now)
         
     | 
| 
       8 
8 
     | 
    
         
             
                  def call
         
     | 
| 
       9 
     | 
    
         
            -
                    s3_dir.delete  
     | 
| 
      
 9 
     | 
    
         
            +
                    s3_dir.delete files_to_delete
         
     | 
| 
       10 
10 
     | 
    
         
             
                  end
         
     | 
| 
       11 
11 
     | 
    
         | 
| 
       12 
     | 
    
         
            -
                  def  
     | 
| 
       13 
     | 
    
         
            -
                    s3_dir. 
     | 
| 
      
 12 
     | 
    
         
            +
                  def files_to_delete
         
     | 
| 
      
 13 
     | 
    
         
            +
                    s3_dir.files.select do |file|
         
     | 
| 
       14 
14 
     | 
    
         
             
                      [
         
     | 
| 
       15 
15 
     | 
    
         
             
                        Filter.new(now, 72, :hours),
         
     | 
| 
       16 
16 
     | 
    
         
             
                        Filter.new(now, 60, :days),
         
     | 
| 
       17 
17 
     | 
    
         
             
                        Filter.new(now, 52, :weeks),
         
     | 
| 
       18 
18 
     | 
    
         
             
                        Filter.new(now, 48, :months),
         
     | 
| 
       19 
19 
     | 
    
         
             
                        Filter.new(now, 100, :years),
         
     | 
| 
       20 
     | 
    
         
            -
                      ].all? { |filter| !filter.cover?( 
     | 
| 
      
 20 
     | 
    
         
            +
                      ].all? { |filter| !filter.cover?(file) }
         
     | 
| 
       21 
21 
     | 
    
         
             
                    end
         
     | 
| 
       22 
22 
     | 
    
         
             
                  end
         
     | 
| 
       23 
23 
     | 
    
         | 
| 
       24 
24 
     | 
    
         
             
                  class Filter < Struct.new(:now, :limit, :unit)
         
     | 
| 
       25 
     | 
    
         
            -
                    def cover?  
     | 
| 
       26 
     | 
    
         
            -
                      remote = DateTime.parse( 
     | 
| 
      
 25 
     | 
    
         
            +
                    def cover? file
         
     | 
| 
      
 26 
     | 
    
         
            +
                      remote = DateTime.parse(file).beginning_of_hour
         
     | 
| 
       27 
27 
     | 
    
         
             
                      limit.times.any? do |count|
         
     | 
| 
       28 
28 
     | 
    
         
             
                        remote == ago(count)
         
     | 
| 
       29 
29 
     | 
    
         
             
                      end
         
     | 
| 
         @@ -1,10 +1,12 @@ 
     | 
|
| 
      
 1 
     | 
    
         
            +
            require "backhoe"
         
     | 
| 
      
 2 
     | 
    
         
            +
             
     | 
| 
       1 
3 
     | 
    
         
             
            module Bard
         
     | 
| 
       2 
4 
     | 
    
         
             
              module Backup
         
     | 
| 
       3 
     | 
    
         
            -
                class  
     | 
| 
       4 
     | 
    
         
            -
                  def call
         
     | 
| 
      
 5 
     | 
    
         
            +
                class LocalBackhoe
         
     | 
| 
      
 6 
     | 
    
         
            +
                  def self.call s3_dir, now
         
     | 
| 
       5 
7 
     | 
    
         
             
                    filename = "#{now.iso8601}.sql.gz"
         
     | 
| 
       6 
8 
     | 
    
         
             
                    path = "/tmp/#{filename}"
         
     | 
| 
       7 
     | 
    
         
            -
                     
     | 
| 
      
 9 
     | 
    
         
            +
                    Backhoe.dump path
         
     | 
| 
       8 
10 
     | 
    
         
             
                    s3_dir.mv path
         
     | 
| 
       9 
11 
     | 
    
         
             
                  end
         
     | 
| 
       10 
12 
     | 
    
         
             
                end
         
     | 
    
        data/lib/bard/backup/s3_dir.rb
    CHANGED
    
    | 
         @@ -4,13 +4,15 @@ require "rexml" 
     | 
|
| 
       4 
4 
     | 
    
         
             
            module Bard
         
     | 
| 
       5 
5 
     | 
    
         
             
              module Backup
         
     | 
| 
       6 
6 
     | 
    
         
             
                class S3Dir < Data.define(:path, :access_key, :secret_key, :region)
         
     | 
| 
       7 
     | 
    
         
            -
                  def  
     | 
| 
      
 7 
     | 
    
         
            +
                  def files
         
     | 
| 
       8 
8 
     | 
    
         
             
                    response = client.list_objects_v2({
         
     | 
| 
       9 
9 
     | 
    
         
             
                      bucket: bucket_name,
         
     | 
| 
       10 
10 
     | 
    
         
             
                      prefix: folder_prefix,
         
     | 
| 
       11 
11 
     | 
    
         
             
                    })
         
     | 
| 
       12 
12 
     | 
    
         
             
                    raise if response.is_truncated
         
     | 
| 
       13 
     | 
    
         
            -
                    response.contents.map 
     | 
| 
      
 13 
     | 
    
         
            +
                    response.contents.map do |object|
         
     | 
| 
      
 14 
     | 
    
         
            +
                      object.key.sub("#{folder_prefix}/", "")
         
     | 
| 
      
 15 
     | 
    
         
            +
                    end
         
     | 
| 
       14 
16 
     | 
    
         
             
                  end
         
     | 
| 
       15 
17 
     | 
    
         | 
| 
       16 
18 
     | 
    
         
             
                  def put file_path, body: File.read(file_path)
         
     | 
| 
         @@ -21,14 +23,25 @@ module Bard 
     | 
|
| 
       21 
23 
     | 
    
         
             
                    })
         
     | 
| 
       22 
24 
     | 
    
         
             
                  end
         
     | 
| 
       23 
25 
     | 
    
         | 
| 
      
 26 
     | 
    
         
            +
                  def presigned_url file_path
         
     | 
| 
      
 27 
     | 
    
         
            +
                    presigner = Aws::S3::Presigner.new(client: client)
         
     | 
| 
      
 28 
     | 
    
         
            +
                    presigner.presigned_url(
         
     | 
| 
      
 29 
     | 
    
         
            +
                      :put_object,
         
     | 
| 
      
 30 
     | 
    
         
            +
                      bucket: bucket_name,
         
     | 
| 
      
 31 
     | 
    
         
            +
                      key: [folder_prefix, File.basename(file_path)].compact.join("/"),
         
     | 
| 
      
 32 
     | 
    
         
            +
                    )
         
     | 
| 
      
 33 
     | 
    
         
            +
                  end
         
     | 
| 
      
 34 
     | 
    
         
            +
             
     | 
| 
       24 
35 
     | 
    
         
             
                  def mv file_path, body: File.read(file_path)
         
     | 
| 
       25 
36 
     | 
    
         
             
                    put file_path, body: body
         
     | 
| 
       26 
37 
     | 
    
         
             
                    FileUtils.rm file_path
         
     | 
| 
       27 
38 
     | 
    
         
             
                  end
         
     | 
| 
       28 
39 
     | 
    
         | 
| 
       29 
     | 
    
         
            -
                  def delete  
     | 
| 
       30 
     | 
    
         
            -
                    return if  
     | 
| 
       31 
     | 
    
         
            -
                    objects_to_delete = Array( 
     | 
| 
      
 40 
     | 
    
         
            +
                  def delete file_paths
         
     | 
| 
      
 41 
     | 
    
         
            +
                    return if file_paths.empty?
         
     | 
| 
      
 42 
     | 
    
         
            +
                    objects_to_delete = Array(file_paths).map do |file_path|
         
     | 
| 
      
 43 
     | 
    
         
            +
                      { key: [folder_prefix, File.basename(file_path)].compact.join("/") }
         
     | 
| 
      
 44 
     | 
    
         
            +
                    end
         
     | 
| 
       32 
45 
     | 
    
         
             
                    client.delete_objects({
         
     | 
| 
       33 
46 
     | 
    
         
             
                      bucket: bucket_name,
         
     | 
| 
       34 
47 
     | 
    
         
             
                      delete: {
         
     | 
| 
         @@ -39,8 +52,8 @@ module Bard 
     | 
|
| 
       39 
52 
     | 
    
         
             
                  end
         
     | 
| 
       40 
53 
     | 
    
         | 
| 
       41 
54 
     | 
    
         
             
                  def empty!
         
     | 
| 
       42 
     | 
    
         
            -
                     
     | 
| 
       43 
     | 
    
         
            -
                      delete  
     | 
| 
      
 55 
     | 
    
         
            +
                    files.each_slice(1000) do |batch|
         
     | 
| 
      
 56 
     | 
    
         
            +
                      delete batch
         
     | 
| 
       44 
57 
     | 
    
         
             
                    end
         
     | 
| 
       45 
58 
     | 
    
         
             
                  end
         
     | 
| 
       46 
59 
     | 
    
         | 
    
        data/lib/bard/backup/version.rb
    CHANGED
    
    
    
        data/lib/bard/backup.rb
    CHANGED
    
    | 
         @@ -1,14 +1,12 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require "bard/backup/controller"
         
     | 
| 
       2 
     | 
    
         
            -
            require "bard/backup/deleter"
         
     | 
| 
       3 
1 
     | 
    
         
             
            require "bard/backup/s3_dir"
         
     | 
| 
       4 
     | 
    
         
            -
            require " 
     | 
| 
      
 2 
     | 
    
         
            +
            require "bard/backup/local_backhoe"
         
     | 
| 
      
 3 
     | 
    
         
            +
            require "bard/backup/deleter"
         
     | 
| 
       5 
4 
     | 
    
         | 
| 
       6 
5 
     | 
    
         
             
            module Bard
         
     | 
| 
       7 
6 
     | 
    
         
             
              module Backup
         
     | 
| 
       8 
     | 
    
         
            -
                def self.call s3_path, access_key:, secret_key:, region: "us-west-2", now: Time.now.utc
         
     | 
| 
       9 
     | 
    
         
            -
                  dumper = Backhoe
         
     | 
| 
      
 7 
     | 
    
         
            +
                def self.call s3_path, access_key:, secret_key:, region: "us-west-2", now: Time.now.utc, strategy: LocalBackhoe
         
     | 
| 
       10 
8 
     | 
    
         
             
                  s3_dir = S3Dir.new(path: s3_path, access_key:, secret_key:, region:)
         
     | 
| 
       11 
     | 
    
         
            -
                   
     | 
| 
      
 9 
     | 
    
         
            +
                  strategy.call(s3_dir, now)
         
     | 
| 
       12 
10 
     | 
    
         
             
                  Deleter.new(s3_dir, now).call
         
     | 
| 
       13 
11 
     | 
    
         
             
                end
         
     | 
| 
       14 
12 
     | 
    
         
             
              end
         
     | 
    
        metadata
    CHANGED
    
    | 
         @@ -1,14 +1,14 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            --- !ruby/object:Gem::Specification
         
     | 
| 
       2 
2 
     | 
    
         
             
            name: bard-backup
         
     | 
| 
       3 
3 
     | 
    
         
             
            version: !ruby/object:Gem::Version
         
     | 
| 
       4 
     | 
    
         
            -
              version: 0. 
     | 
| 
      
 4 
     | 
    
         
            +
              version: 0.5.0
         
     | 
| 
       5 
5 
     | 
    
         
             
            platform: ruby
         
     | 
| 
       6 
6 
     | 
    
         
             
            authors:
         
     | 
| 
       7 
7 
     | 
    
         
             
            - Micah Geisel
         
     | 
| 
       8 
8 
     | 
    
         
             
            autorequire: 
         
     | 
| 
       9 
9 
     | 
    
         
             
            bindir: exe
         
     | 
| 
       10 
10 
     | 
    
         
             
            cert_chain: []
         
     | 
| 
       11 
     | 
    
         
            -
            date: 2024-08- 
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2024-08-26 00:00:00.000000000 Z
         
     | 
| 
       12 
12 
     | 
    
         
             
            dependencies:
         
     | 
| 
       13 
13 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     | 
| 
       14 
14 
     | 
    
         
             
              name: backhoe
         
     | 
| 
         @@ -81,8 +81,8 @@ files: 
     | 
|
| 
       81 
81 
     | 
    
         
             
            - Rakefile
         
     | 
| 
       82 
82 
     | 
    
         
             
            - lib/bard-backup.rb
         
     | 
| 
       83 
83 
     | 
    
         
             
            - lib/bard/backup.rb
         
     | 
| 
       84 
     | 
    
         
            -
            - lib/bard/backup/controller.rb
         
     | 
| 
       85 
84 
     | 
    
         
             
            - lib/bard/backup/deleter.rb
         
     | 
| 
      
 85 
     | 
    
         
            +
            - lib/bard/backup/local_backhoe.rb
         
     | 
| 
       86 
86 
     | 
    
         
             
            - lib/bard/backup/s3_dir.rb
         
     | 
| 
       87 
87 
     | 
    
         
             
            - lib/bard/backup/version.rb
         
     | 
| 
       88 
88 
     | 
    
         
             
            - sig/bard/backup.rbs
         
     |