webtranslateit-safe 0.4.0

data/spec/webtranslateit/safe/mysqldump_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+describe WebTranslateIt::Safe::Mysqldump do
+
+  def def_config(extra = {})
+    {
+      options: 'OPTS',
+      user: 'User',
+      password: 'pwd',
+      host: 'localhost',
+      port: 7777,
+      socket: 'socket',
+      skip_tables: [:bar, :baz]
+    }.merge(extra)
+  end
+
+  def mysqldump(id = :foo, config = def_config)
+    WebTranslateIt::Safe::Mysqldump.new(id, WebTranslateIt::Safe::Config::Node.new(nil, config))
+  end
+
+  before(:each) do
+    stub(Time).now.stub!.strftime { 'NOW' }
+  end
+
+  after(:each) { WebTranslateIt::Safe::TmpFile.cleanup }
+
+  describe :backup do
+    before(:each) do
+      @mysql = mysqldump
+      stub(@mysql).mysql_password_file { '/tmp/pwd' }
+    end
+
+    {
+      id: 'foo',
+      kind: 'mysqldump',
+      extension: '.sql',
+      filename: 'mysqldump-foo.NOW',
+      command: 'mysqldump --defaults-extra-file=/tmp/pwd OPTS --ignore-table=foo.bar --ignore-table=foo.baz foo'
+    }.each do |k, v|
+      it "sets #{k} to #{v}" do
+        @mysql.backup.send(k).should == v
+      end
+    end
+
+  end
+
+  describe :mysql_skip_tables do
+    it 'returns nil if no skip_tables' do
+      config = def_config.dup
+      config.delete(:skip_tables)
+      m = mysqldump(:foo, WebTranslateIt::Safe::Config::Node.new(nil, config))
+      stub(m).timestamp { 'NOW' }
+      m.send(:mysql_skip_tables).should be_nil
+      m.backup.command.should_not match(/ignore-table/)
+    end
+
+    it "returns '' if skip_tables empty" do
+      config = def_config.dup
+      config[:skip_tables] = []
+      m = mysqldump(:foo, WebTranslateIt::Safe::Config::Node.new(nil, config))
+      stub(m).timestamp { 'NOW' }
+      m.send(:mysql_skip_tables).should
+      m.backup.command.should_not match(/ignore-table/)
+    end
+
+  end
+
+  describe :mysql_password_file do
+    it 'creates passwords file with quoted values' do
+      m = mysqldump(:foo, def_config(password: '#qwe"asd\'zxc'))
+      file = m.send(:mysql_password_file)
+      File.exist?(file).should
+      File.read(file).should == <<~PWD
+        [mysqldump]
+        user = "User"
+        password = "#qwe\\"asd'zxc"
+        socket = "socket"
+        host = "localhost"
+        port = 7777
+      PWD
+    end
+  end
+end

data/spec/webtranslateit/safe/pgdump_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+describe WebTranslateIt::Safe::Pgdump do
+
+  def def_config
+    {
+      options: 'OPTS',
+      user: 'User',
+      password: 'pwd',
+      host: 'localhost',
+      port: 7777,
+      skip_tables: [:bar, :baz]
+    }
+  end
+
+  def pgdump(id = :foo, config = def_config)
+    WebTranslateIt::Safe::Pgdump.new(id, WebTranslateIt::Safe::Config::Node.new(nil, config))
+  end
+
+  before(:each) do
+    stub(Time).now.stub!.strftime { 'NOW' }
+  end
+
+  after(:each) { WebTranslateIt::Safe::TmpFile.cleanup }
+
+  describe :backup do
+    before(:each) do
+      @pg = pgdump
+    end
+
+    {
+      id: 'foo',
+      kind: 'pgdump',
+      extension: '.sql',
+      filename: 'pgdump-foo.NOW',
+      command: "pg_dump OPTS --username='User' --host='localhost' --port='7777' foo"
+    }.each do |k, v|
+      it "sets #{k} to #{v}" do
+        @pg.backup.send(k).should == v
+      end
+    end
+
+  end
+
+end

data/spec/webtranslateit/safe/s3_spec.rb

@@ -0,0 +1,168 @@
+require 'spec_helper'
+
+describe WebTranslateIt::Safe::S3 do
+
+  def def_config
+    {
+      :s3 => {
+        :bucket => '_bucket',
+        :key    => '_key',
+        :secret => '_secret',
+      },
+      :keep => {
+        :s3 => 2
+      }
+    }
+  end
+
+  def def_backup(extra = {})
+    {
+      :kind      => '_kind',
+      :filename  => '/backup/somewhere/_kind-_id.NOW.bar',
+      :extension => '.bar',
+      :id        => '_id',
+      :timestamp => 'NOW'
+    }.merge(extra)
+  end
+
+  def s3(config = def_config, backup = def_backup)
+    WebTranslateIt::Safe::S3.new(
+      WebTranslateIt::Safe::Config::Node.new.merge(config),
+      WebTranslateIt::Safe::Backup.new(backup)
+    )
+  end
+
+  describe :cleanup do
+
+    before(:each) do
+      @s3 = s3
+
+      @files = [4,1,3,2].map { |i| stub(o = {}).key {"aaaaa#{i}"}; o }
+
+      stub(AWS::S3::Bucket).objects('_bucket', :prefix => '_kind/_id/_kind-_id.', :max_keys => 4) {@files}
+      stub(AWS::S3::Bucket).objects('_bucket', :prefix => anything).stub![0].stub!.delete
+    end
+
+    it 'should check [:keep, :s3]' do
+      @s3.config[:keep].data['s3'] = nil
+      dont_allow(@s3.backup).filename
+      @s3.send :cleanup
+    end
+
+    it 'should delete extra files' do
+      mock(AWS::S3::Bucket).objects('_bucket', :prefix => 'aaaaa1').mock![0].mock!.delete
+      mock(AWS::S3::Bucket).objects('_bucket', :prefix => 'aaaaa2').mock![0].mock!.delete
+      @s3.send :cleanup
+    end
+
+  end
+
+  describe :active do
+    before(:each) do
+      @s3 = s3
+    end
+
+    it 'should be true when all params are set' do
+      expect(@s3.active?).to be_truthy
+    end
+
+    it 'should be false if bucket is missing' do
+      @s3.config[:s3].data['bucket'] = nil
+      expect(@s3.active?).to be_falsy
+    end
+
+    it 'should be false if key is missing' do
+      @s3.config[:s3].data['key'] = nil
+      expect(@s3.active?).to be_falsy
+    end
+
+    it 'should be false if secret is missing' do
+      @s3.config[:s3].data['secret'] = nil
+      expect(@s3.active?).to be_falsy
+    end
+  end
+
+  describe :path do
+    before(:each) do
+      @s3 = s3
+    end
+    it 'should use s3/path 1st' do
+      @s3.config[:s3].data['path'] = 's3_path'
+      @s3.config[:local] = {:path => 'local_path'}
+      @s3.send(:path).should == 's3_path'
+    end
+
+    it 'should use local/path 2nd' do
+      @s3.config.merge local: {path: 'local_path'}
+      @s3.send(:path).should == 'local_path'
+    end
+
+    it 'should use constant 3rd' do
+      @s3.send(:path).should == '_kind/_id'
+    end
+
+  end
+
+  describe :save do
+    def add_stubs(*stubs)
+      stubs.each do |s|
+        case s
+        when :connection
+          stub(AWS::S3::Base).establish_connection!(:access_key_id => '_key', :secret_access_key => '_secret', :use_ssl => true)
+        when :stat
+          stub(File).stat('foo').stub!.size {123}
+        when :create_bucket
+          stub(AWS::S3::Bucket).find('_bucket') { raise_error AWS::S3::NoSuchBucket }
+          stub(AWS::S3::Bucket).create
+        when :file_open
+          stub(File).open('foo') {|f, block| block.call(:opened_file)}
+        when :s3_store
+          stub(AWS::S3::S3Object).store(@full_path, :opened_file, '_bucket')
+        end
+      end
+    end
+
+    before(:each) do
+      @s3 = s3(def_config, def_backup(:path => 'foo'))
+      @full_path = '_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar'
+    end
+
+    it 'should fail if no backup.file is set' do
+      @s3.backup.path = nil
+      proc {@s3.send(:save)}.should raise_error(RuntimeError)
+    end
+
+    it 'should establish s3 connection' do
+      mock(AWS::S3::Base).establish_connection!(:access_key_id => '_key', :secret_access_key => '_secret', :use_ssl => true)
+      add_stubs(:stat, :create_bucket, :file_open, :s3_store)
+      @s3.send(:save)
+    end
+
+    it 'should open local file' do
+      add_stubs(:connection, :stat, :create_bucket)
+      mock(File).open('foo')
+      @s3.send(:save)
+    end
+
+    it 'should upload file' do
+      add_stubs(:connection, :stat, :create_bucket, :file_open)
+      mock(AWS::S3::S3Object).store(@full_path, :opened_file, '_bucket')
+      @s3.send(:save)
+    end
+
+    it 'should fail on files bigger then 5G' do
+      add_stubs(:connection)
+      mock(File).stat('foo').stub!.size {5*1024*1024*1024+1}
+      mock(STDERR).puts(anything)
+      dont_allow(Benchmark).realtime
+      @s3.send(:save)
+    end
+
+    it 'should not create a bucket that already exists' do
+      add_stubs(:connection, :stat, :file_open, :s3_store)
+      stub(AWS::S3::Bucket).find('_bucket') { true }
+      dont_allow(AWS::S3::Bucket).create
+      @s3.send(:save)
+    end
+  end
+end

data/spec/webtranslateit/safe/svndump_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+describe WebTranslateIt::Safe::Svndump do
+  def def_config
+    {
+      options: 'OPTS',
+      repo_path: 'bar/baz'
+    }
+  end
+
+  def svndump(id = :foo, config = def_config)
+    WebTranslateIt::Safe::Svndump.new(id, WebTranslateIt::Safe::Config::Node.new(nil, config))
+  end
+
+  before(:each) do
+    stub(Time).now.stub!.strftime { 'NOW' }
+  end
+
+  after(:each) { WebTranslateIt::Safe::TmpFile.cleanup }
+
+  describe :backup do
+    before(:each) do
+      @svn = svndump
+    end
+
+    {
+      id: 'foo',
+      kind: 'svndump',
+      extension: '.svn',
+      filename: 'svndump-foo.NOW',
+      command: 'svnadmin dump OPTS bar/baz'
+    }.each do |k, v|
+      it "sets #{k} to #{v}" do
+        @svn.backup.send(k).should == v
+      end
+    end
+
+  end
+end

data/templates/script.rb

@@ -0,0 +1,183 @@
+safe do
+  # same as --verbose on the command line
+  # verbose true
+
+  # same as --local on the command line
+  # local_only true
+
+  # same as --dry-run on the command line
+  # dry_run true
+
+  # backup file path (not including filename)
+  # supported substitutions:
+  #   :kind -> backup 'engine' kind, e.g. "mysqldump" or "archive"
+  #   :id -> backup 'id', e.g. "blog", "production", etc.
+  #   :timestamp -> current run timestamp (same for all the backups in the same 'run')
+  # you can set separate :path for all backups (or once globally here)
+  local do
+    path '/backup/:kind'
+  end
+
+  ## uncomment to enable uploads to Amazon S3
+  ## Amazon S3 auth (optional)
+  # s3 do
+  #   key YOUR_S3_KEY
+  #   secret YOUR_S3_SECRET
+  #   bucket S3_BUCKET
+  #   # path for uploads to S3. supports same substitution like :local/:path
+  #   path ":kind/" # this is default
+  # end
+
+  ## alternative style:
+  # s3 :key => YOUR_S3_KEY, :secret => YOUR_S3_SECRET, :bucket => S3_BUCKET, :path => ":kind/"
+
+  ## uncomment to enable uploads to Rackspace Cloud Files
+  ## http://www.rackspacecloud.com/cloud_hosting_products/files
+  ## Rackspace auth (optional)
+  # cloudfiles do
+  #   user "YOUR_RACKSPACE_CLOUD_USERNAME"
+  #   api_key "YOUR_RACKSPACE_API_KEY"
+  #   container "YOUR_CONTAINER_NAME"
+  #   # path for uploads to Cloud Files, supports same substitution like :local/:path
+  #   path ":kind/" # this is default
+  #   # If you are running the backup from a system within the Rackspace/Slicehost network and would like
+  #   # to back up over the private (unbilled) service net, set this value to true.
+  #   # service_net true
+  # end
+
+  ## uncomment to enable uploads via SFTP
+  # sftp do
+  #   host "YOUR_REMOTE_HOSTNAME"
+  #   user "YOUR_REMOTE_USERNAME"
+  #   # port "NON STANDARD SSH PORT"
+  #   password "YOUR_REMOTE_PASSWORD"
+  #   path ":kind/:id" # this is the default
+  # end
+
+  ## uncomment to enable uploads via FTP
+  # ftp do
+  #   host "YOUR_REMOTE_HOSTNAME"
+  #   user "YOUR_REMOTE_USERNAME"
+  #   # port "NON STANDARD FTP PORT"
+  #   password "YOUR_REMOTE_PASSWORD"
+  #   path ":kind/:id" # this is the default
+  # end
+
+  ## uncomment to enable GPG encryption.
+  ## Note: you can use public 'key' or symmetric password but not both!
+  # gpg do
+  #   # you can specify your own gpg executable with the 'command' options
+  #   # this can be useful for example to choose b/w gpg and gpg2 if both are installed
+  #   # some gpg installations will automatically set 'use-agent' option in the
+  #   # config file on the 1st run. see README for more details
+  #   options "--no-use-agent"
+  #   # command "/usr/local/bin/gpg"
+  #   # key "backup@astrails.com"
+  #   password "astrails"
+  # end
+
+  ## uncomment to enable backup rotation. keep only given number of latest
+  ## backups. remove the rest
+  # keep do
+  #   local 4 # keep 4 local backups
+  #   s3 20 # keep 20 S3 backups
+  # end
+
+  # backup mysql databases with mysqldump
+  mysqldump do
+    # you can override any setting from parent in a child:
+    options '-ceKq --single-transaction --create-options'
+
+    user 'astrails'
+    password ''
+    # host "localhost"
+    # port 3306
+    socket '/var/run/mysqld/mysqld.sock'
+
+    # database is a 'collection' element. it must have a hash or block parameter
+    # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
+    # the following code will create mysqldump/databases/blog and mysqldump/databases/mysql configuration 'nodes'
+
+    # backup database with default values
+    # database :blog
+
+    # backup overriding some values
+    # database :production do
+    #   # you can override 'partially'
+    #   keep :local => 3
+    #   # keep/local is 3, and keep/s3 is 20 (from parent)
+
+    #   # local override for gpg password
+    #   gpg do
+    #     password "custom-production-pass"
+    #   end
+    #   # skip those tables during backup
+    #   # you can pass an array
+    #   skip_tables [:logger_exceptions, :request_logs]
+    #   # or pass them all separately
+    #   skip_tables :test1
+    #   skip_tables :test2
+    # end
+
+  end
+
+  # # uncomment to enable
+  # # backup PostgreSQL databases with pg_dump
+  # pgdump do
+  #   options "-i -x -O"
+  #
+  #   user "markmansour"
+  #   # password "" - leave this out if you have ident setup
+  #
+  #   # database is a 'collection' element. it must have a hash or block parameter
+  #   # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key
+  #   database :blog
+  #   database :production
+  # end
+
+  tar do
+    # options "-h" # uncomment this to dereference symbolic links
+
+    # 'archive' is a collection item, just like 'database'
+    # archive "git-repositories" do
+    #   # files and directories to backup
+    #   files "/home/git/repositories"
+    #   # can have more then one 'files' lines or/and use an array
+    #   files ["/home/dev/work/foo", "/home/dev/work/bar"]
+    # end
+
+    # archive "etc-files" do
+    #   files "/etc"
+    #   # exlude those files/directories
+    #   exclude "/etc/puppet/other"
+    #   # can have multiple 'exclude' lines or/and use an array
+    #   exclude ["/etc/tmp/a", "/etc/tmp/b"]
+    # end
+
+    # archive "dot-configs" do
+    #   files "/home/*/.[^.]*"
+    # end
+
+    # archive "blog" do
+    #   files "/var/www/blog.astrails.com/"
+    #   # specify multiple files/directories as array
+    #   exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"]
+    # end
+
+    # archive "site" do
+    #   files "/var/www/astrails.com/"
+    #   exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"]
+    # end
+
+    # archive :misc do
+    #   files [ "/backup/*.rb" ]
+    # end
+  end
+
+  # svndump do
+  #   repo :my_repo do
+  #     repo_path "/home/svn/my_repo"
+  #   end
+  # end
+
+end

data/webtranslateit-safe.gemspec

@@ -0,0 +1,32 @@
+require 'English'
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'webtranslateit/safe/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'webtranslateit-safe'
+  spec.version       = WebTranslateIt::Safe::VERSION
+  spec.authors       = ['Edouard Briere', 'Vitaly Kushner']
+  spec.email         = ['support@webtranslateit.com']
+  spec.required_ruby_version = '>= 3.2'
+  spec.description = <<~DESC
+    WebTranslateIt-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
+    Backups can be stored locally or remotely and can be enctypted.
+    Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain FTP/SFTP.
+  DESC
+  spec.summary       = 'Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)'
+  spec.homepage      = 'https://github.com/webtranslateit/safe'
+  spec.license       = 'MIT'
+
+  spec.default_executable = 'webtranslateit-safe'
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'aws-s3'
+  spec.add_dependency 'cloudfiles'
+  spec.add_dependency 'net-sftp'
+
+  spec.metadata['rubygems_mfa_required'] = 'true'
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: webtranslateit-safe
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- Edouard Briere
+- Vitaly Kushner
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cloudfiles
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  WebTranslateIt-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files.
+  Backups can be stored locally or remotely and can be enctypted.
+  Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain FTP/SFTP.
+email:
+- support@webtranslateit.com
+executables:
+- webtranslateit-safe
+extensions: []
+extra_rdoc_files: []
+files:
+- ".autotest"
+- ".document"
+- ".github/dependabot.yml"
+- ".github/release-drafter.yml"
+- ".github/workflows/ci.yml"
+- ".github/workflows/release-drafter.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- CHANGELOG
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.markdown
+- Rakefile
+- TODO
+- bin/webtranslateit-safe
+- lib/extensions/mktmpdir.rb
+- lib/webtranslateit/safe.rb
+- lib/webtranslateit/safe/archive.rb
+- lib/webtranslateit/safe/backup.rb
+- lib/webtranslateit/safe/cloudfiles.rb
+- lib/webtranslateit/safe/config/builder.rb
+- lib/webtranslateit/safe/config/node.rb
+- lib/webtranslateit/safe/ftp.rb
+- lib/webtranslateit/safe/gpg.rb
+- lib/webtranslateit/safe/gzip.rb
+- lib/webtranslateit/safe/local.rb
+- lib/webtranslateit/safe/mongodump.rb
+- lib/webtranslateit/safe/mysqldump.rb
+- lib/webtranslateit/safe/pgdump.rb
+- lib/webtranslateit/safe/pipe.rb
+- lib/webtranslateit/safe/s3.rb
+- lib/webtranslateit/safe/sftp.rb
+- lib/webtranslateit/safe/sink.rb
+- lib/webtranslateit/safe/source.rb
+- lib/webtranslateit/safe/stream.rb
+- lib/webtranslateit/safe/svndump.rb
+- lib/webtranslateit/safe/tmp_file.rb
+- lib/webtranslateit/safe/version.rb
+- spec/integration/archive_integration_spec.rb
+- spec/integration/cleanup_spec.rb
+- spec/spec_helper.rb
+- spec/webtranslateit/safe/archive_spec.rb
+- spec/webtranslateit/safe/cloudfiles_spec.rb
+- spec/webtranslateit/safe/config_spec.rb
+- spec/webtranslateit/safe/gpg_spec.rb
+- spec/webtranslateit/safe/gzip_spec.rb
+- spec/webtranslateit/safe/local_spec.rb
+- spec/webtranslateit/safe/mongodump_spec.rb
+- spec/webtranslateit/safe/mysqldump_spec.rb
+- spec/webtranslateit/safe/pgdump_spec.rb
+- spec/webtranslateit/safe/s3_spec.rb
+- spec/webtranslateit/safe/svndump_spec.rb
+- templates/script.rb
+- webtranslateit-safe.gemspec
+homepage: https://github.com/webtranslateit/safe
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.13
+signing_key:
+specification_version: 4
+summary: Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote
+  server/service (with encryption)
+test_files: []