elastic-backup-2s3 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 47eee9a9335bffd0b8bab48968951cdc083e2681
+  data.tar.gz: f7fd70416ea1b88a132cb96a4bdf3a02beadf151
+SHA512:
+  metadata.gz: 9c0d514cee046bc8d4e5ab3d4ed51924a2826fc48613c63272b71e56cff4cab5c79b8443a3cb8c8937edf6da4accabcc4118057ea8d41f3250ace73529edcc06
+  data.tar.gz: 36e251166434dc4ac969621bc73e3d2690ec57616913934a3a30a0a05412dc81f0885db2294f9aafa129ed0173ba052593d1ee4a71e0b64d68c3bfb8205df073

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/.ruby-version

@@ -0,0 +1 @@
+2.3.0

data/.semver

@@ -0,0 +1,5 @@
+---
+:major: 0
+:minor: 0
+:patch: 1
+:special: ''

data/Gemfile

@@ -0,0 +1,26 @@
+source "https://rubygems.org"
+
+gem "semver", "~> 1"
+gem "aws-sdk", "~> 2"
+gem "elasticsearch", "~> 1"
+gem "multi_json",  "~> 1"
+gem "faraday",  "~> 0"
+gem "activesupport", "~> 4"
+gem "awesome_print",  "~> 1"
+gem "text-table",  "~> 1"
+
+group :development do
+  gem "rspec", "~> 2.8.0"
+  gem "yard", "~> 0.7"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0"
+  gem "juwelier", "~> 2.1.0"
+  gem "simplecov", ">= 0"
+
+  gem 'pry',            '~> 0'
+  gem 'pry-byebug',     '~> 3'
+  gem 'pry-doc',        '~> 0'
+  gem 'pry-remote',     '~> 0'
+  gem 'pry-rescue',     '~> 1'
+  gem 'pry-stack_explorer', '~> 0'
+end

data/Gemfile.lock

@@ -0,0 +1,149 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.6)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    addressable (2.4.0)
+    awesome_print (1.6.1)
+    aws-sdk (2.2.34)
+      aws-sdk-resources (= 2.2.34)
+    aws-sdk-core (2.2.34)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.2.34)
+      aws-sdk-core (= 2.2.34)
+    binding_of_caller (0.7.2)
+      debug_inspector (>= 0.0.1)
+    builder (3.2.2)
+    byebug (8.2.4)
+    coderay (1.1.1)
+    debug_inspector (0.0.2)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.1.3)
+    docile (1.1.5)
+    elasticsearch (1.0.17)
+      elasticsearch-api (= 1.0.17)
+      elasticsearch-transport (= 1.0.17)
+    elasticsearch-api (1.0.17)
+      multi_json
+    elasticsearch-transport (1.0.17)
+      faraday
+      multi_json
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    git (1.3.0)
+    github_api (0.13.1)
+      addressable (~> 2.4.0)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.4)
+      multi_json (>= 1.7.5, < 2.0)
+      oauth2
+    hashie (3.4.3)
+    highline (1.7.8)
+    i18n (0.7.0)
+    interception (0.5)
+    jmespath (1.2.4)
+      json_pure (>= 1.8.1)
+    json (1.8.3)
+    json_pure (1.8.3)
+    juwelier (2.1.2)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+      semver
+    jwt (1.5.1)
+    method_source (0.8.2)
+    mini_portile2 (2.0.0)
+    minitest (5.8.4)
+    multi_json (1.11.2)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.7.2)
+      mini_portile2 (~> 2.0.0.rc2)
+    oauth2 (1.1.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0, < 1.5.2)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (>= 1.2, < 3)
+    pry (0.10.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (3.3.0)
+      byebug (~> 8.0)
+      pry (~> 0.10)
+    pry-doc (0.8.0)
+      pry (~> 0.9)
+      yard (~> 0.8)
+    pry-remote (0.1.8)
+      pry (~> 0.9)
+      slop (~> 3.0)
+    pry-rescue (1.4.2)
+      interception (>= 0.5)
+      pry
+    pry-stack_explorer (0.4.9.2)
+      binding_of_caller (>= 0.7)
+      pry (>= 0.9.11)
+    rack (1.6.4)
+    rake (11.1.2)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
+    semver (1.0.1)
+    simplecov (0.11.2)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    slop (3.6.0)
+    text-table (1.2.4)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    yard (0.8.7.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 4)
+  awesome_print (~> 1)
+  aws-sdk (~> 2)
+  bundler (~> 1.0)
+  elasticsearch (~> 1)
+  faraday (~> 0)
+  juwelier (~> 2.1.0)
+  multi_json (~> 1)
+  pry (~> 0)
+  pry-byebug (~> 3)
+  pry-doc (~> 0)
+  pry-remote (~> 0)
+  pry-rescue (~> 1)
+  pry-stack_explorer (~> 0)
+  rdoc (~> 3.12)
+  rspec (~> 2.8.0)
+  semver (~> 1)
+  simplecov
+  text-table (~> 1)
+  yard (~> 0.7)
+
+BUNDLED WITH
+   1.11.2

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2016 Fred Mitchell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.org

@@ -0,0 +1,106 @@
+* elastic-backup-2s3 -- Snapshot, Backup, and Restore Elasicsearch Indicies to and from S3
+  
+  When you have an itch to scratch...
+
+  I simply want to be able to control the backup and
+  restoration of the Elasticsearch cluster to S3
+  without any fuss or having to dilly around with
+  curl -XPUTS and friends.
+
+  So here it is, and I make no apologies about not 
+  supporting the "shared volume" option of Elasticsearch
+  snapshots. I may add support at a later date. Or
+  feel free to add it and do a pull request.
+
+  There are many features I wish to add to this, and
+  if you have any suggestions, please feel free to send
+  them my way!
+
+** Overview
+   Elastic Backup simply allows you to specifiy some
+   or all of the indicies, some target bucket, and 
+   run the backup / snapshot.
+
+   It hides all the sordid details about "registration"
+   and the like, and will optionally clean up behind 
+   itself and remove the registration if you did not
+   specify a name for the registration.
+
+** TODO Documentation
+   Documentation is comming. Basically,
+   just running the command without any parameters
+   will give you some basic usage. 
+
+*** S3URL format
+    We adopt a convention to make it easy for you
+    to specify the S3 bucket, path, and snapshot
+    name stored in the bucket. The S3 URL format is:
+
+    s3://AWS_S3_BUCKET/path/in/bucket:SNAPSHOT
+
+    In some cases, :SNAPSHOT is optional, but usually
+    expected to be there.
+
+** Examples
+   To backup/snapshot your ES to S3
+   and wait until the backup is completed:
+   #+BEGIN_SRC bash
+   es-snapshot snapshot localhost s3://your-backup-bucket/your-backup-path:snap_1 -v2 -w
+   #+END_SRC
+   
+   The default repository of 'elastic-backup' is used.
+   To specify your own repository name:
+   #+BEGIN_SRC bash
+   es-snapshot snapshot localhost s3://your-backup-bucket/your-backup-path:snap_1 -v2 -w --repo your-repo-name
+   #+END_SRC
+
+
+   To backup/snapshot two indices index1 and index2:
+   #+BEGIN_SRC bash
+   es-snapshot snapshot localhost s3://your-backup-bucket/your-backup-path:snap_1 -v2 -w --indices index1 index2
+   #+END_SRC
+
+   To restore your snapshot
+   #+BEGIN_SRC bash
+   es-snapshot restore s3://your-backup-bucket/your-backup-path:snap_1 localhost -v2 -w
+   #+END_SRC
+
+   To restore your snapshot using your-repo-name:
+   #+BEGIN_SRC bash
+   es-snapshot restore s3://your-backup-bucket/your-backup-path:snap_1 localhost -v2 -w --repo your-repo-name
+   #+END_SRC
+   
+   You may specify --dryrun (-u) to see the JSON that will
+   be sent without actually executing the commands. Also
+   -v2 will show you the same while allowing your 
+   commands to execute.
+
+   To see the status of your snapshots:
+   #+BEGIN_SRC bash
+   es-snapshot list status localhost
+   #+END_SRC
+
+   To see a list of your indices:
+   #+BEGIN_SRC bash
+   es-snapshot list indices localhost
+   #+END_SRC
+
+** Scratchpad
+   My own personal notes on this project. These
+   are not gauranteed to be useful to you nor 
+   up-to-date.
+
+** Contributing to elastic-backup-2s3
+
++ Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
++ Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
++ Fork the project.
++ Start a feature/bugfix branch.
++ Commit and push until you are happy with your contribution.
++ Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
++ Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+** Copyright
+
+   Copyright (c) 2016 Fred Mitchell. See LICENSE.txt for
+   further details.

data/Rakefile

@@ -0,0 +1,66 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+require 'semver'
+
+def s_version
+  SemVer.find.format "%M.%m.%p%s"
+end
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'juwelier'
+Juwelier::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "elastic-backup-2s3"
+  gem.homepage = "http://github.com/flajann2/elastic-backup-2s3"
+  gem.license = "MIT"
+  gem.version = s_version
+  gem.summary = %Q{Elasticsearch to AWS S3 Backup, Snapshotting, and Restore Tool}
+  gem.description = %Q{
+  I simply want to be able to control the backup and
+  restoration of the Elasticsearch cluster to S3
+  without any fuss or having to dilly around with
+  curl -XPUTS and friends.
+
+  So here it is, and I make no apologies about not 
+  supporting the "shared volume" option of Elasticsearch
+  snapshots. I may add support at a later date. Or
+  feel free to add it and do a pull request.
+
+  There are many features I wish to add to this, and
+  if you have any suggestions, please feel free to send
+  them my way!}
+
+  gem.email = "fred.mitchell@gmx.de"
+  gem.authors = ["Fred Mitchell"]
+  gem.required_ruby_version = '>= 2.0'
+
+  # dependencies defined in Gemfile
+end
+Juwelier::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+desc "Code coverage detail"
+task :simplecov do
+  ENV['COVERAGE'] = "true"
+  Rake::Task['spec'].execute
+end
+
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/bin/es-snapshot

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+ESS_PATH = File.expand_path '..', File.dirname(__FILE__)
+ESS_EXECUTABLE = File.expand_path 'bin/es-snapshot', Dir.pwd
+
+$:.unshift File.join([ESS_PATH, 'lib'])
+
+require 'elastic-backup-2s3/cli'
+
+ElasticBackup::Cli::Main.start

data/elastic-backup-2s3.gemspec

@@ -0,0 +1,117 @@
+# Generated by juwelier
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: elastic-backup-2s3 0.0.1 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "elastic-backup-2s3"
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
+  s.authors = ["Fred Mitchell"]
+  s.date = "2016-04-15"
+  s.description = "\n  I simply want to be able to control the backup and\n  restoration of the Elasticsearch cluster to S3\n  without any fuss or having to dilly around with\n  curl -XPUTS and friends.\n\n  So here it is, and I make no apologies about not \n  supporting the \"shared volume\" option of Elasticsearch\n  snapshots. I may add support at a later date. Or\n  feel free to add it and do a pull request.\n\n  There are many features I wish to add to this, and\n  if you have any suggestions, please feel free to send\n  them my way!"
+  s.email = "fred.mitchell@gmx.de"
+  s.executables = ["es-snapshot"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.org"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    ".ruby-version",
+    ".semver",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.org",
+    "Rakefile",
+    "bin/es-snapshot",
+    "elastic-backup-2s3.gemspec",
+    "lib/elastic-backup-2s3.rb",
+    "lib/elastic-backup-2s3/cli.rb",
+    "lib/elastic-backup-2s3/cli/delete.rb",
+    "lib/elastic-backup-2s3/cli/list.rb",
+    "lib/elastic-backup-2s3/snapshot.rb",
+    "spec/elastic-backup-2s3_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = "http://github.com/flajann2/elastic-backup-2s3"
+  s.licenses = ["MIT"]
+  s.required_ruby_version = Gem::Requirement.new(">= 2.0")
+  s.rubygems_version = "2.5.1"
+  s.summary = "Elasticsearch to AWS S3 Backup, Snapshotting, and Restore Tool"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<semver>, ["~> 1"])
+      s.add_runtime_dependency(%q<aws-sdk>, ["~> 2"])
+      s.add_runtime_dependency(%q<elasticsearch>, ["~> 1"])
+      s.add_runtime_dependency(%q<multi_json>, ["~> 1"])
+      s.add_runtime_dependency(%q<faraday>, ["~> 0"])
+      s.add_runtime_dependency(%q<activesupport>, ["~> 4"])
+      s.add_runtime_dependency(%q<awesome_print>, ["~> 1"])
+      s.add_runtime_dependency(%q<text-table>, ["~> 1"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_development_dependency(%q<yard>, ["~> 0.7"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<juwelier>, ["~> 2.1.0"])
+      s.add_development_dependency(%q<simplecov>, [">= 0"])
+      s.add_development_dependency(%q<pry>, ["~> 0"])
+      s.add_development_dependency(%q<pry-byebug>, ["~> 3"])
+      s.add_development_dependency(%q<pry-doc>, ["~> 0"])
+      s.add_development_dependency(%q<pry-remote>, ["~> 0"])
+      s.add_development_dependency(%q<pry-rescue>, ["~> 1"])
+      s.add_development_dependency(%q<pry-stack_explorer>, ["~> 0"])
+    else
+      s.add_dependency(%q<semver>, ["~> 1"])
+      s.add_dependency(%q<aws-sdk>, ["~> 2"])
+      s.add_dependency(%q<elasticsearch>, ["~> 1"])
+      s.add_dependency(%q<multi_json>, ["~> 1"])
+      s.add_dependency(%q<faraday>, ["~> 0"])
+      s.add_dependency(%q<activesupport>, ["~> 4"])
+      s.add_dependency(%q<awesome_print>, ["~> 1"])
+      s.add_dependency(%q<text-table>, ["~> 1"])
+      s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_dependency(%q<yard>, ["~> 0.7"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<juwelier>, ["~> 2.1.0"])
+      s.add_dependency(%q<simplecov>, [">= 0"])
+      s.add_dependency(%q<pry>, ["~> 0"])
+      s.add_dependency(%q<pry-byebug>, ["~> 3"])
+      s.add_dependency(%q<pry-doc>, ["~> 0"])
+      s.add_dependency(%q<pry-remote>, ["~> 0"])
+      s.add_dependency(%q<pry-rescue>, ["~> 1"])
+      s.add_dependency(%q<pry-stack_explorer>, ["~> 0"])
+    end
+  else
+    s.add_dependency(%q<semver>, ["~> 1"])
+    s.add_dependency(%q<aws-sdk>, ["~> 2"])
+    s.add_dependency(%q<elasticsearch>, ["~> 1"])
+    s.add_dependency(%q<multi_json>, ["~> 1"])
+    s.add_dependency(%q<faraday>, ["~> 0"])
+    s.add_dependency(%q<activesupport>, ["~> 4"])
+    s.add_dependency(%q<awesome_print>, ["~> 1"])
+    s.add_dependency(%q<text-table>, ["~> 1"])
+    s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+    s.add_dependency(%q<yard>, ["~> 0.7"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<juwelier>, ["~> 2.1.0"])
+    s.add_dependency(%q<simplecov>, [">= 0"])
+    s.add_dependency(%q<pry>, ["~> 0"])
+    s.add_dependency(%q<pry-byebug>, ["~> 3"])
+    s.add_dependency(%q<pry-doc>, ["~> 0"])
+    s.add_dependency(%q<pry-remote>, ["~> 0"])
+    s.add_dependency(%q<pry-rescue>, ["~> 1"])
+    s.add_dependency(%q<pry-stack_explorer>, ["~> 0"])
+  end
+end
+

data/lib/elastic-backup-2s3/cli/delete.rb

@@ -0,0 +1,8 @@
+module ElasticBackup
+  module Cli
+    class Delete < Thor
+
+    end
+  end
+end
+

data/lib/elastic-backup-2s3/cli/list.rb

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+module ElasticBackup
+  module Cli
+    class List < Thor
+
+      desc 'indices [ES]', 'List indicies in Elasticsearch'
+      option :detailed, type: :boolean, aliases: '-d', desc: "Give full detailed dump of the indices.", default: false
+      def indices es = "localhost"
+        esurl = Snapshot.esurl suri: es
+        puts "query #{esurl}" unless options[:verbose] < 1
+        cli = Snapshot.elastic esurl
+        indices = MultiJson.load cli.cat.indices
+
+        unless options[:detailed]
+          table = Text::Table.new
+          table.head = ['Index',
+                        'Status',
+                        'Health',
+                        'Docs',
+                        'Size']
+          table.rows = indices.map { |idx|
+            [idx["index"],
+             idx["status"],
+             idx["health"],
+             idx["docs.count"],
+             idx["store.size"]]
+          }
+          puts table
+        else
+          ap indices
+        end
+      end
+
+      desc 'snaphots [S3URL]', 'List all snapshots or clusters stored at S3URL. A cluster is assumed to be a collection of clusters within the S3URL, each containing one or more snapshots.'
+      #option :detailed, type: :boolean, aliases: '-d', desc: "Give full detailed dump of the snapshots.", default: false
+      #option :clusters, type: :boolean, aliases: '-c', desc: "List clusters instead of snapshots. NOT SUPPORTED YET", default: false
+      def snapshots surl
+        bucket_name, path, snapshot = Snapshot.s3url_splice surl
+        puts "from bucket #{bucket_name}, path #{path}" unless options[:verbose] < 1
+        bucket = Snapshot.s3.bucket bucket_name
+        raise "bucket #{bucket_name} does not exist" unless bucket.exists?
+        ap bucket.objects(prefix: "#{path}/snapshot").map { |ob|
+          ob.key.split('snapshot-').last
+        }
+      end
+
+      desc 'status [ES]', 'List the status of the snapshots on the given ES cluster.'
+      option :detailed, type: :boolean, aliases: '-d', desc: "Give full detailed dump of the status.", default: false
+      def status es = "localhost"
+        esurl = Snapshot.esurl suri: es
+        puts "query #{esurl}" unless options[:verbose] < 1
+        cli = Snapshot.elastic esurl
+        statuses = MultiJson.load cli.snapshot.get(repository: options[:repo],
+                                                   snapshot: '_all')
+        unless options[:detailed]
+          table = Text::Table.new
+          table.head = ['Snapshot', 'State', 'Started']
+          table.rows = statuses['snapshots'].map { |s|
+            [s["snapshot"],
+             s["state"],
+             s["start_time"]]
+          }
+          puts table
+        else
+          ap statuses
+        end
+
+      end
+    end
+  end
+end

data/lib/elastic-backup-2s3/cli.rb

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+require 'elastic-backup-2s3'
+require_relative 'cli/delete'
+require_relative 'cli/list'
+
+module ElasticBackup
+  module Cli
+    class Main < Thor
+      long_desc <<-LONGDESC
+        Main long desc
+      LONGDESC
+
+      class_option :verbose, type: :numeric, 
+                   banner: '[1|2|3]',
+                   desc: "Verbosity setting.",
+                   aliases: '-v', default: 0
+
+      class_option :repo, type: :string, 
+                   aliases: '-r',
+                   banner: "[NAME]",
+                   default: 'elastic-backup',
+                   desc: "Repository name to use."
+
+      class_option :monitor, type: :boolean, aliases: '-m', desc: "Monitor the progress.", default: false
+      class_option :wait, type: :boolean, aliases: '-w', desc: "Wait for completion.", default: false
+      class_option :timeout, type: :numeric,
+                   banner: '[SECONDS]',
+                   desc: "Explicit operation timeout for connection to master node.",
+                   aliases: '-t', default: 60
+
+      class_option :indices,  type: :array,   aliases: ['-i', '--indexes'],
+                   banner: "[INDEX1[ INDEX2...]|all]",
+                   required: false,
+                   desc: 'A list of indices to snapshot'
+
+      class_option :dryrun, type: :boolean,
+                   aliases: '-u',
+                   desc: "Dry run, do not actually execute."
+
+      desc 'snapshot [ES S3URL]', 'Backups Elasticsearch indices to S3'
+      def snapshot es, s3url
+        Snapshot.snapshot Snapshot.esurl(suri: es), s3url, options
+      end
+
+      desc 'restore [S3URL ES]', 'Restore indices from S3 to Elasticsearch.'
+      def restore s3url, es
+        Snapshot.restore s3url, Snapshot.esurl(suri: es), options
+      end
+      
+      desc 'monitor [ES, SNAPSHOT]', 'Not Implemented Yet -- Monitor the progress of an ongoing snapshot or restore.'
+      def monitor 
+      end
+
+      desc 'delete', 'Not Implemented Yet -- Delete snapshots, indicies, registrations'
+      subcommand 'delete', Delete
+
+      desc 'list', 'list indicies, snapshots'
+      subcommand 'list', List
+
+      no_commands do
+        def massage(options)
+          opt = Thor::CoreExt::HashWithIndifferentAccess.new options
+          opt[:extra] = Thor::CoreExt::HashWithIndifferentAccess.new opt[:extra].map{ |s| s.split(':', 2)}.to_h
+          opt[:tags] = opt[:tags].join(',') unless opt[:tags].nil?
+          opt[:sktags] = opt[:sktags].join(',') unless opt[:sktags].nil?
+          opt
+        end
+      end
+    end
+  end
+end

data/lib/elastic-backup-2s3/snapshot.rb

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+module ElasticBackup
+  module Snapshot
+    class << self
+      def s3
+        @s3 ||= Aws::S3::Resource.new
+      end
+
+      def opt
+        @opt
+      end
+
+      # We only need to set this once anyway
+      def set_opts(options)
+        @opt ||= options
+      end
+
+      # Take the s3 url and break it down to
+      # its components [BUCKET, PATH, SNAPSHOT]
+      def s3url_splice(surl)
+        protocol, b, snapshot = surl.split(':')
+        empty, empty, bucket, path = b.split('/', 4)
+        raise "Protocol must be 's3' in #{surl}" unless protocol.downcase == 's3'
+        [bucket, path, snapshot]
+      end
+ 
+      def esurl(suri: 'localhost', port: 9200)
+        _suri, _port = suri.split(':') 
+        "http://#{_suri}:#{_port || port}"
+      end
+
+      # The first time this is called must have a 
+      # valid esurl!!!!
+      def elastic(esurl = nil)
+        @elastic ||= ESClient.new esurl
+      end
+
+      # For now, this will overwrite the repo if it is
+      # there already.
+      def set_repository s3url
+        bucket, base_path, _ignore = s3url_splice s3url
+        cmd = { repository: opt[:repo], 
+          body: {
+            type: 's3',
+            settings:  { 
+              bucket: bucket,
+              base_path: base_path
+            }}}
+        ap cmd if opt[:dryrun] || (opt[:verbose] >= 2)
+        unless opt[:dryrun]
+          ret = MultiJson.load elastic.snapshot.create_repository(cmd)
+          ap ret unless opt[:verbose] < 2
+          raise "Error #{ret['status']} detected: #{ret['error']}" unless ret['error'].nil?
+        end
+      end
+
+      def initiate_snapshot s3url
+        _ignore, _ignore, snapname = s3url_splice s3url
+        raise "Must specify :SNAPSHOTNAME at the end of your S3URL #{s3url}" if snapname.nil?
+
+        cmd = { 
+          repository: opt[:repo], 
+          snapshot: snapname,
+          wait_for_completion: opt[:wait],
+          master_timeout: opt[:timeout],
+          body: {}}
+        cmd[:body][:indices] = opt[:indices].join(',') unless opt[:indices].nil?
+        ap cmd if opt[:dryrun] || (opt[:verbose] >= 2)
+        unless opt[:dryrun]
+          ret = MultiJson.load elastic.snapshot.create(cmd)
+          ap ret unless opt[:verbose] < 2
+          raise "Error #{ret['status']} detected: #{ret['error']}" unless ret['error'].nil?
+        end
+      end
+
+      def initiate_restore s3url
+        _ignore, _ignore, snapname = s3url_splice s3url
+        raise "Must specify :SNAPSHOTNAME at the end of your S3URL #{s3url}" if snapname.nil?
+
+        cmd = { 
+          repository: opt[:repo], 
+          snapshot: snapname,
+          wait_for_completion: opt[:wait],
+          master_timeout: opt[:timeout]
+        }
+        ap cmd if opt[:dryrun] || (opt[:verbose] >= 2)
+        unless opt[:dryrun]
+          ret = MultiJson.load elastic.snapshot.restore(cmd)
+          ap ret unless opt[:verbose] < 2
+          raise "Error #{ret['status']} detected: #{ret['error']}" unless ret['error'].nil?
+        end
+      end
+
+      # Do a snapshot of an elasticsearch cluster
+      def snapshot esurl, s3url, options
+        elastic esurl
+        set_opts(options)
+        set_repository s3url
+        initiate_snapshot s3url
+      end
+      
+      def restore s3url, esurl, options
+        elastic esurl
+        set_opts(options)
+        set_repository s3url
+        initiate_restore s3url
+      end
+    end
+
+    class ESClient
+      include Elasticsearch::API
+      attr :conn
+
+      def initialize(esurl)
+        @conn = ::Faraday::Connection.new url: esurl
+      end
+
+      def perform_request(method, path, params, body)
+        conn.run_request method.downcase.to_sym, path,
+        ( body ? MultiJson.dump(body) : nil ),
+        {'Content-Type' => 'application/json'}
+      end
+    end    
+  end
+end

data/lib/elastic-backup-2s3.rb

@@ -0,0 +1,18 @@
+require 'thor'
+require 'semver'
+require 'pp'
+require 'open3'
+require 'colorize'
+require 'elasticsearch'
+require 'aws-sdk'
+require 'multi_json'
+require 'faraday'
+require 'elasticsearch/api'
+require 'awesome_print'
+require 'text-table'
+require 'active_support/hash_with_indifferent_access'
+
+require_relative 'elastic-backup-2s3/snapshot'
+
+module ElasticBackup
+end

data/spec/elastic-backup-2s3_spec.rb

@@ -0,0 +1,37 @@
+require_relative 'spec_helper'
+
+include ElasticBackup
+
+S3URL = "s3://mybucket/my/path:my_snapshot"
+S3URL2 = "s3://mybucket/my/path"
+
+describe Snapshot do
+  it "parses the S3 Url" do
+    bucket, path, snapshot = Snapshot.s3url_splice(S3URL)
+    expect(bucket).to eq("mybucket")
+    expect(path).to eq("my/path")
+    expect(snapshot).to eq("my_snapshot")
+  end
+
+  it "parses the S3 Url without the snapshot" do
+    bucket, path, snapshot = Snapshot.s3url_splice(S3URL2)
+    expect(bucket).to eq("mybucket")
+    expect(path).to eq("my/path")
+    expect(snapshot).to eq(nil)
+  end
+
+  it "has a valid s3 resource" do
+    expect(s3 = Snapshot.s3).to_not be nil
+  end
+
+  it "has a valid Elastic resource" do
+    expect(Snapshot.elastic).to_not be nil
+  end
+
+  it "esurl defaults nicely" do
+    expect(Snapshot.esurl).to  eq('http://localhost:9200')
+    expect(Snapshot.esurl port: 5900).to  eq('http://localhost:5900')
+    expect(Snapshot.esurl suri: 'localhost:5900').to  eq('http://localhost:5900')
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,29 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_adapter 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'rspec'
+require 'elastic-backup-2s3'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+
+end

metadata

@@ -0,0 +1,350 @@
+--- !ruby/object:Gem::Specification
+name: elastic-backup-2s3
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Fred Mitchell
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-04-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: semver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: text-table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: juwelier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: pry-doc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-remote
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-rescue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: pry-stack_explorer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: "\n  I simply want to be able to control the backup and\n  restoration
+  of the Elasticsearch cluster to S3\n  without any fuss or having to dilly around
+  with\n  curl -XPUTS and friends.\n\n  So here it is, and I make no apologies about
+  not \n  supporting the \"shared volume\" option of Elasticsearch\n  snapshots. I
+  may add support at a later date. Or\n  feel free to add it and do a pull request.\n\n
+  \ There are many features I wish to add to this, and\n  if you have any suggestions,
+  please feel free to send\n  them my way!"
+email: fred.mitchell@gmx.de
+executables:
+- es-snapshot
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.org
+files:
+- ".document"
+- ".rspec"
+- ".ruby-version"
+- ".semver"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.org
+- Rakefile
+- bin/es-snapshot
+- elastic-backup-2s3.gemspec
+- lib/elastic-backup-2s3.rb
+- lib/elastic-backup-2s3/cli.rb
+- lib/elastic-backup-2s3/cli/delete.rb
+- lib/elastic-backup-2s3/cli/list.rb
+- lib/elastic-backup-2s3/snapshot.rb
+- spec/elastic-backup-2s3_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/flajann2/elastic-backup-2s3
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Elasticsearch to AWS S3 Backup, Snapshotting, and Restore Tool
+test_files: []