wonderdog 0.1.1 → 0.2.0

data/.gitignore

@@ -49,3 +49,5 @@ TAGS*
 a.out
 doc
 nohup.out
+test/flight_count_20130405
+test/flight_count_20130405_unpacked

data/.idea/encodings.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding" useUTFGuessing="true" native2AsciiForPropertiesFiles="false" />
+</project>
+

data/.idea/misc.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="rbenv: 1.9.3-p392" project-jdk-type="RUBY_SDK" />
+</project>
+

data/.idea/modules.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/wonderdog.iml" filepath="$PROJECT_DIR$/.idea/wonderdog.iml" />
+    </modules>
+  </component>
+</project>
+

data/.idea/scopes/scope_settings.xml

@@ -0,0 +1,5 @@
+<component name="DependencyValidationManager">
+  <state>
+    <option name="SKIP_IMPORT_STATEMENTS" value="false" />
+  </state>
+</component>

data/.idea/vcs.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
+

data/.idea/wonderdog.iml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="RUBY_MODULE" version="4">
+  <component name="FacetManager">
+    <facet type="gem" name="Gem">
+      <configuration>
+        <option name="GEM_APP_ROOT_PATH" value="$MODULE_DIR$" />
+        <option name="GEM_APP_TEST_PATH" value="$MODULE_DIR$/test" />
+        <option name="GEM_APP_LIB_PATH" value="$MODULE_DIR$/lib" />
+      </configuration>
+    </facet>
+  </component>
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/test" isTestSource="true" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" scope="PROVIDED" name="bundler (v1.3.5, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="configliere (v0.4.18, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="diff-lcs (v1.2.4, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="eventmachine (v1.0.3, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="forgery (v0.5.0, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="gorillib (v0.5.0, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="highline (v1.6.18, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="json (v1.7.7, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="log4r (v1.1.10, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="multi_json (v1.7.2, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rake (v0.9.6, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="redcarpet (v2.2.2, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec (v2.13.0, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-core (v2.13.1, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-expectations (v2.13.0, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rspec-mocks (v2.13.1, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="uuidtools (v2.1.3, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="vayacondios-client (v0.1.12, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="wukong (v3.0.1, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="wukong-hadoop (v0.1.1, rbenv: 1.9.3-p392) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="yard (v0.8.6.1, rbenv: 1.9.3-p392) [gem]" level="application" />
+  </component>
+</module>
+

data/Gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source 'https://rubygems.org'
 
 gemspec
 

data/bin/estool

@@ -23,6 +23,9 @@ Commands include:
     optimize                         Optimizes the specified INDEX to (-s) number of segments
     snapshot                         Snapshots the specified INDEX to the gateway
     segments                         Returns the segment information. Requires ElasticSearch v
+    aliases                          Returns a list of Index/Alias pairs
+    ialiases                         Returns a list of Alias/Index pairs
+    count                            The number of documents in an index
     mapping
     set_replication
     search
@@ -89,7 +92,7 @@ class ESTool
   
   def status()     shell_response(File.join(options.index, "_status?")) ; end
 
-  def list()       status["indices"].keys ; end
+  def list()       status["indices"].keys.sort ; end
 
   def health()     shell_response("_cluster/health?") ; end
 
@@ -111,6 +114,24 @@ class ESTool
   def segments()   shell_response(File.join(options.index, "_segments")) ; end
 
   def mapping()    shell_response(File.join(options.index, "_mapping")) ; end
+  
+  def aliases()    shell_response('_aliases?').sort.inject({}){ |hsh, (index, info)| hsh[index] = info['aliases'].keys ; hsh } ; end
+  
+  def ialiases() 
+    inverse = Hash.new{ |hsh, key| hsh[key] = [] }
+    aliases.each{ |idx, als| als.each{ |a| inverse[a] << idx } }
+    inverse
+  end
+
+  # estool status -r | ruby -rjson -e 'puts JSON.parse($stdin.read)["indices"]["item_count_legacy"]["docs"]'
+  # {"num_docs"=>187749, "max_doc"=>187749, "deleted_docs"=>0}
+  def count()
+    if options.index == '_all'
+      status['indices'].inject({}){ |hsh, (index, info)| hsh[index] = info['docs']['num_docs'] ; hsh }
+    else
+      { options.index => shell_response(File.join(options.index, '_count'))['count'] }
+    end
+  end
 
   # curl -s -XPUT http://host:port/index/_settings -d '{"index":{"number_of_replicas":num}}'
   def set_replication() { "error" => "method not yet implemented" }; end

data/bin/squirrel.rb

@@ -0,0 +1,108 @@
+#! /usr/bin/ruby
+
+require "configliere"
+require_relative "../squirrel/squirrel.rb"
+
+doc = <<DOC
+This is the uber script the arguements you give it decide what happens
+squirrel => Standard Query Ultracrepidate Iamatology Ruby Resource for Elasticsearch Labarum ##
+  example commands:
+    clear all caches
+      ruby squirrel.rb cache --host=localhost --port=9200 --clear_all_cache=true
+    run slow log queries
+      ruby squirrel.rb replay --host=localhost --port=9200 --preference=true,_primary_first --execute_slow_queries=/var/log/elasticsearch/padraig.log
+    get backup an index aka generate a dumpfile
+      ruby squirrel.rb backup --host=localhost --port=9200 --output_dir="." --dump_index=flight_count_20130405 --batch_size=100 --dump_mapping=flight_count_20130405_mapping.json
+    get the cardinality of a dumpfile(card_file)
+      ruby squirrel.rb cardinality --host=localhost --port=9200 --output_dir="." --card_file=flight_count_20130405 --cardinality=cnt,metric
+    restore an index from a dumpfile
+      ruby squirrel.rb restore --host=localhost --port=9200 --output_dir="." --restore_file=flight_count_20130405.gz --restore_index=flight_count_20130405 --restore_mapping=flight_count_20130405_mapping.json --batch_size=100
+    duplicate files in an index from a dumpfile(duplicate_file)
+      ruby squirrel.rb duplicate --host=localhost --port=9200 --output_dir="." --duplicate_file=flight_count_20130405.gz --duplicate_index=eight_flight_count_20130405 --duplicate_mapping=flight_count_20130405_mapping.json --batch_size=100
+    add warmer
+      ruby squirrel.rb warmer --host=localhost --port=9200 --output_dir="." --new_warmers_name=polite_warmer --warmers_index=flight_count_20130408 --create_warmer='{"sort" : ["_state", "flight_id","metric", "tb_h", "feature", "seconds", "base_feature", "metric_feature", "cnt", "_score"],"query":{"match_all":{}}}'
+    remove warmer
+      ruby squirrel.rb warmer --host=localhost --port=9200 --output_dir="." --remove_warmer=polite_warmer --warmers_index=flight_count_20130408
+    disable warmers
+      ruby squirrel.rb warmer --host=localhost --port=9200 --output_dir="." --warmers=false --warmers_index=flight_count_20130405
+    enable warmers
+      ruby squirrel.rb warmer --host=localhost --port=9200 --output_dir="." --warmers=false --warmers_index=flight_count_20130405
+    remove warmer
+      ruby squirrel.rb warmer --host=localhost --port=9200 --output_dir="." --remove_warmer=polite_warmer --warmers_index=flight_count_20130405
+    change index settings
+      ruby squirrel.rb index_settings --host=localhost --port=9200 --output_dir="." --settings_index=flight_count_20130405 --es_index_settings=refresh_interval,refresh_interval --es_index_settings_values=-1,0
+DOC
+
+Settings.use :commandline
+Settings.use :commands
+Settings.description = doc
+Settings.define_command :backup, :description => "Create a dump gzip file of an index" do |cmd|
+  cmd.define :output_dir,                     :default => nil,  :description => 'Directory to put output, defaults to nil'
+  cmd.define :dump_file,                      :default => nil,  :description => 'The name of the dumpfile to use, default is nil'
+  cmd.define :dump_index,                     :default => nil,  :description => 'Index to use, default is nil'
+  cmd.define :query,                          :default => nil,  :description => 'Query to use in order to limit the data extracted from the index, default nil'
+  cmd.define :host,                           :default => nil,  :description => 'The host to connect to, defaults to nil'
+  cmd.define :port,         :type => Integer, :default => nil,  :description => 'The port to connect to on the host, defaults to nil'
+  cmd.define :dump_mapping,                   :default => nil,  :description => 'The file to put the json mapping in, defaults to nil'
+  cmd.define :batch_size,   :type => Integer, :default => nil,  :description => 'The number of lines to process at once, defaults to nil'
+end
+Settings.define_command :restore, :description => "Take the data from a dump gzip file and use it to populate an index" do |cmd|
+  cmd.define :output_dir,                         :default => nil,  :description => 'Directory to put output, defaults to nil'
+  cmd.define :restore_file,                       :default => nil,  :description => 'The name of the dumpfile to use, default is nil'
+  cmd.define :restore_index,                      :default => nil,  :description => 'Index to use, default is nil'
+  cmd.define :host,                               :default => nil,  :description => 'The host to connect to, defaults to nil'
+  cmd.define :port,             :type => Integer, :default => nil,  :description => 'The port to connect to on the host, defaults to nil'
+  cmd.define :restore_mapping,                    :default => nil,  :description => 'The mapping file to use when restoring an index, defaults to nil'
+  cmd.define :batch_size,       :type => Integer, :default => nil,  :description => 'The number of lines to process at once, defaults to nil'
+end
+Settings.define_command :duplicate, :description => "Take the data from a dump gzip file and add it into an index allowing documents to be duplicated" do |cmd|
+  cmd.define :output_dir,                           :default => nil,  :description => 'Directory to put output, defaults to nil'
+  cmd.define :duplicate_file,                       :default => nil,  :description => 'The name of the dumpfile to use, default is nil'
+  cmd.define :duplicate_index,                      :default => nil,  :description => 'Index to use, default is nil'
+  cmd.define :host,                                 :default => nil,  :description => 'The host to connect to, defaults to nil'
+  cmd.define :port,               :type => Integer, :default => nil,  :description => 'The port to connect to on the host, defaults to nil'
+  cmd.define :duplicated_mapping,                   :default => nil,  :description => 'The mapping file to use when restoring an index, defaults to nil'
+  cmd.define :batch_size,         :type => Integer, :default => nil,  :description => 'The number of lines to process at once, defaults to nil'
+end
+Settings.define_command  :cardinality, :description => "Count the number of unique valuse for the given field(s)" do |cmd|
+  cmd.define :output_dir,                   :default => nil,  :description => 'Directory to put output, defaults to nil'
+  cmd.define :cardinality, :type => Array,  :default => nil,  :description => 'Return the cardinality of the given fields, defaults to nil'
+  cmd.define :card_file,                    :default => nil,  :description => 'The dump file to grab info from when determining cardinality MUST NOT be compressed, defaults to nil'
+end
+Settings.define_command :warmer, :description => "Interact elasticsearch warmers" do |cmd|
+  cmd.define :warmers,                            :default => nil,  :description => 'Use warmers expected values true/false, defaults to nil'
+  cmd.define :warmers_index,                      :default => nil,  :description => 'The index to add the warmer too, remove it from or disable/enable it on, defaults to nil'
+  cmd.define :new_warmers_name,                   :default => nil,  :description => 'Name of warmer to create, defaults to nil'
+  cmd.define :create_warmer,                      :default => nil,  :description => 'Query to create warmer, defaults to nil'
+  cmd.define :remove_warmer,                      :default => nil,  :description => 'Name of warmer to remove, defaults to nil'
+  cmd.define :host,                               :default => nil,  :description => 'The host to connect to, defaults to nil'
+  cmd.define :port,             :type => Integer, :default => nil,  :description => 'The port to connect to on the host, defaults to nil'
+end
+Settings.define_command :cache, :description => "Interact with elasticsearch caches" do |cmd|
+  cmd.define :host,                                   :default => nil,  :description => 'The host to connect to, defaults to nil'
+  cmd.define :port,               :type => Integer,   :default => nil,  :description => 'The port to connect to on the host, defaults to nil'
+  cmd.define :clear_all_cache,    :type => :boolean,  :default => nil,  :description => 'Clear all caches expected true/false, defaults to nil'
+  cmd.define :clear_fielddata,    :type => :boolean,  :default => nil,  :description => 'Clear filter cache expected true/false, defaults to nil'
+  cmd.define :clear_filter_cache, :type => :boolean,  :default => nil,  :description => 'Clear filter cache expected true/false, defaults to nil'
+end
+Settings.define_command :replay, :description => "Replay slow log queries" do |cmd|
+  cmd.define :host,                                   :default => nil,                      :description => 'The host to connect to, defaults to nil'
+  cmd.define :port,                 :type => Integer, :default => nil,                      :description => 'The port to connect to on the host, defaults to nil'
+  cmd.define :execute_slow_queries,                   :default => nil,                      :description => 'Execute the slow log queries in the provided log file,ie --execute_slow_log=/var/log/elasticsearch/padraig.log, defaults to nil'
+  cmd.define :batch_size,           :type => Integer, :default => nil,                      :description => 'The number of lines to process at once, defaults to nil'
+  cmd.define :preference,           :type => Array,   :default => [true,"_primary_first"],  :description => 'Turn on/off preference and set it,ie --preference=true,_primary_first'
+  cmd.define :routing,              :type => Array,   :default => [false, ""],              :description => 'Add routing, ie --routing=true,flight_id'
+end
+Settings.define_command :index_settings, :description => "Change the index settings" do |cmd|
+  cmd.define :host,                                       :default => nil,  :description => 'The host to connect to, defaults to nil'
+  cmd.define :port,                     :type => Integer, :default => nil,  :description => 'The port to connect to on the host, defaults to nil'
+  cmd.define :settings_index,                             :default => nil,  :description => 'The index that the settings listed in index_settings will be changed for, defaults to nil'
+  cmd.define :es_index_settings,        :type => Array,   :default => nil,  :description => 'A comma deliminated list of elasticsearch index settings to be set for --settings_index, defaults to []'
+  cmd.define :es_index_settings_values, :type => Array,   :default => nil,  :description => 'A comma deliminated list of elasticsearch index settings values to be set for --settings_index, defaults to []'
+end
+Settings.resolve!
+
+
+Squirrel.new(Settings.command_name, Settings.to_hash).task_caller
+
+

data/lib/wonderdog.rb

@@ -26,6 +26,9 @@ module Wukong
       settings.define(:es_id_field,       :description => "If this field is present in a record, make an update request, otherwise make a create request", :wukong_hadoop => true)
       settings.define(:es_bulk_size,      :description => "Number of requests to batch locally before making a request to ElasticSearch", :type => Integer, :wukong_hadoop => true)
       settings.define(:es_query,          :description => "Query to use when defining input splits for ElasticSearch input",    :wukong_hadoop => true)
+      settings.define(:es_transport,      :description => "Use a transport client to an existing node instead of spinning up a new node", :default => true, type: :boolean, :wukong_hadoop => true)
+      settings.define(:es_transport_host, :description => "Host of existing node for transport client", default: 'localhost', :wukong_hadoop => true)
+      settings.define(:es_transport_port, :description => "Port of existing node for transport client", default: 9300, type: Integer, :wukong_hadoop => true)
     end
 
     # Boot Wonderdog with the given `settings` in the given `dir`.

data/lib/wonderdog/hadoop_invocation_override.rb

@@ -105,7 +105,10 @@ module Wukong
         
         super() + [].tap do |o|
           if (reads_from_elasticsearch? || writes_to_elasticsearch?)
-            o << java_opt('es.config', settings[:es_config])
+            o << java_opt('es.config',                    settings[:es_config])
+            o << java_opt('elasticsearch.transport',      settings[:es_transport])
+            o << java_opt('elasticsearch.transport.host', settings[:es_transport_host]) if settings[:es_transport] && settings[:es_transport_host]
+            o << java_opt('elasticsearch.transport.port', settings[:es_transport_port]) if settings[:es_transport] && settings[:es_transport_port]
           end
           
           if reads_from_elasticsearch?

data/lib/wonderdog/version.rb

@@ -1,3 +1,3 @@
 module Wonderdog
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

data/pom.xml

@@ -18,7 +18,7 @@
     <dependency>
       <groupId>org.elasticsearch</groupId>
       <artifactId>elasticsearch</artifactId>
-      <version>0.19.8</version>
+      <version>0.90.5</version>
     </dependency>
 
     <dependency>

data/spec/spec_helper.rb

@@ -21,7 +21,7 @@ RSpec.configure do |config|
 
   def hadoop_runner *args, &block
     runner(Wukong::Hadoop::HadoopRunner, 'wu-hadoop', *args) do
-      stub!(:execute_command!)
+      stub(:execute_command!)
       instance_eval(&block) if block_given?
     end
   end

data/spec/wonderdog/hadoop_invocation_override_spec.rb

@@ -8,7 +8,7 @@ describe Wukong::Elasticsearch::HadoopInvocationOverride do
   let(:es_complex) { hadoop_runner('regexp',  'count', input: 'es://the_index/the_map', output: 'es:///the_index/the_map', es_query: '{"hi": "there"}', es_request_size: 1000, es_index_field: 'ID', map_speculative: true, reduce_speculative: true) }
 
   context "passing necessary jars to Hadoop streaming" do
-    before  { Dir.stub!(:[]).and_return(["/lib/dir/elasticsearch.jar"], ["/lib/dir/wonderdog.jar"]) }
+    before  { Dir.stub(:[]).and_return(["/lib/dir/elasticsearch.jar"], ["/lib/dir/wonderdog.jar"]) }
     context "when not given explicit jars" do
       context "and not interacting with Elasticsearch" do
         it "doesn't add jars" do

data/squirrel/all_facets.rb

@@ -0,0 +1,95 @@
+require 'configliere'
+require 'json'
+
+Settings.use :commandline
+Settings.use :config_block
+Settings.define :es_index, default: nil
+Settings.define :fields, default: nil
+Settings.define :one_index, default: false, flag: 'o'
+Settings.define :composite_key, default: nil
+Settings.define :es_type
+Settings.finally do |settings|
+  settings.es_index ||= settings.one_index ? [settings.es_type, "cnt"].join : 'ad_activity'
+end
+Settings.resolve!
+
+puts "using es index #{Settings.es_index} and type #{Settings.es_type}"
+
+index = Settings.es_index 
+type = Settings.es_type 
+
+response = `curl localhost:9200/#{Settings.es_index}/_mapping/`
+puts JSON.parse(response)["#{index}"]["#{type}"]#["properties"]
+fields = JSON.parse(response)["#{index}"]["#{type}"]["properties"].keys#.select{|x| x.end_with?("_id")}
+#fields += %w[metric feature]# browser_ua]
+#fields = JSON.parse(response)["#{type}_legacy"][type]["properties"].keys.select{|x| x.end_with?("_id")}
+
+puts "got fields #{Settings.fields || fields}"
+
+# site_count => site_composite
+# placement_count => pl_composite
+# flight_count => metric_feature
+
+composite_key = case Settings.es_type
+                when "site_count" then "site_composite"
+                when "placement_count" then "pl_composite"
+                when "flight_count" then "metric_feature"
+                else nil
+                end
+
+doc = {
+  "query"=> {
+    "match_all"=> {}
+  },
+  "facets"=> {
+    # "ignz"=> {
+    #   "statistical"=> {
+    #     "field" => "cnt"
+    #   }
+    # },
+    "igna"=> {
+      "date_histogram"=> {
+        "field"=> "tb_h",
+	"interval" => "day"
+      }
+    },
+    "ignb"=> {
+      "terms"=> {
+        "fields"=> [Settings.fields || fields].flatten
+      }
+    },
+  }
+}
+
+fields.each do |field|
+  doc["facets"]["ign_#{field}"] = {
+    "terms_stats" => {
+      "key_field" => field,
+      "value_field" => "cnt"
+    }
+  }
+end
+
+if not composite_key.nil?
+  doc["facets"]["ignc"] = {
+    "terms_stats" => {
+      "key_field" => Settings.composite_key || composite_key,
+      "value_field" => "cnt"
+    }
+  }
+end
+
+puts "about to query with " + doc.to_s
+
+results = `curl localhost:9200/#{Settings.es_index}/#{Settings.es_type}/_search/?pretty=true -d '#{JSON.generate(doc)}'`
+term_hash = {}
+resultsLineArray = results.split( /\r?\n/ )
+copy_resultsLineArray = resultsLineArray - []
+copy_resultsLineArray.each_with_index do |line, index|
+  if line.include?("\"term\" : \"")
+	term_hash[line] = index
+  end
+end
+puts `curl localhost:9200/#{Settings.es_index}/#{Settings.es_type}/_search/?pretty=true -d '#{JSON.generate(doc)}'`
+puts "from command: \n curl localhost:9200/#{Settings.es_index}/#{Settings.es_type}/_search/ -d '#{JSON.generate(doc)}'"
+puts "number of terms: #{term_hash.keys.size()}"

data/squirrel/change_es_index_settings.rb

@@ -0,0 +1,19 @@
+class ChangeESIndexSettings
+  def initialize(options = {})
+    @host = options[:host]
+    @port = options[:port]
+    @index = options[:index]
+    @settings_and_values = options[:settings_and_values]
+  end
+
+  def change_setting(setting, value)
+    puts "changing setting #{setting} to value #{value}"
+    `curl -s -XPUT 'http://#{@host}:#{@port}/#{@index}/_settings?pretty=true' -d '{ "#{setting}":"#{value}" }'`
+  end
+
+  def run
+    @settings_and_values.each do |setting, value|
+      change_setting(setting, value)
+    end
+  end
+end

data/squirrel/clear_es_caches.rb

@@ -0,0 +1,30 @@
+class ClearESCaches
+  def initialize(options={})
+    @to_clear = options[:type]
+    @host = options[:host]
+    @port = options[:port]
+  end
+
+  def clear_all
+    `curl -s -XPOST 'http://#{@host}:#{@port}/_all/_cache/clear?field_data=true&filter=true&bloom=true' ; echo`
+  end
+
+  def clear_filter_cache
+    `curl -s -XPOST 'http://#{@host}:#{@port}/_all/_cache/clear?field_data=false&filter=true&bloom=true' ; echo`
+  end
+
+  def clear_fielddata
+    `curl -s -XPOST 'http://#{@host}:#{@port}/_all/_cache/clear?field_data=true&filter=false&bloom=true' ; echo`
+  end
+
+  def run
+    puts @to_clear
+    case command = @to_clear.to_sym
+      when :all then clear_all
+      when :filter then clear_filter_cache
+      when :fielddata then clear_fielddata
+      else abort "#{command} not recognized"
+    end
+  end
+
+end