logstash-input-elasticsearch 0.1.0 → 0.1.1

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ZGM3OGYyOWE4NTc1ODgxNGFmOTBkNzI1MWExYmMyOWExZTZlZjQyNg==
-  data.tar.gz: !binary |-
-    YjczYjE1ZWRhMjg4NmMzNjg0M2EwZTExMjEyNDI4YTk4MzU1MmZhYw==
+SHA1:
+  metadata.gz: 1bc979eb2c5fe6019bd02366835ed40451cf3b6d
+  data.tar.gz: 60dfe9d9f4445652dbdf03145690ddd492c98d08
 SHA512:
-  metadata.gz: !binary |-
-    NDNiNmFhMDRjMjEwMjAyMDg3NmE3ZmM2M2I4MzdiMThhMDI0Mjk0YmJiYWY1
-    NzFjNjM1YTcyOWYyY2NkMzkwMTVhOGQ5N2NmNTZjZGVmODQ3YTYwN2VmMDAy
-    MGM5NTViNThhNzJmNWU4NWYyOTg0MTk2NjQwOWFkYzY5Zjk2YTM=
-  data.tar.gz: !binary |-
-    NGYxYjNlY2I4NjJiNWU1YTkyZjYwYmNjYzFhNGE3ZDI1MWY3N2VlMTFjNjBm
-    OGEzNWUyZjQ2Yjc0ZmJmNGU0NTVmYTE2Y2JhMTlhMDgwY2UzZjBhZWFkZTI4
-    ZmYwNTk4MmNiMjRhYWVlMzEyNzEyMDI0MjYyMTM3OTQwZDY3MWE=
+  metadata.gz: 4c2f801e6ca9e6acf2a8b4ead31fa9ca0f3251afe52f989cf15792893118aaf410dca883761e1c195eeb5ff3a50970c7a8b86a7a24057d4055422ed921c60e76
+  data.tar.gz: 141701089ffcfbfd945d21a30fd5c45377eb683a8ac5b263d639ec6e7a0fcb7b022f4c9e0a951a56b11d170fcd84d2b279694acd03d4f551eba2160e172a6297

data/Gemfile

@@ -1,4 +1,3 @@
-source 'http://rubygems.org'
-gem 'rake'
-gem 'gem_publisher'
-gem 'archive-tar-minitar'
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -4,3 +4,4 @@ task :default do
   system("rake -T")
 end
 
+require "logstash/devutils/rake"

data/lib/logstash/inputs/elasticsearch.rb

@@ -1,35 +1,34 @@
 # encoding: utf-8
 require "logstash/inputs/base"
 require "logstash/namespace"
-require "logstash/util/socket_peer"
-require "logstash/json"
+require "base64"
 
 # Read from an Elasticsearch cluster, based on search query results.
 # This is useful for replaying test logs, reindexing, etc.
 #
 # Example:
-#
+# [source,ruby]
 #     input {
 #       # Read all documents from Elasticsearch matching the given query
 #       elasticsearch {
 #         host => "localhost"
-#         query => "ERROR"
+#         query => '{ "query": { "match": { "statuscode": 200 } } }'
 #       }
 #     }
 #
 # This would create an Elasticsearch query with the following format:
+# [source,json]
+#     http://localhost:9200/logstash-*/_search?q='{ "query": { "match": { "statuscode": 200 } } }'&scroll=1m&size=1000
 #
-#     http://localhost:9200/logstash-*/_search?q=ERROR&scroll=1m&size=1000
-#
-# * TODO(sissel): Option to keep the index, type, and doc id so we can do reindexing?
+# TODO(sissel): Option to keep the index, type, and doc id so we can do reindexing?
 class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   config_name "elasticsearch"
   milestone 1
 
   default :codec, "json"
 
-  # The IP address or hostname of your Elasticsearch server.
-  config :host, :validate => :string, :required => true
+  # List of elasticsearch hosts to use for querying.
+  config :hosts, :validate => :array
 
   # The HTTP port of your Elasticsearch server's REST interface.
   config :port, :validate => :number, :default => 9200
@@ -52,83 +51,74 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # round trip (i.e. between the previous scan scroll request, to the next).
   config :scroll, :validate => :string, :default => "1m"
 
+  # Basic Auth - username
+  config :user, :validate => :string
+
+  # Basic Auth - password
+  config :password, :validate => :password
+
+  # SSL
+  config :ssl, :validate => :boolean, :default => false
+
+  # SSL Certificate Authority file
+  config :ca_file, :validate => :path
+
   public
   def register
-    require "ftw"
-    @agent = FTW::Agent.new
+    require "elasticsearch"
 
-    params = {
-      "q" => @query,
-      "scroll" => @scroll,
-      "size" => "#{@size}",
+    @options = {
+      index: @index,
+      body: @query,
+      scroll: @scroll,
+      size: @size
     }
-    params['search_type'] = "scan" if @scan
 
-    @search_url = "http://#{@host}:#{@port}/#{@index}/_search?#{encode(params)}"
-    @scroll_url = "http://#{@host}:#{@port}/_search/scroll?#{encode({"scroll" => @scroll})}"
-  end # def register
+    @options[:search_type] = 'scan' if @scan
 
-  private
-  def encode(hash)
-    return hash.collect do |key, value|
-      CGI.escape(key) + "=" + CGI.escape(value)
-    end.join("&")
-  end # def encode
+    transport_options = {}
 
-  private
-  def execute_search_request
-    response = @agent.get!(@search_url)
-    json = ""
-    response.read_body { |c| json << c }
-    json
-  end
+    if @user && @password
+      token = Base64.strict_encode64("#{@user}:#{@password.value}")
+      transport_options[:headers] = { Authorization: "Basic #{token}" }
+    end
 
-  private
-  def execute_scroll_request(scroll_id)
-    response = @agent.post!(@scroll_url, :body => scroll_id)
-    json = ""
-    response.read_body { |c| json << c }
-    json
-  end
+    hosts = if @ssl then
+      @hosts.map {|h| { host: h, scheme: 'https' } }
+    else
+      @hosts
+    end
+
+    if @ssl && @ca_file
+      transport_options[:ssl] = { ca_file: @ca_file }
+    end
+
+    @client = Elasticsearch::Client.new hosts: hosts, transport_options: transport_options
+
+  end # def register
 
   public
   def run(output_queue)
-    result = LogStash::Json.load(execute_search_request)
-    scroll_id = result["_scroll_id"]
 
-    # When using the search_type=scan we don't get an initial result set.
-    # So we do it here.
+    # get first wave of data
+    r = @client.search @options
+
+    # since 'scan' doesn't return data on the search call, do an extra scroll
     if @scan
-      result = LogStash::Json.load(execute_scroll_request(scroll_id))
+      r = scroll_request(r['_scroll_id'])
     end
 
-    loop do
-      break if result.nil?
-      hits = result["hits"]["hits"]
-      break if hits.empty?
-
-      hits.each do |hit|
-        # Hack to make codecs work
-        @codec.decode(LogStash::Json.dump(hit["_source"])) do |event|
-          decorate(event)
-          output_queue << event
-        end
-      end
-
-      # Get the scroll id from the previous result set and use it for getting the next data set
-      scroll_id = result["_scroll_id"]
-
-      # Fetch the next result set
-      result = LogStash::Json.load(execute_scroll_request(scroll_id))
-
-      if result["error"]
-        @logger.warn(result["error"], :request => scroll_url)
-        # TODO(sissel): raise an error instead of breaking
-        break
+    while r['hits']['hits'].any? do
+      r['hits']['hits'].each do |event|
+        decorate(event)
+        output_queue << event
       end
-
+      r = scroll_request(r['_scroll_id'])
     end
-  rescue LogStash::ShutdownSignal
-    # Do nothing, let us quit.
   end # def run
+
+  private
+  def scroll_request scroll_id
+    @client.scroll(body: scroll_id, scroll: @scroll)
+  end
 end # class LogStash::Inputs::Elasticsearch

data/logstash-input-elasticsearch.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '0.1.0'
+  s.version         = '0.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read from an Elasticsearch cluster, based on search query results"
-  s.description     = "Read from an Elasticsearch cluster, based on search query results"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elasticsearch"]
-  s.email           = 'richard.pijnenburg@elasticsearch.com'
-  s.homepage        = "http://logstash.net/"
+  s.email           = 'info@elasticsearch.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
   s.require_paths = ["lib"]
 
   # Files
@@ -17,13 +17,15 @@ Gem::Specification.new do |s|
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "group" => "input" }
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
   s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
 
-  s.add_runtime_dependency 'ftw', ['~> 0.0.39']
+  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.6', '~> 1.0']
+
   s.add_runtime_dependency 'logstash-codec-json'
 
+  s.add_development_dependency 'logstash-devutils'
 end
 

data/spec/inputs/elasticsearch_spec.rb

@@ -1,60 +1,115 @@
-require "spec_helper"
+require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/elasticsearch"
+require "elasticsearch"
 
 describe "inputs/elasticsearch" do
-  
-
-  search_response = <<-RESPONSE
-    {
-      "_scroll_id":"xxx",
-      "took":5,
-      "timed_out":false,
-      "_shards":{"total":15,"successful":15,"failed":0},
-      "hits":{
-        "total":1000050,
-        "max_score":1.0,
-        "hits":[
-          {
-            "_index":"logstash2",
-            "_type":"logs",
-            "_id":"AmaqL7VuSWKF-F6N_Gz72g",
-            "_score":1.0,
-            "_source" : {
-              "message":"foobar",
-              "@version":"1",
-              "@timestamp":"2014-05-19T21:08:39.000Z",
-              "host":"colin-mbp13r"
-            }
-          }
-        ]
+
+  it "should retrieve json event from elasticseach" do
+
+    config = %q[
+      input {
+        elasticsearch {
+          hosts => ["node01"]
+          scan => false
+          query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+        }
       }
-    }
-  RESPONSE
+    ]
 
-  scroll_response = <<-RESPONSE
-    {
-      "hits":{
-        "hits":[]
+    response = {
+      "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
+      "took" => 27,
+      "timed_out" => false,
+      "_shards" => {
+        "total" => 169,
+        "successful" => 169,
+        "failed" => 0
+      },
+      "hits" => {
+        "total" => 1,
+        "max_score" => 1.0,
+        "hits" => [ {
+          "_index" => "logstash-2014.10.12",
+          "_type" => "logs",
+          "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
+          "_score" => 1.0,
+          "fields" => {"message" => ["ohayo"] }
+        } ]
       }
     }
-  RESPONSE
 
-  config <<-CONFIG
-    input {
-      elasticsearch {
-        host => "localhost"
-        scan => false
+    scroll_reponse = {
+      "_scroll_id" => "r453Wc1jh0caLJhSDg",
+      "hits" => { "hits" => [] }
+    }
+
+    client = Elasticsearch::Client.new
+    expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+    expect(client).to receive(:search).with(any_args).and_return(response)
+    expect(client).to receive(:scroll).with({:body=>"cXVlcnlUaGVuRmV0Y2g", :scroll=>"1m"}).and_return(scroll_reponse)
+
+    pipeline = LogStash::Pipeline.new(config)
+    queue = Queue.new
+    pipeline.instance_eval do
+      @output_func = lambda { |event| queue << event }
+    end
+    pipeline_thread = Thread.new { pipeline.run }
+    event = queue.pop
+
+    insist { event["fields"]["message"] } == [ "ohayo" ]
+
+    pipeline_thread.join
+  end
+
+  it "should retrieve json event from elasticseach with scan" do
+
+    config = %q[
+      input {
+        elasticsearch {
+          hosts => ["node01"]
+          scan => true
+          query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+        }
       }
+    ]
+
+    scan_response = {
+      "_scroll_id" => "DcrY3G1xff6SB",
     }
-  CONFIG
 
-  it "should retrieve json event from elasticseach" do
-    # I somewhat duplicated our "input" rspec extension because I needed to add mocks for the the actual ES calls
-    # and rspec expectations need to be in "it" statement but the "input" extension defines the "it"
-    # TODO(colin) see how we can improve our rspec extension to better integrate in these scenarios
+    scroll_responses = [
+      {
+        "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
+        "took" => 27,
+        "timed_out" => false,
+        "_shards" => {
+          "total" => 169,
+          "successful" => 169,
+          "failed" => 0
+        },
+        "hits" => {
+          "total" => 1,
+          "max_score" => 1.0,
+          "hits" => [ {
+            "_index" => "logstash-2014.10.12",
+            "_type" => "logs",
+            "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
+            "_score" => 1.0,
+            "fields" => {"message" => ["ohayo"] }
+          } ]
+        }
+      },
+      {
+        "_scroll_id" => "r453Wc1jh0caLJhSDg",
+        "hits" => { "hits" => [] }
+      }
+    ]
 
-    expect_any_instance_of(LogStash::Inputs::Elasticsearch).to receive(:execute_search_request).and_return(search_response)
-    expect_any_instance_of(LogStash::Inputs::Elasticsearch).to receive(:execute_scroll_request).with(any_args).and_return(scroll_response)
+    client = Elasticsearch::Client.new
+    expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+    expect(client).to receive(:search).with(any_args).and_return(scan_response)
+    expect(client).to receive(:scroll).with({:body=>"DcrY3G1xff6SB", :scroll=>"1m"}).and_return(scroll_responses.first)
+    expect(client).to receive(:scroll).with({:body=>"cXVlcnlUaGVuRmV0Y2g", :scroll=>"1m"}).and_return(scroll_responses.last)
 
     pipeline = LogStash::Pipeline.new(config)
     queue = Queue.new
@@ -64,17 +119,8 @@ describe "inputs/elasticsearch" do
     pipeline_thread = Thread.new { pipeline.run }
     event = queue.pop
 
-    insist { event["message"] } == "foobar"
-
-    # do not call pipeline.shutdown here, as it will stop the plugin execution randomly
-    # and maybe kill input before calling execute_scroll_request.
-    # TODO(colin) we should rework the pipeliene shutdown to allow a soft/clean shutdown mecanism,
-    # using a shutdown event which can be fed into each plugin queue and when the plugin sees it
-    # exits after completing its processing.
-    #
-    # pipeline.shutdown
-    #
-    # instead, since our scroll_response will terminate the plugin, we can just join the pipeline thread
+    insist { event["fields"]["message"] } == [ "ohayo" ]
+
     pipeline_thread.join
   end
 end

metadata

@@ -1,101 +1,120 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Elasticsearch
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-11-03 00:00:00.000000000 Z
+date: 2014-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash
-  requirement: !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.0
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.4.0
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: ftw
-  requirement: !ruby/object:Gem::Requirement
+  name: elasticsearch
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.6
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.39
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+        version: '1.0'
+  requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.6
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.0.39
+        version: '1.0'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: logstash-codec-json
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
   prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Read from an Elasticsearch cluster, based on search query results
-email: richard.pijnenburg@elasticsearch.com
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: info@elasticsearch.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
+- LICENSE
 - Rakefile
 - lib/logstash/inputs/elasticsearch.rb
 - logstash-input-elasticsearch.gemspec
-- rakelib/publish.rake
-- rakelib/vendor.rake
 - spec/inputs/elasticsearch_spec.rb
-homepage: http://logstash.net/
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
 metadata:
   logstash_plugin: 'true'
-  group: input
-post_install_message: 
+  logstash_group: input
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.1
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.2.2
+signing_key:
 specification_version: 4
 summary: Read from an Elasticsearch cluster, based on search query results
 test_files:

data/rakelib/publish.rake

@@ -1,9 +0,0 @@
-require "gem_publisher"
-
-desc "Publish gem to RubyGems.org"
-task :publish_gem do |t|
-  gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
-  gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
-  puts "Published #{gem}" if gem
-end
-

data/rakelib/vendor.rake

@@ -1,169 +0,0 @@
-require "net/http"
-require "uri"
-require "digest/sha1"
-
-def vendor(*args)
-  return File.join("vendor", *args)
-end
-
-directory "vendor/" => ["vendor"] do |task, args|
-  mkdir task.name
-end
-
-def fetch(url, sha1, output)
-
-  puts "Downloading #{url}"
-  actual_sha1 = download(url, output)
-
-  if actual_sha1 != sha1
-    fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
-  end
-end # def fetch
-
-def file_fetch(url, sha1)
-  filename = File.basename( URI(url).path )
-  output = "vendor/#{filename}"
-  task output => [ "vendor/" ] do
-    begin
-      actual_sha1 = file_sha1(output)
-      if actual_sha1 != sha1
-        fetch(url, sha1, output)
-      end
-    rescue Errno::ENOENT
-      fetch(url, sha1, output)
-    end
-  end.invoke
-
-  return output
-end
-
-def file_sha1(path)
-  digest = Digest::SHA1.new
-  fd = File.new(path, "r")
-  while true
-    begin
-      digest << fd.sysread(16384)
-    rescue EOFError
-      break
-    end
-  end
-  return digest.hexdigest
-ensure
-  fd.close if fd
-end
-
-def download(url, output)
-  uri = URI(url)
-  digest = Digest::SHA1.new
-  tmp = "#{output}.tmp"
-  Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
-    request = Net::HTTP::Get.new(uri.path)
-    http.request(request) do |response|
-      fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
-      size = (response["content-length"].to_i || -1).to_f
-      count = 0
-      File.open(tmp, "w") do |fd|
-        response.read_body do |chunk|
-          fd.write(chunk)
-          digest << chunk
-          if size > 0 && $stdout.tty?
-            count += chunk.bytesize
-            $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
-          end
-        end
-      end
-      $stdout.write("\r      \r") if $stdout.tty?
-    end
-  end
-
-  File.rename(tmp, output)
-
-  return digest.hexdigest
-rescue SocketError => e
-  puts "Failure while downloading #{url}: #{e}"
-  raise
-ensure
-  File.unlink(tmp) if File.exist?(tmp)
-end # def download
-
-def untar(tarball, &block)
-  require "archive/tar/minitar"
-  tgz = Zlib::GzipReader.new(File.open(tarball))
-  # Pull out typesdb
-  tar = Archive::Tar::Minitar::Input.open(tgz)
-  tar.each do |entry|
-    path = block.call(entry)
-    next if path.nil?
-    parent = File.dirname(path)
-    
-    mkdir_p parent unless File.directory?(parent)
-
-    # Skip this file if the output file is the same size
-    if entry.directory?
-      mkdir path unless File.directory?(path)
-    else
-      entry_mode = entry.instance_eval { @mode } & 0777
-      if File.exists?(path)
-        stat = File.stat(path)
-        # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
-        # expose headers in the entry.
-        entry_size = entry.instance_eval { @size }
-        # If file sizes are same, skip writing.
-        next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
-      end
-      puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
-      File.open(path, "w") do |fd|
-        # eof? check lets us skip empty files. Necessary because the API provided by
-        # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
-        # IO object. Something about empty files in this EntryStream causes
-        # IO.copy_stream to throw "can't convert nil into String" on JRuby
-        # TODO(sissel): File a bug about this.
-        while !entry.eof?
-          chunk = entry.read(16384)
-          fd.write(chunk)
-        end
-          #IO.copy_stream(entry, fd)
-      end
-      File.chmod(entry_mode, path)
-    end
-  end
-  tar.close
-  File.unlink(tarball) if File.file?(tarball)
-end # def untar
-
-def ungz(file)
-
-  outpath = file.gsub('.gz', '')
-  tgz = Zlib::GzipReader.new(File.open(file))
-  begin
-    File.open(outpath, "w") do |out|
-      IO::copy_stream(tgz, out)
-    end
-    File.unlink(file)
-  rescue
-    File.unlink(outpath) if File.file?(outpath)
-   raise
-  end
-  tgz.close
-end
-
-desc "Process any vendor files required for this plugin"
-task "vendor" do |task, args|
-
-  @files.each do |file| 
-    download = file_fetch(file['url'], file['sha1'])
-    if download =~ /.tar.gz/
-      prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
-      untar(download) do |entry|
-        if !file['files'].nil?
-          next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
-          out = entry.full_name.split("/").last
-        end
-        File.join('vendor', out)
-      end
-    elsif download =~ /.gz/
-      ungz(download)
-    end
-  end
-
-end