fluent-plugin-s3-input 0.0.1 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: abd2bf6ecf2453a5d47cf8702ff3fe95f50dbf79
-  data.tar.gz: 9c73ec74af0ef99f899d0dec6e225ee4abd105dc
+  metadata.gz: 2cd64e1194e16d1821e054cf9dde5cd870ac1475
+  data.tar.gz: d91e30a771fa8afa9000b216b777c3bbc5f3c6cf
 SHA512:
-  metadata.gz: feed1255cafe0de56b2a0df87aa82093e2b07f87a5372f51512527ad4bb2db3c37177e017908aca09f041499d3e70232436101bb1b429c116a27d95ef94da3eb
-  data.tar.gz: 664811196401b7a22cb7078ad39de99a27f865dd7602d351327c595c991e7f54bbf923e7199b078871a5baf927f3646b42b61bb4e08c1204ace0f552f3c04def
+  metadata.gz: 7425c4192cf17c805da883e8cd97ee86956a2f00992503f9e4c74895da7ce039665a941fac78aab879fe56323c39c29bc202feb2e3f8b000fe9a41023398c32a
+  data.tar.gz: f825c630de89ba79c732f97a85a230ce62d604ba3c81e988e8fdf729a82f9a69288004de65035e0b27a2bb8f1ba6e7d53c704e8dccdf4f22b23e5694139a5e7d

data/Gemfile.lock

@@ -1,48 +1,50 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-s3-input (0.0.1)
+    fluent-plugin-s3-input (0.0.11)
       aws-sdk
       fluentd
       oj
+      rubyzip
 
 GEM
   remote: https://rubygems.org/
   specs:
-    aws-sdk (2.3.22)
-      aws-sdk-resources (= 2.3.22)
-    aws-sdk-core (2.3.22)
+    aws-sdk (2.9.21)
+      aws-sdk-resources (= 2.9.21)
+    aws-sdk-core (2.9.21)
+      aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.3.22)
-      aws-sdk-core (= 2.3.22)
-    cool.io (1.4.4)
-    fluentd (0.14.1)
-      cool.io (>= 1.4.3, < 2.0.0)
+    aws-sdk-resources (2.9.21)
+      aws-sdk-core (= 2.9.21)
+    aws-sigv4 (1.0.0)
+    cool.io (1.5.0)
+    fluentd (0.14.16)
+      cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
-      json (>= 1.4.3)
-      msgpack (>= 0.7.0)
-      serverengine (>= 1.6.4)
+      msgpack (>= 0.7.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.1.7)
-      tzinfo (>= 1.0.0)
-      tzinfo-data (>= 1.0.0)
+      strptime (~> 0.1.7)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
       yajl-ruby (~> 1.0)
     http_parser.rb (0.6.0)
-    jmespath (1.3.0)
-    json (2.0.1)
-    msgpack (1.0.0)
-    oj (2.17.1)
+    jmespath (1.3.1)
+    msgpack (1.1.0)
+    oj (3.0.9)
     rake (11.2.2)
-    serverengine (1.6.4)
+    rubyzip (1.2.1)
+    serverengine (2.0.5)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    strptime (0.1.8)
-    thread_safe (0.3.5)
-    tzinfo (1.2.2)
+    strptime (0.1.9)
+    thread_safe (0.3.6)
+    tzinfo (1.2.3)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2016.6)
+    tzinfo-data (1.2017.2)
       tzinfo (>= 1.0.0)
-    yajl-ruby (1.2.1)
+    yajl-ruby (1.3.0)
 
 PLATFORMS
   ruby
@@ -52,4 +54,4 @@ DEPENDENCIES
   rake
 
 BUNDLED WITH
-   1.12.5
+   1.14.6

data/README.md

@@ -45,6 +45,7 @@ S3 Event Example Intake
         # this plugin!
 	<match sqs.s3.event>
 	  type s3_input
+          merge_record no
 	  s3_bucket_key s3_bucket
 	  s3_object_key_key s3_object
           uncompress gzip
@@ -57,3 +58,13 @@ S3 Event Example Intake
 	  split_key Records
 	  tag cloudtrail
 	</match>
+# params
+    tag my.new.tag : tag name to emit new record as
+    uncompress gzip : decompression algorithm (only gzip:-/)
+    s3_bucket_key my_s3_bucket : The name of your S3 bucket
+    s3_object_key_key /some/cool/object : The path to your S3 object
+    merge_record  yes|no : Do we merge or replace the input record
+    remove_keys key1, key2 : keys that we remove after reading the s3 object
+    compression_exts gz, zip : extensions that we uncompress.  Allows you to ingest both compressed and uncompressed files
+    record_key : if set, the record will be placed in this key 
+

data/fluent-plugin-s3-input.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-s3-input"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.11"
   spec.authors       = ["Anthony Johnson"]
   spec.email         = ["ansoni@gmail.com"]
   spec.description   = %q{Fluentd plugin to read a file from S3 and emit it}
@@ -21,4 +21,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency     "fluentd"
   spec.add_runtime_dependency     "aws-sdk"
   spec.add_runtime_dependency     "oj"
+  spec.add_runtime_dependency     "rubyzip"
 end

data/lib/fluent/plugin/out_s3_input.rb

@@ -1,3 +1,6 @@
+require 'csv'
+require 'json'
+require 'zip'
 
 module Fluent
   class S3InputOutput < Output
@@ -14,8 +17,14 @@ module Fluent
     config_param :s3_bucket_key
     config_param :s3_object_key_key
     config_param :tag
-    # supports: gzip
-    config_param :uncompress, :string
+    config_param :merge_record, :bool, :default => false
+    config_param :record_key, :string, :default => nil
+    config_param :remove_keys, :array, :default => []
+    config_param :time_keys, :array, :default => []
+    config_param :time_format, :string, :default => "%Y-%m-%dT%H:%M:%S"
+    config_param :gzip_exts, :array, :default => []
+    config_param :zip_exts, :array, :default => []
+    config_param :format, :string, :default => 'json'
 
     attr_accessor :s3
 
@@ -28,6 +37,7 @@ module Fluent
 
     def configure(conf)
       super
+
       if @aws_key_id and @aws_sec_key
         @s3 = Aws::S3::Client.new(
           region: "us-east-1",
@@ -35,27 +45,86 @@ module Fluent
           secret_access_key: @aws_sec_key,
         )
       else
-        @s3 = Aws::S3::Client.new(region: "us-east-1")
+        @s3 = Aws::S3::Client.new()
+      end
+    end
+    
+    # Allow JSON data in a couple of formats
+    # {} single event
+    # [{},{}] array of events
+    # {}\n{}\n{} concatenated events (flume)
+    def normalize_json(json)
+      if json[0] != "["
+        json=json.gsub /}\n{/,"},{" 
+        json="[#{json}]"
       end
+      json
     end
 
     def emit(tag, es, chain)
-      tag_parts = tag.split('.')
-      es.each { |time, record|
-        s3_bucket = record[s3_bucket_key]
-        s3_key = record[s3_object_key_key]
-        resp = s3.get_object(bucket: s3_bucket, key: s3_key) 
-        if @uncompress && @uncompress == "gzip"
-          input = Zlib::GzipReader.new(resp.body)
-        else
-          input = resp.body
-        end
-        new_record = Oj.load(input.read)
-        router.emit(@tag, time, new_record)
-      }
-      chain.next
-    rescue => e
-      $log.warn "s3_input: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
+      begin
+        tag_parts = tag.split('.')
+        es.each { |time, record|
+          s3_bucket = record[s3_bucket_key]
+          s3_key = record[s3_object_key_key]
+          s3_key_ext = s3_key.split(".")[-1]
+          resp = s3.get_object(bucket: s3_bucket, key: s3_key) 
+
+          if @gzip_exts.include?(s3_key_ext)
+            input = Zlib::GzipReader.new(resp.body)
+          elsif @zip_exts.include?(s3_key_ext)
+            io = Zip::InputStream.new(resp.body)
+            input = io.get_next_entry
+            #input = Zip::File.open(resp.body).entries.first.get_input_stream
+          else
+            input = resp.body
+          end
+
+          new_record = {}
+          if @merge_record
+            new_record = {}.merge(record)
+          end
+   
+          s3_record = {} 
+          if @format == 'json'
+            json_data=normalize_json input.read
+            s3_record = Oj.load(json_data)
+          elsif @format == 'csv'
+            data = input.read
+            File.open("/tmp/s3debug", 'w') { |file| file.write(data) } 
+            s3_record=CSV.parse(data).to_json
+          else
+            raise "Unsupported format - #{@format}"
+          end
+
+          # parse the time from the record
+          @time_keys.each do |time_key|
+            puts "Look for #{time_key} in #{new_record}"
+            if s3_record.include? time_key
+              puts "Reset time for #{time_key}"
+              time=Time.strptime(new_record[time_key], @time_format).to_i
+              puts "Setting time to #{time}"
+              break
+            end
+          end
+
+          if @record_key == nil  
+            tmp_record=s3_record.merge(new_record)
+            new_record=tmp_record
+          else
+            new_record[record_key]=s3_record
+          end
+        
+          @remove_keys.each do |key_to_remove|
+            new_record.delete(key_to_remove)
+          end
+
+          router.emit(@tag, time, new_record)
+        }
+        chain.next
+      rescue StandardError => e
+        $log.warn "s3_input: #{e.class} #{e.message} #{e.backtrace.join(', ')}"
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3-input
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.11
 platform: ruby
 authors:
 - Anthony Johnson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-18 00:00:00.000000000 Z
+date: 2017-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fluentd plugin to read a file from S3 and emit it
 email:
 - ansoni@gmail.com