fluent-plugin-aliyun-oss 0.0.1

data/test/plugin/test_in_oss.rb

@@ -0,0 +1,166 @@
+require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/test/log'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_oss'
+require 'test/unit/rr'
+require 'uuidtools'
+
+class OSSInputTest < Test::Unit::TestCase
+
+  include Fluent::Test::Helpers
+  include REXML
+
+  def setup
+    Fluent::Test.setup
+    @time = Time.now
+    stub(Fluent::EventTime).now { @time } if Fluent.const_defined?(:EventTime)
+  end
+
+  CONFIG = %(
+    endpoint  #{ENV['OSS_ENDPOINT']}
+    bucket    #{ENV['OSS_BUCKET']}
+    access_key_id #{ENV['ACCESS_KEY_ID']}
+    access_key_secret #{ENV['ACCESS_KEY_SECRET']}
+    oss_sdk_log_dir .
+    store_as #{ENV['STORE_AS']}
+    store_local #{ENV['STORE_LOCAL']}
+    <mns>
+      endpoint #{ENV['MNS_ENDPOINT']}
+      queue #{ENV['MNS_QUEUE']}
+      wait_seconds #{ENV['WAIT_SECONDS']}
+      poll_interval_seconds #{ENV['POLL_INTERVAL_SECONDS']}
+    </mns>
+    <parse>
+      @type json
+    </parse>
+  ).freeze
+
+  def create_driver
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::OSSInput)
+  end
+
+  def test_configuration
+    driver = create_driver
+    driver.configure(CONFIG)
+
+    assert_equal(driver.instance.endpoint, ENV['OSS_ENDPOINT'])
+    assert_equal(driver.instance.bucket, ENV['OSS_BUCKET'])
+    assert_equal(driver.instance.access_key_id, ENV['ACCESS_KEY_ID'])
+    assert_equal(driver.instance.access_key_secret, ENV['ACCESS_KEY_SECRET'])
+    assert_equal(driver.instance.oss_sdk_log_dir, '.')
+    assert_equal(driver.instance.store_as, ENV['STORE_AS'])
+    assert_not_equal(driver.instance.store_as, ENV['STORE_AS'] + '-')
+
+    assert_equal(driver.instance.store_local, ENV['STORE_LOCAL'].to_bool) unless ENV['STORE_LOCAL'].nil?
+
+    assert_not_equal(driver.instance.store_local, !ENV['STORE_LOCAL'].to_bool) unless ENV['STORE_LOCAL'].nil?
+
+    assert_equal(driver.instance.mns.endpoint, ENV['MNS_ENDPOINT'])
+    assert_equal(driver.instance.mns.queue, ENV['MNS_QUEUE'])
+    assert_equal(driver.instance.mns.wait_seconds, ENV['WAIT_SECONDS'].to_i) unless ENV['WAIT_SECONDS'].nil?
+    assert_equal(driver.instance.mns.poll_interval_seconds, ENV['POLL_INTERVAL_SECONDS'].to_i) unless ENV['POLL_INTERVAL_SECONDS'].nil?
+
+    assert_equal(driver.instance.parser_configs[0]['@type'], 'json')
+
+    driver.instance.shutdown
+    puts driver.logs
+  end
+
+  def test_emit_events
+    driver = create_driver
+    driver.configure(CONFIG)
+
+    expect_records = 12_345
+    oss = create_oss(driver)
+
+    bucket = oss.get_bucket(driver.instance.bucket)
+
+    # create test object to bucket
+    content = get_random_content(expect_records)
+    object = get_test_object(content, bucket, driver.instance.store_as)
+
+    driver.run(expect_records: expect_records)
+
+    assert_equal(driver.events.size, expect_records)
+
+    result = ''
+    driver.events.each do |event|
+      result << event[2].to_s.gsub(/=>/, ':') << "\n"
+    end
+
+    assert_equal(result, content)
+
+    assert_false(driver.instance.instance_variable_get(:@running))
+    puts driver.logs
+
+    # delete test object
+    bucket.delete_object(object)
+  end
+
+  def test_no_events
+    driver = create_driver
+    driver.configure(CONFIG)
+
+    expect_records = 0
+    driver.run(expect_records: expect_records)
+
+    assert_equal(driver.events.size, expect_records)
+
+    assert_false(driver.instance.instance_variable_get(:@running))
+    puts driver.logs
+  end
+
+  def get_test_object(content, bucket, store_as)
+    object = 'fluentd-oss-test-' + ::UUIDTools::UUID.random_create.to_s
+    io = Tempfile.new(object)
+    io.binmode
+    io.write(content)
+    io.rewind
+
+    out = Tempfile.new('chunk-test-' + store_as + '-out-')
+    out.binmode
+
+    case store_as
+    when 'text'
+      object += '.txt'
+      bucket.put_object(object, file: io.path)
+    when 'json'
+      object += '.' + store_as
+      bucket.put_object(object, file: io.path)
+    when 'gzip_command', 'gzip'
+      object += '.gz'
+      system "gzip -c #{io.path} > #{out.path}"
+      bucket.put_object(object, file: out.path)
+    when 'lzo'
+      object += '.' + store_as
+      system "lzop -qf1 -c #{io.path} > #{out.path}"
+      bucket.put_object(object, file: out.path)
+    when 'lzma2'
+      object += '.xz'
+      system "xz -qf0 -c #{io.path} > #{out.path}"
+      bucket.put_object(object, file: out.path)
+    end
+
+    io.close(true) rescue nil
+    out.close(true) rescue nil
+    object
+  end
+
+  def get_random_content(lines)
+    content = []
+    lines.times do |i|
+      value = ::UUIDTools::UUID.random_create.to_s + '-' + i.to_s
+      content.append('{"message":"' + value + '"}' + "\n")
+    end
+    content.join
+  end
+
+  def create_oss(driver)
+    Aliyun::OSS::Client.new(
+      endpoint: driver.instance.endpoint,
+      access_key_id: driver.instance.access_key_id,
+      access_key_secret: driver.instance.access_key_secret
+    )
+  end
+end

data/test/plugin/test_out_oss.rb

@@ -0,0 +1,175 @@
+require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/test/log'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_oss'
+require 'test/unit/rr'
+require 'uuidtools'
+
+class OSSOutputTest < Test::Unit::TestCase
+
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+    @time = Time.now
+    stub(Fluent::EventTime).now { @time } if Fluent.const_defined?(:EventTime)
+  end
+
+  CONFIG = %(
+    endpoint  #{ENV['OSS_ENDPOINT']}
+    bucket    #{ENV['OSS_BUCKET']}
+    access_key_id #{ENV['ACCESS_KEY_ID']}
+    access_key_secret #{ENV['ACCESS_KEY_SECRET']}
+    oss_sdk_log_dir .
+    path #{ENV['OSS_PATH']}
+    store_as #{ENV['STORE_AS']}
+    <buffer tag,time>
+      @type memory
+      timekey 30
+      timekey_wait 1s
+    </buffer>
+    <format>
+      @type json
+    </format>
+  ).freeze
+
+  def create_driver
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::OSSOutput)
+  end
+
+  def test_configuration
+    driver = create_driver
+    driver.configure(CONFIG)
+
+    assert_equal(driver.instance.endpoint, ENV['OSS_ENDPOINT'])
+    assert_equal(driver.instance.bucket, ENV['OSS_BUCKET'])
+    assert_equal(driver.instance.access_key_id, ENV['ACCESS_KEY_ID'])
+    assert_equal(driver.instance.access_key_secret, ENV['ACCESS_KEY_SECRET'])
+    assert_equal(driver.instance.oss_sdk_log_dir, '.')
+    assert_equal(driver.instance.path, ENV['OSS_PATH'])
+
+    assert_equal(driver.instance.store_as, ENV['STORE_AS'])
+    assert_not_equal(driver.instance.store_as, ENV['STORE_AS'] + '-')
+
+    assert_equal(driver.instance.buffer_config['@type'], 'memory')
+    assert_equal(driver.instance.buffer_config['timekey'], 30)
+    assert_equal(driver.instance.buffer_config['timekey_wait'], 1)
+    assert_equal(driver.instance.formatter_configs[0]['@type'], 'json')
+
+    driver.instance.shutdown
+    puts driver.logs
+  end
+
+  def test_write_lines
+    driver = create_driver
+    driver.configure(CONFIG)
+
+    time = event_time('2019-04-16 14:26:22 UTC')
+
+    expect_records = 12_345
+    content = get_random_content(expect_records)
+
+    expected_content = []
+    driver.run(default_tag: 'oss.output') do
+      content.each do |line|
+        driver.feed(time, 'message' => line)
+        expected_content.append("{\"message\":\"#{line}\"}")
+      end
+    end
+
+    oss = create_oss(driver)
+
+    bucket = oss.get_bucket(driver.instance.bucket)
+
+    verify_object_content(bucket, driver.instance.path,
+                          driver.instance.store_as,
+                          expected_content.join("\n") + "\n")
+    puts driver.logs
+  end
+
+  def test_no_writes
+    driver = create_driver
+    driver.configure(CONFIG)
+
+    driver.run(default_tag: 'oss.output') do
+    end
+
+    oss = create_oss(driver)
+
+    bucket = oss.get_bucket(driver.instance.bucket)
+
+    verify_object_content(bucket, driver.instance.path, driver.instance.store_as, nil)
+    puts driver.logs
+  end
+
+  def verify_object_content(bucket, prefix, store_as, expected_content)
+    objects = []
+    bucket.list_objects(prefix: prefix).each do |object|
+      objects.append(object.key)
+    end
+    assert_true(objects.size >= 1) unless expected_content.nil?
+    assert_equal(objects.size, 0) if expected_content.nil?
+
+    contents = []
+    unless expected_content.nil?
+      objects.each do |object|
+        contents.append(decompress_object(bucket, object, store_as))
+        bucket.delete_object(object)
+      end
+      assert_equal(contents.join, expected_content)
+    end
+  end
+
+  def create_oss(driver)
+    Aliyun::OSS::Client.new(
+      endpoint: driver.instance.endpoint,
+      access_key_id: driver.instance.access_key_id,
+      access_key_secret: driver.instance.access_key_secret
+    )
+  end
+
+  def get_random_content(lines)
+    content = []
+    lines.times do |i|
+      value = ::UUIDTools::UUID.random_create.to_s + '-' + i.to_s
+      content.append(value)
+    end
+    content
+  end
+
+  def decompress_object(bucket, key, store_as)
+    decompressed_content = ''
+
+    contents = []
+    bucket.get_object(key) do |content|
+      contents.append(content)
+    end
+
+    object = 'fluentd-oss-test-' + ::UUIDTools::UUID.random_create.to_s
+    io = Tempfile.new(object)
+    io.binmode
+    io.write(contents.join)
+    io.rewind
+
+    case store_as
+    when 'text', 'json'
+      decompressed_content = contents
+    when 'gzip_command', 'gzip'
+      stdout, succeeded = Open3.capture2("gzip -dc #{io.path}")
+    when 'lzo'
+      stdout, succeeded = Open3.capture2("lzop -qdc #{io.path}")
+    when 'lzma2'
+      stdout, succeeded = Open3.capture2("xz -qdc #{io.path}")
+    end
+
+    io.close(true) rescue nil
+
+    if succeeded
+      stdout.each_line do |line|
+        decompressed_content << line
+      end
+    end
+    decompressed_content
+  end
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-aliyun-oss
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jinhu Wu
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-04-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aliyun-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+description: Aliyun OSS plugin for Fluentd event collector
+email: jinhu.wu.nju@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- AUTHORS
+- ChangeLog
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-oss.gemspec
+- lib/fluent/plugin/in_oss.rb
+- lib/fluent/plugin/mns/message.rb
+- lib/fluent/plugin/mns/request.rb
+- lib/fluent/plugin/oss_compressor_gzip_command.rb
+- lib/fluent/plugin/oss_compressor_lzma2.rb
+- lib/fluent/plugin/oss_compressor_lzo.rb
+- lib/fluent/plugin/oss_decompressor_gzip_command.rb
+- lib/fluent/plugin/oss_decompressor_lzma2.rb
+- lib/fluent/plugin/oss_decompressor_lzo.rb
+- lib/fluent/plugin/out_oss.rb
+- test/plugin/test_in_oss.rb
+- test/plugin/test_out_oss.rb
+homepage: https://github.com/aliyun/fluent-plugin-oss
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.9
+signing_key: 
+specification_version: 4
+summary: Aliyun OSS plugin for Fluentd event collector
+test_files:
+- test/plugin/test_in_oss.rb
+- test/plugin/test_out_oss.rb