logstash-output-azureblob 0.9.0

data/spec/outputs/blob/temporary_file_spec.rb

@@ -0,0 +1,46 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/blob/temporary_file'
+require 'stud/temporary'
+require 'fileutils'
+require 'securerandom'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile do
+  let(:content) { 'hello world' }
+  let(:key) { 'foo' }
+  let(:uuid) { SecureRandom.uuid }
+  let(:temporary_file) { ::File.open(::File.join(temporary_directory, uuid, key), 'w+') }
+  let(:temporary_directory) { Stud::Temporary.directory }
+
+  before :each do
+    FileUtils.mkdir_p(::File.join(temporary_directory, uuid))
+  end
+
+  subject { described_class.new(key, temporary_file, temporary_directory) }
+
+  it 'returns the key of the file' do
+    expect(subject.key).to eq(key)
+  end
+
+  it 'saves content to a file' do
+    subject.write(content)
+    subject.close
+    expect(File.read(subject.path).strip).to eq(content)
+  end
+
+  it 'deletes a file' do
+    expect(File.exist?(subject.path)).to be_truthy
+    subject.delete!
+    expect(File.exist?(subject.path)).to be_falsey
+  end
+
+  it 'successfully delete a file already closed' do
+    subject.close
+    expect(File.exist?(subject.path)).to be_truthy
+    subject.delete!
+    expect(File.exist?(subject.path)).to be_falsey
+  end
+
+  it 'returns the creation time' do
+    expect(subject.ctime).to be < Time.now + 0.5
+  end
+end

data/spec/outputs/blob/time_rotation_policy_spec.rb

@@ -0,0 +1,59 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/blob/time_rotation_policy'
+require 'logstash/outputs/blob/temporary_file'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::TimeRotationPolicy do
+  subject { described_class.new(max_time) }
+
+  let(:max_time) { 1 }
+  let(:temporary_directory) { Stud::Temporary.directory }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:name) { 'foobar' }
+  let(:content) { 'hello' * 1000 }
+  let(:file) { LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile.new(name, temporary_file, temporary_directory) }
+
+  it 'raises an exception if the `file_time` is set to 0' do
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+  end
+
+  it 'raises an exception if the `file_time` is < 0' do
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+  end
+
+  context 'when the size of the file is superior to 0' do
+    before :each do
+      file.write(content)
+      file.fsync
+    end
+
+    it 'returns true if the file old enough' do
+      allow(file).to receive(:ctime).and_return(Time.now - (max_time * 2 * 60))
+      expect(subject.rotate?(file)).to be_truthy
+    end
+
+    it 'returns false is not old enough' do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context 'When the size of the file is 0' do
+    it 'returns false if the file old enough' do
+      allow(file).to receive(:ctime).and_return(Time.now - (max_time * 2 * 60))
+      expect(subject.rotate?(file)).to be_falsey
+    end
+
+    it 'returns false is not old enough' do
+      expect(subject.rotate?(file)).to be_falsey
+    end
+  end
+
+  context '#needs_periodic?' do
+    it 'return false' do
+      expect(subject.needs_periodic?).to be_truthy
+    end
+  end
+
+  it 'convert minute into seconds' do
+    expect(subject.time_file).to eq(60)
+  end
+end

data/spec/outputs/blob/uploader_spec.rb

@@ -0,0 +1,61 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/blob/uploader'
+require 'logstash/outputs/blob/temporary_file'
+require 'azure/storage/common'
+require 'azure/storage/blob'
+require 'stud/temporary'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::Uploader do
+  let(:logger) { spy(:logger) }
+  let(:max_upload_workers) { 1 }
+  let(:storage_account_name) { 'test-cointainer' }
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:temporary_file) { Stud::Temporary.file }
+  let(:storage_access_key) { 'foobar' }
+  let(:threadpool) do
+    Concurrent::ThreadPoolExecutor.new(min_threads: 1,
+                                       max_threads: 8,
+                                       max_queue: 1,
+                                       fallback_policy: :caller_runs)
+  end
+
+  let(:file) do
+    f = LogStash::Outputs::LogstashAzureBlobOutput::TemporaryFile.new(storage_access_key, temporary_file, temporary_directory)
+    f.write('random content')
+    f.fsync
+    f
+  end
+
+  # subject { described_class.new(storage_account_name, logger, threadpool) }
+
+  # it "upload file to the blob" do
+  #  expect { subject.upload(file) }.not_to raise_error
+  # end
+
+  # it "execute a callback when the upload is complete" do
+  #  callback = proc { |f| }
+
+  #  expect(callback).to receive(:call).with(file)
+  #  subject.upload(file, { :on_complete => callback })
+  # end
+
+  # it 'the content in the blob and sent should be equal' do
+  #  blob = subject.upload(file)
+  #  md5 = Digest::MD5.base64digest(Object::File.open(file.path).read)
+  #  expect(blob.properties[:content_md5]).to eq(md5)
+  # end
+
+  #  xit "retries errors indefinitively" do
+  #    blob = double("blob").as_null_object
+
+  #    expect(logger).to receive(:error).with(any_args).once
+
+  #    expect(storage_account_name).to receive(:object).with(file.key).and_return(blob).twice
+
+  #    expect(blob).to receive(:upload_file).with(any_args).and_raise(StandardError)
+
+  #    expect(blob).to receive(:upload_file).with(any_args).and_return(true)
+
+  #    subject.upload(file)
+  #  end
+end

data/spec/outputs/blob/writable_directory_validator_spec.rb

@@ -0,0 +1,39 @@
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/blob/writable_directory_validator'
+require 'stud/temporary'
+
+describe LogStash::Outputs::LogstashAzureBlobOutput::WritableDirectoryValidator do
+  let(:temporary_directory) { File.join(Stud::Temporary.directory, Time.now.to_i.to_s) }
+
+  subject { described_class }
+
+  context "when the directory doesn't exists" do
+    it 'creates the directory' do
+      expect(Dir.exist?(temporary_directory)).to be_falsey
+      expect(subject.valid?(temporary_directory)).to be_truthy
+      expect(Dir.exist?(temporary_directory)).to be_truthy
+    end
+  end
+
+  context 'when the directory exist' do
+    before do
+      FileUtils.mkdir_p(temporary_directory)
+    end
+
+    it "doesn't change the directory" do
+      expect(Dir.exist?(temporary_directory)).to be_truthy
+      expect(subject.valid?(temporary_directory)).to be_truthy
+      expect(Dir.exist?(temporary_directory)).to be_truthy
+    end
+  end
+
+  it 'return false if the directory is not writable' do
+    expect(::File).to receive(:writable?).with(temporary_directory).and_return(false)
+    expect(subject.valid?(temporary_directory)).to be_falsey
+  end
+
+  it 'return true if the directory is writable' do
+    expect(::File).to receive(:writable?).with(temporary_directory).and_return(true)
+    expect(subject.valid?(temporary_directory)).to be_truthy
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'logstash/devutils/rspec/spec_helper'
+require_relative 'supports/helpers'
+require 'logstash/logging/logger'
+
+LogStash::Logging::Logger.configure_logging('debug') if ENV['DEBUG']

data/spec/supports/helpers.rb

@@ -0,0 +1,27 @@
+shared_context 'setup plugin' do
+  let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:storage_account_name) { ENV['AZURE_STORAGE_ACCOUNT'] }
+  let(:storage_access_key) { ENV['AZURE_STORAGE_ACCESS_KEY'] }
+  let(:size_file) { 100 }
+  let(:time_file) { 100 }
+  let(:tags) { [] }
+  let(:prefix) { 'home' }
+
+  let(:main_options) do
+    {
+      'storage_account_name' => bucket,
+      'prefix' => prefix,
+      'temporary_directory' => temporary_directory,
+      'storage_access_key' => access_key_id,
+      'size_file' => size_file,
+      'time_file' => time_file,
+      'tags' => []
+    }
+  end
+
+  subject { LogStash::Outputs::LogstashAzureBlobOutput.new(options) }
+end
+
+def clean_remote_files(prefix = '')
+  bucket_resource.objects(prefix: prefix).each(&:delete)
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-azureblob
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- Sean Stark
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-02-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: azure-storage-blob
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This output plugin will send logs to azure blob storage. This does not
+  support ADLS2 endpoints
+email: sean.stark@microsoft.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/azure_blob.rb
+- lib/logstash/outputs/blob/file_repository.rb
+- lib/logstash/outputs/blob/path_validator.rb
+- lib/logstash/outputs/blob/size_and_time_rotation_policy.rb
+- lib/logstash/outputs/blob/size_rotation_policy.rb
+- lib/logstash/outputs/blob/temporary_file.rb
+- lib/logstash/outputs/blob/temporary_file_factory.rb
+- lib/logstash/outputs/blob/time_rotation_policy.rb
+- lib/logstash/outputs/blob/uploader.rb
+- lib/logstash/outputs/blob/writable_directory_validator.rb
+- lib/logstash/outputs/template.rb
+- logstash-output-azureblob.gemspec
+- spec/outputs/azureblob_spec.rb
+- spec/outputs/blob/file_repository_spec.rb
+- spec/outputs/blob/size_and_time_rotation_policy_spec.rb
+- spec/outputs/blob/size_rotation_policy_spec.rb
+- spec/outputs/blob/temporary_file_factory_spec.rb
+- spec/outputs/blob/temporary_file_spec.rb
+- spec/outputs/blob/time_rotation_policy_spec.rb
+- spec/outputs/blob/uploader_spec.rb
+- spec/outputs/blob/writable_directory_validator_spec.rb
+- spec/spec_helper.rb
+- spec/supports/helpers.rb
+homepage: https://github.com/seanstark/logstash-output-azureblob
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Output plugin for logstash to send output to Azure Blob Storage
+test_files:
+- spec/outputs/azureblob_spec.rb
+- spec/outputs/blob/file_repository_spec.rb
+- spec/outputs/blob/size_and_time_rotation_policy_spec.rb
+- spec/outputs/blob/size_rotation_policy_spec.rb
+- spec/outputs/blob/temporary_file_factory_spec.rb
+- spec/outputs/blob/temporary_file_spec.rb
+- spec/outputs/blob/time_rotation_policy_spec.rb
+- spec/outputs/blob/uploader_spec.rb
+- spec/outputs/blob/writable_directory_validator_spec.rb
+- spec/spec_helper.rb
+- spec/supports/helpers.rb