cwlogs_io 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e76cb59bec88c77636194c3a5fe6c45cd14a306cdf2ad8622ba834b591ed7cd0
+  data.tar.gz: 11f88c626fff3efc54171baddc89c317746e0ec60d830ac329378744805f0deb
+SHA512:
+  metadata.gz: 841c88a803b94f77ea5bbdc6c08517bc00ce0316c26c50c41e9e57d53e6aa666be1bb17433b8571a0ddc5c1f32ab85f02ee3b75715ac2fe337568bfd4e2afc82
+  data.tar.gz: 11893c237bd7484ee82fd219a22d1414620151b8072470312df95f28a6033603e6110a32f1380a88b944b5a6cec16709a54c0da6804bef61fccdba19e7f329e5

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rufo

@@ -0,0 +1,5 @@
+align_case_when false
+align_chained_calls false
+parens_in_def :yes
+trailing_commas true
+quote_style :single

data/Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cwlogs_io.gemspec
+gemspec
+
+gem 'rake', '~> 13.0'
+
+gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -0,0 +1,55 @@
+PATH
+  remote: .
+  specs:
+    cwlogs_io (0.1.0)
+      aws-sdk-cloudwatchlogs (~> 1.69)
+      aws-sdk-core (~> 3)
+      concurrent-ruby (~> 1.2)
+      nokogiri (~> 1.15)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-eventstream (1.2.0)
+    aws-partitions (1.785.0)
+    aws-sdk-cloudwatchlogs (1.69.0)
+      aws-sdk-core (~> 3, >= 3.177.0)
+      aws-sigv4 (~> 1.1)
+    aws-sdk-core (3.178.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.5)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sigv4 (1.6.0)
+      aws-eventstream (~> 1, >= 1.0.2)
+    concurrent-ruby (1.2.2)
+    diff-lcs (1.5.0)
+    jmespath (1.6.2)
+    nokogiri (1.15.3-x86_64-linux)
+      racc (~> 1.4)
+    racc (1.7.1)
+    rake (13.0.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.6)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  cwlogs_io!
+  rake (~> 13.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.3.13

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Hyeonjun Lee
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# CWlogsIO
+
+ruby IO-like streams which is connected to cloudwatch logs.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add cwlogs_io
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install cwlogs_io
+
+## Usage
+
+```ruby
+
+auth = {
+    region: 'ap-northeast-2'
+    credentials: Aws::InstanceProfileCredentials.new
+}
+
+log_group = 'foo'
+log_stream = 'bar'
+
+logger = Logger.new(CWlogsIO.new(auth, log_group, log_stream))
+logger.info('hello with CWlogsIO')
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/privatenote/cwlogs_io.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/cwlogs_io.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative 'lib/cwlogs_io/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'cwlogs_io'
+  spec.version = CWlogsIO::VERSION
+  spec.authors = ['Hyeonjun Lee']
+  spec.email = ['hyeonjun.lee@privatenote.co.kr']
+
+  spec.summary = 'IO-like streams for CloudWatch Logs.'
+  spec.homepage = 'https://github.com/privatenote/cwlogs_io'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 2.6.0'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/privatenote/cwlogs_io'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  # Uncomment to register a new dependency of your gem
+  spec.add_dependency 'aws-sdk-cloudwatchlogs', '~> 1.69'
+  spec.add_dependency 'aws-sdk-core', '~> 3'
+  spec.add_dependency 'concurrent-ruby', '~> 1.2'
+  spec.add_dependency 'nokogiri', '~> 1.15'
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/lib/cwlogs_io/client.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-cloudwatchlogs'
+
+module CWlogsIO
+  class Client
+    def initialize(auth, log_group, log_stream)
+      @auth = auth
+      @log_group = log_group
+      @log_stream = log_stream
+    end
+
+    def create_log_group
+      client.create_log_group({ log_group_name: @log_group })
+    end
+
+    def create_log_stream
+      client.create_log_stream(
+        {
+          log_group_name: @log_group,
+          log_stream_name: @log_stream,
+        }
+      )
+    end
+
+    def put_log_events(log_events)
+      client.put_log_events(
+        {
+          log_events: to_params(log_events),
+          log_group_name: @log_group,
+          log_stream_name: @log_stream,
+        }
+      )
+    end
+
+    private
+
+    def client
+      @client ||= Aws::CloudWatchLogs::Client.new(
+        region: @auth[:region],
+        credentials: @auth[:credentials],
+      )
+    end
+
+    def to_params(events)
+      events.map do |event|
+        {
+          message: event.message,
+          timestamp: (event.timestamp.to_f * 1000).to_i
+        }
+      end
+    end
+  end
+end

data/lib/cwlogs_io/handler.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'singleton'
+require 'concurrent/executor/single_thread_executor'
+
+module CWlogsIO
+  class HandlerWrapper
+    def initialize(handler)
+      @handler = handler
+    end
+
+    def self.create(handler_class, *params)
+      handler = HandlerWrapper.new(handler_class.new(*params))
+      HandlerManager.instance.register(handler)
+      handler
+    end
+
+    def close
+      @handler.close
+      HandlerManager.instance.deregister(self)
+    end
+
+    def method_missing(method, *params)
+      @handler.send(method, *params)
+    end
+
+    def respond_to_missing?(method, include_private = false)
+      @handler.respond_to?(method, include_private)
+    end
+  end
+
+  class HandlerManager
+    include Singleton
+
+    def initialize
+      @handlers = Set.new
+    end
+
+    def respawn_all
+      @handlers.each(&:respawn)
+    end
+
+    def register(handler)
+      @handlers << handler
+    end
+
+    def deregister(handler)
+      @handlers.delete(handler)
+    end
+  end
+
+  class Handler
+    DEFAULT_POLLING_PERIOD_IN_SECONDS = 1
+    TERMINATION_TIMEOUT_IN_SECONDS = 5
+
+    def initialize(logger, polling_period = nil)
+      @queue = Queue.new
+      @logger = logger
+      @polling_period = polling_period || DEFAULT_POLLING_PERIOD_IN_SECONDS
+
+      initialize_executor
+    end
+
+    def enq(event)
+      return if closed?
+
+      @queue << event
+    end
+
+    def <<(event)
+      enq(event)
+    end
+
+    def close
+      return if closed?
+
+      @queue.close
+
+      shutdown_executor
+    end
+
+    def closed?
+      @queue.closed?
+    end
+
+    # should be called in child process, not parent.
+    # or, you may lose some events.
+    def respawn
+      return if closed?
+
+      @queue.clear
+      initialize_executor
+    end
+
+    private
+
+    def initialize_executor
+      @executor = Concurrent::SingleThreadExecutor.new
+      @executor.post do
+        serve
+      end
+    end
+
+    def shutdown_executor
+      @executor.shutdown
+      @executor.wait_for_termination(TERMINATION_TIMEOUT_IN_SECONDS)
+      @executor.kill unless @executor.shutdown?
+    end
+
+    def serve
+      @logger.info('ready to process')
+      until @queue.closed?
+        handle_all
+
+        sleep @polling_period
+      end
+
+      handle_all unless @queue.empty?
+      @logger.info('quitting...')
+    rescue Exception => e
+      @logger.error(e)
+    end
+
+    def pop_all_events
+      result = []
+      result << @queue.pop until @queue.empty?
+      result
+    end
+
+    def handle_all
+      process_events(pop_all_events) unless @queue.empty?
+    end
+
+    def process_events(events)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/cwlogs_io/io.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-cloudwatchlogs'
+require 'json/ext'
+require 'logger'
+require_relative 'log_event'
+require_relative 'utils'
+require_relative 'handler'
+require_relative 'client'
+
+module CWlogsIO
+  class LogEventHandler < Handler
+    # https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html
+    # The maximum batch size is 1,048,576 bytes, and this size is calculated as the sum of all event messages in UTF-8, plus 26 bytes for each log event.
+    # 26 bytes is the length of the JSON structure that surrounds the log event information:
+    # Each log event can be no larger than 256 KB.
+    # The maximum number of log events in a batch is 10,000
+    # When there are 100 events in one chunk, the average size of each event can be up to a maximum of 10kB.
+    EVENTS_PER_CHUNK = 100
+
+    def initialize(client, log_group, log_stream, logger, polling_rate = nil)
+      super(logger, polling_rate)
+      @client = client
+      @log_group = log_group
+      @log_stream = log_stream
+    end
+
+    def process_events(events)
+      events.sort_by!(&:timestamp)
+      Utils.chunks(events, EVENTS_PER_CHUNK).each do |event_chunk|
+        send_events(event_chunk)
+      end
+    end
+
+    private
+
+    def send_events(events)
+      return if events.empty?
+
+      @client.put_log_events(events)
+    rescue StandardError => e
+      @logger.error(e)
+    end
+  end
+
+  # ruby 'logger' compatible IO-like class.
+  # it should implement #write, #close method.
+  # https://github.com/ruby/logger/blob/master/lib/logger/log_device.rb
+  class IO
+    def initialize(auth, log_group, log_stream)
+      @auth = auth
+      @log_group = log_group
+      @log_stream = log_stream
+      @handler = HandlerWrapper.create(LogEventHandler, client, log_group, log_stream, logger)
+      close if !ensure_log_group || !ensure_log_stream
+
+      at_exit do
+        close
+      end
+    end
+
+    def write(message)
+      @handler << LogEvent.new(message, Time.now)
+    end
+
+    def close
+      @handler.close
+    end
+
+    private
+
+    def client
+      @client ||= Client.new(@auth, @log_group, @log_stream)
+    end
+
+    def ensure_log_group
+      client.create_log_group
+      true
+    rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+      true
+    rescue Aws::CloudWatchLogs::Errors::ServiceError => e
+      logger.error(e)
+      false
+    end
+
+    def ensure_log_stream
+      client.create_log_stream
+      true
+    rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+      true
+    rescue Aws::CloudWatchLogs::Errors::ServiceError => e
+      logger.error(e)
+      false
+    end
+
+    def logger
+      @@logger ||= Logger.new($stdout)
+    end
+  end
+end

data/lib/cwlogs_io/log_event.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module CWlogsIO
+  class LogEvent
+    attr_reader :message, :timestamp
+
+    def initialize(message, timestamp)
+      @message = message
+      @timestamp = timestamp
+    end
+  end
+end

data/lib/cwlogs_io/utils.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module CWlogsIO
+  module Utils
+    def self.chunks(array, size)
+      array.each_slice(size)
+    end
+  end
+end

data/lib/cwlogs_io/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module CWlogsIO
+  VERSION = '0.1.0'
+end

data/lib/cwlogs_io.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'cwlogs_io/version'
+require_relative 'cwlogs_io/io'
+require_relative 'hooks'
+
+module CWlogsIO
+  def self.new(auth, log_group, log_stream)
+    CWlogsIO::IO.new(auth, log_group, log_stream)
+  end
+end

data/lib/hooks/phusion_passenger.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+if defined?(PhusionPassenger)
+  PhusionPassenger.on_event(:starting_worker_process) do |forked|
+    CWlogsIO::HandlerManager.instance.respawn_all if forked
+  end
+end

data/lib/hooks.rb

@@ -0,0 +1 @@
+require_relative 'hooks/phusion_passenger'

data/sig/cwlogs_io.rbs

@@ -0,0 +1,4 @@
+module CwlogsIo
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: cwlogs_io
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Hyeonjun Lee
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-07-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudwatchlogs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.69'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.69'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+description:
+email:
+- hyeonjun.lee@privatenote.co.kr
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rufo"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- cwlogs_io.gemspec
+- lib/cwlogs_io.rb
+- lib/cwlogs_io/client.rb
+- lib/cwlogs_io/handler.rb
+- lib/cwlogs_io/io.rb
+- lib/cwlogs_io/log_event.rb
+- lib/cwlogs_io/utils.rb
+- lib/cwlogs_io/version.rb
+- lib/hooks.rb
+- lib/hooks/phusion_passenger.rb
+- sig/cwlogs_io.rbs
+homepage: https://github.com/privatenote/cwlogs_io
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/privatenote/cwlogs_io
+  source_code_uri: https://github.com/privatenote/cwlogs_io
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: IO-like streams for CloudWatch Logs.
+test_files: []