cloudwatch_query 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 467d68c5e50ddf28a0be2b7a5e54879c4a2ff0083146a71b359d8d7d7537d485
+  data.tar.gz: 64475bd16a828fb65cd29f18449c81c4bec31e09c9257c35a1f8e635e0bd639b
+SHA512:
+  metadata.gz: aaa0b07437366f8d047b214391e427eaeffd7b9de295c4f80dacd0e57c5462706e21a81beee0050c7bf9259808140be99cd89af1c0635a92329ee1a11afdd534
+  data.tar.gz: 23f498ab01846a21e60e821eb51415ffdf758f5720d7930e014689a4f058509bb1f3134d10ca4f0db4c82b98ba5458c8939ba90fabefdba83ff5f27479e5e066

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 cloudwatch_query contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,298 @@
+# CloudwatchQuery
+
+A Ruby gem for querying AWS CloudWatch Logs with a simple, chainable interface.
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "cloudwatch_query"
+```
+
+Or install directly:
+
+```bash
+gem install cloudwatch_query
+```
+
+## Prerequisites
+
+AWS credentials must be configured. This gem uses the standard AWS credential chain:
+
+1. Environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`)
+2. Shared credentials file (`~/.aws/credentials`)
+3. EC2 instance metadata
+
+## Configuration
+
+```ruby
+CloudwatchQuery.configure do |config|
+  config.region = "us-west-1"           # Default: ENV["AWS_REGION"] or "us-west-1"
+  config.default_limit = 100            # Default result limit
+  config.default_time_range = 3600      # Default lookback in seconds
+end
+```
+
+## Usage
+
+### Basic Query
+
+```ruby
+CloudwatchQuery
+  .logs("my-app-rails-log")
+  .where(level: "ERROR")
+  .past(30, :minutes)
+  .each { |event| puts event.message }
+```
+
+### Quick Search
+
+```ruby
+CloudwatchQuery.search(
+  "OutOfMemoryError",
+  groups: "my-app-rails-log",
+  since: 2.hours.ago,
+  limit: 50
+)
+```
+
+### Multiple Log Groups
+
+```ruby
+CloudwatchQuery
+  .logs(
+    "my-app-rails-log",
+    "my-app-sidekiq-log"
+  )
+  .contains("database")
+  .where(level: "ERROR")
+  .past(1, :hours)
+  .execute
+```
+
+### Time Range Options
+
+```ruby
+query = CloudwatchQuery.logs("my-app-rails-log")
+
+# Relative time
+query.past(30, :minutes)
+query.past(2, :hours)
+query.past(7, :days)
+
+# Absolute time
+query.since(Time.now - 3600)
+query.since(1.hour.ago)            # With ActiveSupport
+query.between(start_time, end_time)
+```
+
+Available time units (singular and plural forms accepted):
+
+| Unit | Symbols |
+|------|---------|
+| Seconds | `:second`, `:seconds` |
+| Minutes | `:minute`, `:minutes` |
+| Hours | `:hour`, `:hours` |
+| Days | `:day`, `:days` |
+| Weeks | `:week`, `:weeks` |
+
+### Field Selection
+
+```ruby
+CloudwatchQuery
+  .logs("my-app-rails-log")
+  .fields(:timestamp, :message, :logStream)
+  .past(1, :hours)
+  .execute
+```
+
+### Query Inspection
+
+```ruby
+query = CloudwatchQuery
+  .logs("my-app-rails-log")
+  .where(level: "ERROR")
+  .contains("timeout")
+  .limit(50)
+
+puts query.to_insights_query
+# => fields @timestamp, @message, @logStream, @log | filter level = 'ERROR' | filter @message like /timeout/ | sort @timestamp desc | limit 50
+```
+
+### Working with Results
+
+```ruby
+results = CloudwatchQuery
+  .logs("my-app-rails-log")
+  .past(1, :hours)
+  .execute
+
+# Enumerable methods
+results.each { |r| puts r.message }
+results.count
+results.empty?
+
+# Access fields
+results.first.timestamp
+results.first.message
+results.first[:logStream]
+results.first.to_h
+```
+
+### Parsers
+
+Results can be automatically parsed into structured objects. The gem ships with two built-in parsers: `RailsParser` and `SidekiqParser`. Parsers are registered globally and run automatically when results are returned.
+
+#### How It Works
+
+Each `Result` has a `parsed` attribute. When a query executes, every result's `message` is passed through the parser registry. The first parser whose `matches?` returns true parses the message into a structured log object (`RailsLog` or `SidekiqLog`).
+
+```ruby
+results = CloudwatchQuery
+  .logs("my-app-rails-log")
+  .past(30, :minutes)
+  .execute
+
+result = results.first
+result.parsed?       # => true
+result.parser_name   # => "rails"
+result.log_type      # => :rails
+result.parsed        # => RailsLog instance
+```
+
+#### Rails Parser
+
+Recognizes log lines containing a Rails request UUID (`[abc-123-def-456]`). Each line is further classified by sub-parsers:
+
+| Sub-parser | Matches | Fields |
+|------------|---------|--------|
+| `:request` | `Started GET "/path" for 1.2.3.4` | `http_method`, `path`, `ip_address` |
+| `:parameters` | `Parameters: {...}` | `params` |
+| `:processing` | `Processing by Controller#action as HTML` | `controller`, `action`, `format` |
+| `:completed` | `Completed 200 OK in 123ms` | `status_code`, `duration_ms` |
+| `:redirect` | `Redirected to https://...` | `redirect_url` |
+| `:active_job` | `[ActiveJob] Enqueued JobClass (Job ID: ...)` | `job_class`, `job_id`, `queue` |
+
+```ruby
+result.parsed.line_type    # => :request
+result.parsed.http_method  # => "GET"
+result.parsed.path         # => "/users/1"
+result.parsed.request_id   # => "abc-123-def-456"
+```
+
+Use only specific sub-parsers:
+
+```ruby
+CloudwatchQuery.parsers.clear
+CloudwatchQuery.parsers.register(
+  CloudwatchQuery::Parsers::RailsParser.new(:request, :completed)
+)
+```
+
+#### Sidekiq Parser
+
+Recognizes Sidekiq log lines (`2026-02-04T20:46:15.201Z pid=123 tid=abc class=Job jid=xyz`). Sub-parsers:
+
+| Sub-parser | Matches | Fields |
+|------------|---------|--------|
+| `:start` | `INFO: start` | `status` |
+| `:done` | `INFO: done` | `status`, `elapsed` |
+| `:fail` | `INFO: fail` or `ERROR:` | `status` |
+
+```ruby
+result.parsed.line_type  # => :done
+result.parsed.job_class  # => "SendEmailJob"
+result.parsed.elapsed    # => 0.152
+result.parsed.jid        # => "abc123"
+```
+
+#### Filtering Parsed Results
+
+```ruby
+results.parsed                    # only successfully parsed results
+results.unparsed                  # results that no parser matched
+results.by_type(:rails)           # filter by log type
+results.group_by_request          # group Rails logs by request_id
+```
+
+#### Custom Parsers
+
+Create a parser that responds to `matches?` and `parse`:
+
+```ruby
+class MyParser
+  def matches?(message)
+    message.include?("[CUSTOM]")
+  end
+
+  def parse(message)
+    OpenStruct.new(type: :custom, body: message)
+  end
+
+  def parser_name
+    "custom"
+  end
+end
+
+CloudwatchQuery.parsers.register(MyParser.new)
+```
+
+Registry methods: `register`, `prepend`, `insert`, `unregister`, `clear`, `list`.
+
+### List Log Groups
+
+```ruby
+CloudwatchQuery.list_log_groups(prefix: "production-")
+```
+
+## API Reference
+
+### Query Builder Methods
+
+| Method | Description |
+|--------|-------------|
+| `.logs(*groups)` | Select log groups to query |
+| `.where(**conditions)` | Filter by field equality |
+| `.contains(text)` | Filter messages containing text |
+| `.matches(pattern)` | Filter messages matching regex |
+| `.since(time)` | Set start time |
+| `.before(time)` | Set end time |
+| `.between(start, end)` | Set time range |
+| `.past(amount, unit)` | Relative time (e.g., `past(30, :minutes)`) |
+| `.fields(*fields)` | Select fields to return |
+| `.limit(n)` | Limit number of results |
+| `.execute` | Run query and return ResultSet |
+| `.to_insights_query` | Return generated query string |
+
+### Result Methods
+
+| Method | Description |
+|--------|-------------|
+| `.timestamp` | Log event timestamp |
+| `.message` | Log message content |
+| `.[](key)` | Access field by name |
+| `.to_h` | Convert to hash |
+
+## Error Handling
+
+```ruby
+begin
+  results = CloudwatchQuery
+    .logs("my-log-group")
+    .past(1, :hours)
+    .execute
+rescue CloudwatchQuery::AuthError => e
+  puts "Authentication failed: #{e.message}"
+rescue CloudwatchQuery::QueryError => e
+  puts "Query failed: #{e.message}"
+rescue CloudwatchQuery::TimeoutError => e
+  puts "Query timed out: #{e.message}"
+rescue CloudwatchQuery::Error => e
+  puts "Error: #{e.message}"
+end
+```
+
+## License
+
+MIT

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/cloudwatch_query.gemspec

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "lib/cloudwatch_query/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "cloudwatch_query"
+  spec.version = CloudwatchQuery::VERSION
+  spec.authors = ["Igor Irianto"]
+
+  spec.summary = "A Ruby gem for querying AWS CloudWatch Logs with a simple, chainable interface"
+  spec.description = "Query AWS CloudWatch Logs using a fluent, ActiveRecord-style interface. " \
+                     "Supports chainable queries, automatic Insights query generation, and enumerable results."
+  spec.homepage = "https://github.com/iggredible/cloudwatch_query"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.7.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
+  spec.metadata["rubygems_mfa_required"] = "true"
+
+  spec.files = Dir.chdir(__dir__) do
+    Dir["lib/**/*", "LICENSE.txt", "README.md", "Rakefile", "cloudwatch_query.gemspec"]
+  end
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "aws-sdk-cloudwatchlogs", "~> 1.0"
+
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "webmock", "~> 3.0"
+end

data/lib/cloudwatch_query/client.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "aws-sdk-cloudwatchlogs"
+
+module CloudwatchQuery
+  class Client
+    DEFAULT_POLL_INTERVAL = 1
+    DEFAULT_TIMEOUT = 60
+
+    attr_reader :aws_client
+
+    def initialize(region: nil, profile: nil)
+      config = CloudwatchQuery.configuration
+      client_options = {
+        region: region || config.region
+      }
+
+      # AWS SDK automatically picks up credentials from:
+      # 1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+      # 2. Shared credentials file (~/.aws/credentials) - set by saml2aws
+      # 3. EC2 instance metadata
+      @aws_client = Aws::CloudWatchLogs::Client.new(client_options)
+    end
+
+    def execute(query_string:, log_group_names:, start_time:, end_time:, limit:)
+      query_id = start_query(
+        query_string: query_string,
+        log_group_names: log_group_names,
+        start_time: start_time,
+        end_time: end_time,
+        limit: limit
+      )
+
+      poll_results(query_id)
+    rescue Aws::CloudWatchLogs::Errors::ServiceError => e
+      handle_aws_error(e)
+    end
+
+    def list_log_groups(prefix: nil)
+      options = {}
+      options[:log_group_name_prefix] = prefix if prefix
+
+      groups = []
+      aws_client.describe_log_groups(options).each_page do |page|
+        groups.concat(page.log_groups.map(&:log_group_name))
+      end
+      groups
+    rescue Aws::CloudWatchLogs::Errors::ServiceError => e
+      handle_aws_error(e)
+    end
+
+    private
+
+    def start_query(query_string:, log_group_names:, start_time:, end_time:, limit:)
+      response = aws_client.start_query(
+        log_group_names: Array(log_group_names),
+        start_time: start_time,
+        end_time: end_time,
+        query_string: query_string,
+        limit: limit
+      )
+      response.query_id
+    end
+
+    def poll_results(query_id, timeout: DEFAULT_TIMEOUT)
+      deadline = Time.now + timeout
+
+      loop do
+        raise TimeoutError, "Query timed out after #{timeout} seconds" if Time.now > deadline
+
+        response = aws_client.get_query_results(query_id: query_id)
+
+        case response.status
+        when "Complete"
+          return build_result_set(response)
+        when "Failed", "Cancelled"
+          raise QueryError, "Query #{response.status.downcase}"
+        when "Running", "Scheduled"
+          sleep(DEFAULT_POLL_INTERVAL)
+        else
+          raise QueryError, "Unknown query status: #{response.status}"
+        end
+      end
+    end
+
+    def build_result_set(response)
+      results = response.results.map do |row|
+        row.each_with_object({}) do |field, hash|
+          key = field.field.sub(/^@/, "")
+          hash[key] = field.value
+        end
+      end
+
+      ResultSet.new(
+        results: results,
+        statistics: response.statistics&.to_h || {},
+        registry: CloudwatchQuery.parsers
+      )
+    end
+
+    def handle_aws_error(error)
+      case error
+      when Aws::CloudWatchLogs::Errors::AccessDeniedException,
+           Aws::CloudWatchLogs::Errors::UnauthorizedException
+        raise AuthError, "AWS authentication failed: #{error.message}"
+      else
+        raise QueryError, "AWS error: #{error.message}"
+      end
+    end
+  end
+end

data/lib/cloudwatch_query/configuration.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module CloudwatchQuery
+  class Configuration
+    attr_accessor :region, :profile, :default_limit, :default_time_range
+
+    def initialize
+      @region = ENV["AWS_REGION"] || "us-west-1"
+      @profile = ENV["AWS_PROFILE"]
+      @default_limit = 100
+      @default_time_range = 3600
+    end
+  end
+end

data/lib/cloudwatch_query/errors.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module CloudwatchQuery
+  class Error < StandardError; end
+  class QueryError < Error; end
+  class ConfigError < Error; end
+  class AuthError < Error; end
+  class TimeoutError < Error; end
+end

data/lib/cloudwatch_query/parsers/base.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module CloudwatchQuery
+  module Parsers
+    class Base
+      class << self
+        # Override in subclass - return true if this parser can handle the message
+        def matches?(message)
+          raise NotImplementedError, "#{name} must implement .matches?(message)"
+        end
+
+        # Override in subclass - parse the message and return a structured object
+        def parse(message)
+          raise NotImplementedError, "#{name} must implement .parse(message)"
+        end
+
+        # Human-readable name for this parser
+        def parser_name
+          name.split("::").last.sub(/Parser$/, "").gsub(/([a-z])([A-Z])/, '\1_\2').downcase
+        end
+      end
+    end
+  end
+end

data/lib/cloudwatch_query/parsers/rails/rails_log.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module CloudwatchQuery
+  module Parsers
+    module Rails
+      class RailsLog
+        attr_reader :request_id, :line_type, :server, :process_id,
+                    # Request fields
+                    :http_method, :path, :ip_address, :request_timestamp,
+                    # Parameters fields
+                    :params,
+                    # Redirect fields
+                    :redirect_url,
+                    # ActiveJob fields
+                    :job_class, :job_id, :queue, :arguments,
+                    # Processing fields
+                    :controller, :action, :format,
+                    # Completed fields
+                    :status_code, :duration_ms,
+                    # Raw message for unparsed line types
+                    :raw_message
+
+        def initialize(attributes = {})
+          attributes.each do |key, value|
+            instance_variable_set("@#{key}", value) if respond_to?(key)
+          end
+          @line_type ||= :unknown
+        end
+
+        def type
+          :rails
+        end
+
+        def request?
+          line_type == :request
+        end
+
+        def parameters?
+          line_type == :parameters
+        end
+
+        def redirect?
+          line_type == :redirect
+        end
+
+        def active_job?
+          line_type == :active_job
+        end
+
+        def processing?
+          line_type == :processing
+        end
+
+        def completed?
+          line_type == :completed
+        end
+
+        def to_h
+          instance_variables.each_with_object({}) do |var, hash|
+            key = var.to_s.delete("@").to_sym
+            value = instance_variable_get(var)
+            hash[key] = value unless value.nil?
+          end
+        end
+
+        def [](key)
+          instance_variable_get("@#{key}")
+        end
+      end
+    end
+  end
+end