logstruct 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a68693a793bd472926424e34f602e5f81a677e0bac2b0b8da189e7bc5adb10a
-  data.tar.gz: 8d5dd33db82a1b4eda1b8f1b0631af0cb3b04eba1e9d034e275192a49d8b9285
+  metadata.gz: 0b926657fc690c72031cf72973642c3654264b3ea348f2212a625788b62d1ade
+  data.tar.gz: 650b11e3b8b3e975feee474994ae9420b68da2eb933942f28cc495649d71f225
 SHA512:
-  metadata.gz: 2c8afb5d952a2ec1ead718b5655cd6af260978cd001502cd2dde3f8fb66e15720a5fca9f1646b8e132cad03ab48c51507b6a5d9948080019ca17ec4011b3b347
-  data.tar.gz: af94c2f9805ec7f121df828e25b1214395450acec0ed475a5b98ca26bef564c537c2bb522f3494152fb2df5d9b0b4850d7c158a3df794fad635a60613796fd51
+  metadata.gz: 2e0339abeafbea5bf239799dcf1ca843f57aae004e356e407a18909d9c0b6e4d1302c897f358be7bf539b5e0534efd032319a0b3055ff29e169c1d3f22557193
+  data.tar.gz: de385bc25a32f189044fbf1749e4511c1f9411d37a9af8628fc655e9706daba1a6b94b9482de756ef12557cc497deffa6f49a39ab9b434b3be6e25c7cb402214

data/CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.1] - 2025-09-29
+
+Added dotenv-rails integration. Many other fixes and improvements.
+
 ## [0.1.0] - 2025-09-07
 
-Initial release.
+Initial beta release.

data/README.md

@@ -48,9 +48,9 @@ Please see the [documentation](https://logstruct.com/docs/configuration/) for co
 
 ### Important Notes on Integration
 
-Once initialized, the gem automatically includes its modules into the appropriate base classes:
+Once initialized (and enabled), the gem automatically includes its modules into the appropriate base classes:
 
-- `ActiveSupport::TaggedLogging` is patched to support both Hashes and Strings
+- `ActiveSupport::TaggedLogging` is patched to support both Hashes and Strings (only when LogStruct is enabled)
 - `ActionMailer::Base` includes error handling and event logging modules
 - We configure `Lograge` for request logging
 - A Rack middleware is inserted to catch and log errors, including security violations (IP spoofing, CSRF, blocked hosts, etc.)
@@ -58,6 +58,19 @@ Once initialized, the gem automatically includes its modules into the appropriat
 - Rails `config.filter_parameters` are merged into LogStruct's filters and then cleared (to avoid double filtering). Configure sensitive keys via `LogStruct.config.filters`.
 - When `RAILS_LOG_TO_STDOUT` is set, we log to STDOUT only. Otherwise, we log to STDOUT by default without adding a file appender to avoid duplicate logs.
 
+### Development behavior
+
+- Disabled by default in development. Enable explicitly via `LOGSTRUCT_ENABLED=true` or `LogStruct.configure { |c| c.enabled = true }`.
+- When enabled in development, LogStruct now defaults to production‑style JSON output so you can preview exactly what will be shipped in prod.
+- You can opt back into the colorful human formatter with:
+
+```ruby
+LogStruct.configure do |c|
+  c.prefer_json_in_development = false
+  c.enable_color_output = true
+end
+```
+
 ## Documentation
 
 Please see the [documentation](https://logstruct.com/docs) for more details. (All code examples are type-checked and tested, and it's harder to keep a README up to date.)

data/lib/log_struct/boot_buffer.rb

@@ -0,0 +1,28 @@
+# typed: strict
+# frozen_string_literal: true
+
+module LogStruct
+  # Collects structured logs during very early boot before the logger is ready.
+  module BootBuffer
+    extend T::Sig
+
+    @@logs = T.let([], T::Array[LogStruct::Log::Interfaces::CommonFields])
+
+    sig { params(log: LogStruct::Log::Interfaces::CommonFields).void }
+    def self.add(log)
+      @@logs << log
+    end
+
+    sig { void }
+    def self.flush
+      return if @@logs.empty?
+      @@logs.each { |l| LogStruct.info(l) }
+      @@logs.clear
+    end
+
+    sig { void }
+    def self.clear
+      @@logs.clear
+    end
+  end
+end

data/lib/log_struct/builders/active_job.rb

@@ -0,0 +1,84 @@
+# typed: strict
+# frozen_string_literal: true
+
+require_relative "../log/active_job/enqueue"
+require_relative "../log/active_job/schedule"
+require_relative "../log/active_job/start"
+require_relative "../log/active_job/finish"
+
+module LogStruct
+  module Builders
+    module ActiveJob
+      extend T::Sig
+
+      sig { params(job: T.untyped).returns(T.nilable(String)) }
+      def self.safe_provider_job_id(job)
+        job.respond_to?(:provider_job_id) ? job.provider_job_id : nil
+      end
+
+      sig { params(job: T.untyped).returns(T.nilable(Integer)) }
+      def self.safe_executions(job)
+        job.respond_to?(:executions) ? job.executions : nil
+      end
+
+      sig { params(job: T.untyped).returns(T.nilable(T::Array[T.untyped])) }
+      def self.safe_arguments(job)
+        return nil unless job.class.respond_to?(:log_arguments?)
+        job.class.log_arguments? ? job.arguments : nil
+      end
+
+      sig { params(job: T.untyped).returns(Log::ActiveJob::Enqueue) }
+      def self.enqueue(job)
+        Log::ActiveJob::Enqueue.new(
+          job_id: job.job_id,
+          job_class: job.class.to_s,
+          queue_name: job.queue_name,
+          arguments: safe_arguments(job),
+          executions: safe_executions(job),
+          provider_job_id: safe_provider_job_id(job)
+        )
+      end
+
+      sig { params(job: T.untyped, scheduled_at: Time).returns(Log::ActiveJob::Schedule) }
+      def self.schedule(job, scheduled_at:)
+        Log::ActiveJob::Schedule.new(
+          job_id: job.job_id,
+          job_class: job.class.to_s,
+          queue_name: job.queue_name,
+          arguments: safe_arguments(job),
+          executions: safe_executions(job),
+          provider_job_id: safe_provider_job_id(job),
+          scheduled_at: scheduled_at
+        )
+      end
+
+      sig { params(job: T.untyped, started_at: Time, attempt: T.nilable(Integer)).returns(Log::ActiveJob::Start) }
+      def self.start(job, started_at:, attempt:)
+        Log::ActiveJob::Start.new(
+          job_id: job.job_id,
+          job_class: job.class.to_s,
+          queue_name: job.queue_name,
+          arguments: safe_arguments(job),
+          executions: safe_executions(job),
+          provider_job_id: safe_provider_job_id(job),
+          started_at: started_at,
+          attempt: attempt
+        )
+      end
+
+      sig { params(job: T.untyped, duration_ms: Float, finished_at: Time).returns(Log::ActiveJob::Finish) }
+      def self.finish(job, duration_ms:, finished_at:)
+        Log::ActiveJob::Finish.new(
+          job_id: job.job_id,
+          job_class: job.class.to_s,
+          queue_name: job.queue_name,
+          arguments: safe_arguments(job),
+          executions: safe_executions(job),
+          provider_job_id: safe_provider_job_id(job),
+          duration_ms: duration_ms,
+          finished_at: finished_at
+        )
+      end
+    end
+  end
+end

data/lib/log_struct/concerns/configuration.rb

@@ -41,16 +41,16 @@ module LogStruct
         def set_enabled_from_rails_env!
           # Set enabled based on current Rails environment and the LOGSTRUCT_ENABLED env var.
           # Precedence:
-          # 1. Check if LOGSTRUCT_ENABLED env var is defined
-          #    - Sets enabled=true only when value is "true"
-          #    - Sets enabled=false when value is "false" (or any non-"true")
+          # 1. Check if LOGSTRUCT_ENABLED env var is defined (not an empty string)
+          #    - Sets enabled=true only when value is "true", "yes", "1", etc.
+          #    - Sets enabled=false when value is any other value
           # 2. Otherwise, check if current Rails environment is in enabled_environments
           # 3. Otherwise, leave as config.enabled (defaults to true)
 
           # Then check if LOGSTRUCT_ENABLED env var is set
           config.enabled = if ENV["LOGSTRUCT_ENABLED"]
             # Override to true only if env var is "true"
-            ENV["LOGSTRUCT_ENABLED"] == "true"
+            %w[true t yes y 1].include?(ENV["LOGSTRUCT_ENABLED"]&.strip&.downcase)
           else
             config.enabled_environments.include?(::Rails.env.to_sym)
           end
@@ -72,20 +72,133 @@ module LogStruct
 
           rails_filter_params = ::Rails.application.config.filter_parameters
           return unless rails_filter_params.is_a?(Array)
+          return if rails_filter_params.empty?
+
+          symbol_filters = T.let([], T::Array[Symbol])
+          matchers = T.let([], T::Array[ConfigStruct::FilterMatcher])
+          leftovers = T.let([], T::Array[T.untyped])
+
+          rails_filter_params.each do |entry|
+            matcher = build_filter_matcher(entry)
+
+            if matcher
+              matchers << matcher
+              next
+            end
+
+            normalized_symbol = normalize_filter_symbol(entry)
+            if normalized_symbol
+              symbol_filters << normalized_symbol
+            else
+              leftovers << entry
+            end
+          end
+
+          if symbol_filters.any?
+            config.filters.filter_keys |= symbol_filters
+          end
 
-          # Convert all Rails filter parameters to symbols and merge with our filter keys
-          converted_params = rails_filter_params.map do |param|
-            param.respond_to?(:to_sym) ? param.to_sym : param
+          if matchers.any?
+            matchers.each do |matcher|
+              existing = config.filters.filter_matchers.any? do |registered|
+                registered.label == matcher.label
+              end
+              config.filters.filter_matchers << matcher unless existing
+            end
           end
 
-          # Add Rails filter parameters to our filter keys
-          config.filters.filter_keys += converted_params
+          replace_filter_parameters(rails_filter_params, leftovers)
+        end
+
+        private
+
+        sig { params(filter: T.untyped).returns(T.nilable(Symbol)) }
+        def normalize_filter_symbol(filter)
+          return filter if filter.is_a?(Symbol)
+          return filter.downcase.to_sym if filter.is_a?(String)
 
-          # Ensure no duplicates
-          config.filters.filter_keys.uniq!
+          return nil unless filter.respond_to?(:to_sym)
+
+          begin
+            sym = filter.to_sym
+            sym.is_a?(Symbol) ? sym : nil
+          rescue
+            nil
+          end
+        end
+
+        sig { params(filter: T.untyped).returns(T.nilable(ConfigStruct::FilterMatcher)) }
+        def build_filter_matcher(filter)
+          case filter
+          when ::Regexp
+            callable = Kernel.lambda do |key, _value|
+              filter.match?(key)
+            end
+            return ConfigStruct::FilterMatcher.new(callable: callable, label: filter.inspect)
+          else
+            return build_callable_filter_matcher(filter) if callable_filter?(filter)
+          end
+
+          nil
+        end
+
+        sig { params(filter: T.untyped).returns(T::Boolean) }
+        def callable_filter?(filter)
+          filter.respond_to?(:call)
+        end
+
+        sig { params(filter: T.untyped).returns(T.nilable(ConfigStruct::FilterMatcher)) }
+        def build_callable_filter_matcher(filter)
+          callable = Kernel.lambda do |key, value|
+            call_args = case arity_for_filter(filter)
+            when 0
+              []
+            when 1
+              [key]
+            else
+              [key, value]
+            end
+
+            result = filter.call(*call_args)
+            !!result
+          rescue ArgumentError
+            begin
+              !!filter.call(key)
+            rescue => e
+              handle_filter_error(e, filter, key)
+              false
+            end
+          rescue => e
+            handle_filter_error(e, filter, key)
+            false
+          end
+          ConfigStruct::FilterMatcher.new(callable: callable, label: filter.inspect)
+        end
+
+        sig { params(filter: T.untyped).returns(Integer) }
+        def arity_for_filter(filter)
+          filter.respond_to?(:arity) ? filter.arity : 2
+        end
+
+        sig { params(filter_params: T::Array[T.untyped], leftovers: T::Array[T.untyped]).void }
+        def replace_filter_parameters(filter_params, leftovers)
+          filter_params.clear
+          filter_params.concat(leftovers)
+        end
 
-          # Clear Rails filter parameters since we've incorporated them
-          ::Rails.application.config.filter_parameters.clear
+        sig { params(error: StandardError, filter: T.untyped, key: String).void }
+        def handle_filter_error(error, filter, key)
+          context = {
+            filter: filter.class.name,
+            key: key,
+            filter_label: begin
+              filter.inspect
+            rescue
+              "unknown"
+            end
+          }
+
+          LogStruct.handle_exception(error, source: Source::Internal, context: context)
         end
       end
     end

data/lib/log_struct/concerns/error_handling.rb

@@ -21,12 +21,12 @@ module LogStruct
           case source
           when Source::TypeChecking
             config.error_handling_modes.type_checking_errors
-          when Source::LogStruct
+          when Source::Internal
             config.error_handling_modes.logstruct_errors
           when Source::Security
             config.error_handling_modes.security_errors
           when Source::Rails, Source::App, Source::Job, Source::Storage, Source::Mailer,
-               Source::Shrine, Source::CarrierWave, Source::Sidekiq
+               Source::Shrine, Source::CarrierWave, Source::Sidekiq, Source::Dotenv, Source::Puma
             config.error_handling_modes.standard_errors
           else
             # Ensures the case statement is exhaustive
@@ -38,11 +38,7 @@ module LogStruct
         sig { params(error: StandardError, source: Source, context: T.nilable(T::Hash[Symbol, T.untyped])).void }
         def log_error(error, source:, context: nil)
           # Create structured log entry
-          error_log = Log::Error.from_exception(
-            source,
-            error,
-            context || {}
-          )
+          error_log = Log.from_exception(source, error, context || {})
           LogStruct.error(error_log)
         end
 

data/lib/log_struct/config_struct/filters.rb

@@ -3,6 +3,18 @@
 
 module LogStruct
   module ConfigStruct
+    class FilterMatcher < T::Struct
+      extend T::Sig
+
+      const :callable, T.proc.params(key: String, value: T.untyped).returns(T::Boolean)
+      const :label, String
+
+      sig { params(key: String, value: T.untyped).returns(T::Boolean) }
+      def matches?(key, value)
+        callable.call(key, value)
+      end
+    end
+
     class Filters < T::Struct
       include Sorbet::SerializeSymbolKeys
 
@@ -75,6 +87,12 @@ module LogStruct
       # Filter MAC addresses
       # Default: false
       prop :mac_addresses, T::Boolean, default: false
+
+      # Additional filter matchers built from Rails filter_parameters entries that aren't simple symbols.
+      # Each matcher receives the key (String) and optional value, returning true when the pair should be filtered.
+      prop :filter_matchers,
+        T::Array[FilterMatcher],
+        factory: -> { [] }
     end
   end
 end

data/lib/log_struct/config_struct/integrations.rb

@@ -60,18 +60,6 @@ module LogStruct
       # Default: true
       prop :enable_semantic_logger, T::Boolean, default: true
 
-      # Enable colored JSON output in development
-      # Default: true
-      prop :enable_color_output, T::Boolean, default: true
-
-      # Color configuration for JSON output
-      # Default: nil (uses SemanticLogger defaults)
-      prop :color_map, T.nilable(T::Hash[Symbol, Symbol]), default: nil
-
-      # Filter noisy loggers (ActionView, etc.)
-      # Default: false
-      prop :filter_noisy_loggers, T::Boolean, default: false
-
       # Enable SQL query logging through ActiveRecord instrumentation
       # Default: false (can be resource intensive)
       prop :enable_sql_logging, T::Boolean, default: false
@@ -92,6 +80,14 @@ module LogStruct
       # Enable ActiveModelSerializers integration
       # Default: true (safe no-op unless ActiveModelSerializers is defined)
       prop :enable_active_model_serializers, T::Boolean, default: true
+
+      # Enable dotenv-rails integration (convert to structured logs)
+      # Default: true
+      prop :enable_dotenv, T::Boolean, default: true
+
+      # Enable Puma integration (convert server lifecycle logs)
+      # Default: true
+      prop :enable_puma, T::Boolean, default: true
     end
   end
 end

data/lib/log_struct/configuration.rb

@@ -20,6 +20,19 @@ module LogStruct
     prop :enabled, T::Boolean, default: true
     prop :enabled_environments, T::Array[Symbol], factory: -> { [:test, :production] }
     prop :local_environments, T::Array[Symbol], factory: -> { [:development, :test] }
+
+    # Prefer production-style JSON in development when LogStruct is enabled
+    prop :prefer_json_in_development, T::Boolean, default: true
+
+    # Enable colorful human formatter in development
+    prop :enable_color_output, T::Boolean, default: true
+
+    # Custom color map for the color formatter
+    prop :color_map, T.nilable(T::Hash[Symbol, Symbol]), default: nil
+
+    # Filter noisy loggers (ActionView, etc.)
+    prop :filter_noisy_loggers, T::Boolean, default: false
+
     const :integrations, ConfigStruct::Integrations, factory: -> { ConfigStruct::Integrations.new }
     const :filters, ConfigStruct::Filters, factory: -> { ConfigStruct::Filters.new }
 

data/lib/log_struct/enums/event.rb

@@ -26,10 +26,23 @@ module LogStruct
       Stream = new(:stream)
       Url = new(:url)
 
+      # Data generation events
+      Generate = new(:generate)
+
       # Email events
       Delivery = new(:delivery)
       Delivered = new(:delivered)
 
+      # Configuration / boot events
+      Load = new(:load)
+      Update = new(:update)
+      Save = new(:save)
+      Restore = new(:restore)
+
+      # Server lifecycle (e.g., Puma)
+      # Start already defined above
+      Shutdown = new(:shutdown)
+
       # Security events
       IPSpoof = new(:ip_spoof)
       CSRFViolation = new(:csrf_violation)

data/lib/log_struct/enums/log_field.rb

@@ -0,0 +1,154 @@
+# typed: strict
+# frozen_string_literal: true
+
+# NOTE:
+# - This enum defines human‑readable field names (constants) that map to compact
+#   JSON key symbols via `serialize` (e.g., Database => :db).
+# - The enum constant names are code‑generated into
+#   `schemas/meta/log-fields.json` by `scripts/generate_structs.rb` and
+#   referenced from `schemas/meta/log-source-schema.json` to strictly validate
+#   field keys in `schemas/log_sources/*`.
+# - When adding or renaming fields here, run the generator so schema validation
+#   stays in sync.
+#
+# Use human-readable field names as the enum values and short field names for the JSON properties
+
+module LogStruct
+  class LogField < T::Enum
+    enums do
+      # Shared fields
+      Source = new(:src)
+      Event = new(:evt)
+      Timestamp = new(:ts)
+      Level = new(:lvl)
+
+      # Common fields
+      Message = new(:msg)
+      Data = new(:data)
+
+      # Request-related fields
+      Path = new(:path)
+      HttpMethod = new(:method) # property name was http_method
+      SourceIp = new(:source_ip)
+      UserAgent = new(:user_agent)
+      Referer = new(:referer)
+      RequestId = new(:request_id)
+
+      # HTTP-specific fields
+      Format = new(:format)
+      Controller = new(:controller)
+      Action = new(:action)
+      Status = new(:status)
+      # DurationMs already defined below for general metrics
+      View = new(:view)
+      Database = new(:db)
+      Params = new(:params)
+
+      # Security-specific fields
+      BlockedHost = new(:blocked_host)
+      BlockedHosts = new(:blocked_hosts)
+      ClientIp = new(:client_ip)
+      XForwardedFor = new(:x_forwarded_for)
+
+      # Email-specific fields
+      To = new(:to)
+      From = new(:from)
+      Subject = new(:subject)
+
+      # Error fields
+      ErrClass = new(:err_class)
+      Backtrace = new(:backtrace)
+
+      # Job-specific fields
+      JobId = new(:job_id)
+      JobClass = new(:job_class)
+      QueueName = new(:queue_name)
+      Arguments = new(:arguments)
+      RetryCount = new(:retry_count)
+      Retries = new(:retries)
+      Attempt = new(:attempt)
+      Executions = new(:executions)
+      ExceptionExecutions = new(:exception_executions)
+      ProviderJobId = new(:provider_job_id)
+      ScheduledAt = new(:scheduled_at)
+      StartedAt = new(:started_at)
+      FinishedAt = new(:finished_at)
+      DurationMs = new(:duration_ms)
+      WaitMs = new(:wait_ms)
+      # Deprecated: ExecutionTime/WaitTime/RunTime
+      ExecutionTime = new(:execution_time)
+      WaitTime = new(:wait_time)
+      RunTime = new(:run_time)
+      Priority = new(:priority)
+      CronKey = new(:cron_key)
+      ErrorMessage = new(:error_message)
+
+      # Dotenv fields
+      File = new(:file)
+      Vars = new(:vars)
+      Snapshot = new(:snapshot)
+
+      # Sidekiq-specific fields
+      ProcessId = new(:pid)
+      ThreadId = new(:tid)
+      Context = new(:ctx)
+
+      # Storage-specific fields (ActiveStorage)
+      Checksum = new(:checksum)
+      Exist = new(:exist)
+      Url = new(:url)
+      Prefix = new(:prefix)
+      Range = new(:range)
+
+      # Storage-specific fields (Shrine)
+      Storage = new(:storage)
+      Operation = new(:op)
+      FileId = new(:file_id)
+      Filename = new(:filename)
+      MimeType = new(:mime_type)
+      Size = new(:size)
+      Metadata = new(:metadata)
+      Location = new(:location)
+      UploadOptions = new(:upload_opts)
+      DownloadOptions = new(:download_opts)
+      Options = new(:opts)
+      Uploader = new(:uploader)
+
+      # CarrierWave-specific fields
+      Model = new(:model)
+      MountPoint = new(:mount_point)
+
+      # SQL-specific fields
+      Sql = new(:sql)
+      Name = new(:name)
+      RowCount = new(:row_count)
+      # Use Adapter for both AMS and SQL adapter name
+      BindParams = new(:bind_params)
+      DatabaseName = new(:db_name)
+      ConnectionPoolSize = new(:pool_size)
+      ActiveConnections = new(:active_count)
+      OperationType = new(:op_type)
+      TableNames = new(:table_names)
+
+      # ActiveModelSerializers fields
+      Serializer = new(:serializer)
+      Adapter = new(:adapter)
+      ResourceClass = new(:resource_class)
+
+      # Ahoy-specific fields
+      AhoyEvent = new(:ahoy_event)
+      Properties = new(:properties)
+
+      # Puma / server lifecycle fields
+      Mode = new(:mode)
+      PumaVersion = new(:puma_version)
+      PumaCodename = new(:puma_codename)
+      RubyVersion = new(:ruby_version)
+      MinThreads = new(:min_threads)
+      MaxThreads = new(:max_threads)
+      Environment = new(:environment)
+      ListeningAddresses = new(:listening_addresses)
+      Address = new(:addr)
+    end
+  end
+end

data/lib/log_struct/enums/source.rb

@@ -7,8 +7,9 @@ module LogStruct
     enums do
       # Error sources
       TypeChecking = new(:type_checking) # For type checking errors (Sorbet)
-      LogStruct = new(:logstruct) # Errors from LogStruct itself
       Security = new(:security) # Security-related events
+      # Errors from LogStruct. (Cannot use LogStruct here because it confuses tapioca.)
+      Internal = new(:logstruct)
 
       # Application sources
       Rails = new(:rails) # For request-related logs/errors
@@ -21,6 +22,8 @@ module LogStruct
       Shrine = new(:shrine)
       CarrierWave = new(:carrierwave)
       Sidekiq = new(:sidekiq)
+      Dotenv = new(:dotenv)
+      Puma = new(:puma)
     end
   end
 end