julewire-core 1.0.0

data/lib/julewire/core/cli/log_formats/core_json_decoder.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    class CLI
+      module LogFormats
+        module CoreJsonDecoder
+          class << self
+            CORE_KINDS = {
+              "point" => true,
+              "summary" => true
+            }.freeze
+
+            def match?(payload)
+              payload.key?("timestamp") &&
+                payload.key?("severity") &&
+                CORE_KINDS.key?(payload["kind"].to_s)
+            end
+
+            def call(payload)
+              record_base(payload).merge(record_sections(payload), error: RecordDecoder.error(payload["error"]))
+            end
+
+            private
+
+            def record_base(source)
+              {
+                timestamp: source["timestamp"],
+                severity: Records::Severity.normalize(source["severity"] || :info),
+                kind: RecordDecoder.kind(source["kind"] || :point),
+                event: source["event"],
+                message: source["message"],
+                logger: source["logger"],
+                source: source["source"]
+              }
+            end
+
+            def record_sections(source)
+              RecordDecoder.sections(source)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/cli/log_formats/core_json_encoder.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    class CLI
+      module LogFormats
+        module CoreJsonEncoder
+          class << self
+            def call(record)
+              json_encoder.call(Records::Formatter.new.call(record))
+            end
+
+            private
+
+            def json_encoder = @json_encoder ||= Serialization::JsonEncoder.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/cli/log_formats/record_decoder.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    class CLI
+      module LogFormats
+        module RecordDecoder
+          class << self
+            def kind(value)
+              case value.to_s
+              when "point" then :point
+              when "summary" then :summary
+              else value.to_sym
+              end
+            end
+
+            def section(value)
+              return {} unless value.is_a?(Hash)
+
+              Fields::FieldSet.deep_symbolize_keys(value)
+            end
+
+            def sections(source, sections: Fields::Bags.record_hash_sections)
+              sections.to_h do |name|
+                value = block_given? ? yield(name, source) : source[name.to_s]
+
+                [name, section(value)]
+              end
+            end
+
+            def error(value)
+              section(value) unless value.nil?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/cli/log_formats.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    class CLI
+      module LogFormats
+        Entry = Data.define(:name, :decoder, :encoder, :priority)
+        AUTO_FORMAT = :auto
+        FORMAT_NAME_PATTERN = /\A[a-z][a-z0-9_]*\z/
+
+        @entries = []
+
+        class << self
+          def register(name, decoder: nil, encoder: nil, priority: 0)
+            name = normalize(name)
+            validate_component(decoder, :decoder) if decoder
+            validate_component(encoder, :encoder) if encoder
+            existing = @entries.find { it.name == name }
+            priority = priority.to_i
+            entry = Entry.new(
+              name: name,
+              decoder: decoder || existing&.decoder,
+              encoder: encoder || existing&.encoder,
+              priority: priority.zero? && existing ? existing.priority : priority
+            )
+            @entries = @entries.reject { it.name == name } + [entry]
+            entry
+          end
+
+          def decode(payload, format: AUTO_FORMAT)
+            raise TypeError, "log entry must be a JSON object" unless payload.is_a?(Hash)
+
+            format = normalize(format)
+            entry = format == AUTO_FORMAT ? auto_decode_entry(payload) : named_decode_entry(format, payload)
+            entry.decoder.call(payload)
+          end
+
+          def encode(record, format:)
+            name = normalize(format)
+            entry = named_encode_entry(name)
+            entry.encoder.call(record)
+          end
+
+          def record_from_json_line(line, line_number:, format: AUTO_FORMAT)
+            payload = JSON.parse(line)
+            Records::Record.from_normalized_hash(decode(payload, format: format))
+          rescue JSON::ParserError => e
+            raise ArgumentError, "line #{line_number}: invalid JSON: #{e.message}"
+          rescue TypeError, ArgumentError => e
+            raise ArgumentError, "line #{line_number}: #{e.message}"
+          end
+
+          def normalize(value)
+            name = Core.normalize_name(value, name: "log format")
+            return name if name.to_s.match?(FORMAT_NAME_PATTERN)
+
+            raise ArgumentError, "log format must contain lowercase letters, digits, or underscores"
+          end
+
+          def load(name)
+            path = "julewire/#{name}"
+            require path
+          rescue LoadError => e
+            raise unless e.path == path
+          end
+
+          private
+
+          def validate_component(component, name)
+            Validation.validate_callable!(component, name: name)
+          end
+
+          def auto_decode_entry(payload)
+            entry = decode_entries.find { decoder_match?(it.decoder, payload) }
+            raise TypeError, "no log decoder accepted JSON object" unless entry
+
+            entry
+          end
+
+          def named_decode_entry(name, payload)
+            load_format(name)
+            entry = @entries.find { it.name == name && it.decoder }
+            raise ArgumentError, "log format #{name} is not available" unless entry
+            unless decoder_match?(entry.decoder, payload)
+              raise TypeError, "log format #{name} did not accept JSON object"
+            end
+
+            entry
+          end
+
+          def named_encode_entry(name)
+            load_format(name)
+            entry = @entries.find { it.name == name && it.encoder }
+            raise ArgumentError, "log format #{name} is not available" unless entry
+
+            entry
+          end
+
+          def load_format(name)
+            return if @entries.any? { it.name == name }
+
+            load(name)
+          end
+
+          def decode_entries
+            @entries.select(&:decoder).sort_by { [-it.priority, @entries.index(it)] }
+          end
+
+          def decoder_match?(decoder, payload)
+            return decoder.match?(payload) if decoder.respond_to?(:match?)
+
+            true
+          rescue StandardError
+            false
+          end
+        end
+
+        register(:core, decoder: CoreJsonDecoder, encoder: CoreJsonEncoder)
+        register(:console, encoder: ConsoleText.new)
+      end
+    end
+  end
+end

data/lib/julewire/core/cli/tail.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    class CLI
+      class Tail
+        include LineHelpers
+
+        DEFAULT_MAX_VALUE_BYTES = Serialization::TextEncoder::DEFAULT_MAX_VALUE_BYTES
+        DEFAULT_POLL_INTERVAL = 0.1
+        FLAGS = {
+          "--color" => [:color, true],
+          "--no-color" => [:color, false],
+          "--follow" => [:follow, true],
+          "--once" => [:follow, false],
+          "--plain" => %i[theme plain],
+          "--punk" => %i[theme punk],
+          "--skip-invalid" => %i[invalid skip],
+          "--raw-invalid" => %i[invalid raw]
+        }.freeze
+
+        def initialize(argv:, stdin:, stdout:)
+          @argv = argv
+          @stdin = stdin
+          @stdout = stdout
+        end
+
+        def call
+          options = tail_options
+          renderer = tail_renderer(options)
+          options.fetch(:path) == "-" ? tail_stdin(options, renderer) : tail_file(options, renderer)
+          0
+        end
+
+        private
+
+        def tail_options
+          parse_command_options(default_tail_options, command: "tail") do |options, value|
+            apply_tail_option(options, value)
+          end
+        end
+
+        def default_tail_options
+          {
+            color: @stdout.respond_to?(:tty?) && @stdout.tty?,
+            format: :auto,
+            follow: true,
+            invalid: :fail,
+            limit: nil,
+            max_value_bytes: DEFAULT_MAX_VALUE_BYTES,
+            path: nil,
+            poll_interval: DEFAULT_POLL_INTERVAL,
+            theme: :plain
+          }
+        end
+
+        def apply_tail_option(options, value)
+          if (assignment = FLAGS[value])
+            options[assignment.fetch(0)] = assignment.fetch(1)
+          elsif value.start_with?("--format=")
+            options[:format] = value.delete_prefix("--format=").to_sym
+          elsif value == "--format"
+            options[:format] = next_symbol_option("--format")
+          elsif value == "--theme"
+            options[:theme] = next_symbol_option("--theme")
+          elsif value == "--limit"
+            options[:limit] = positive_integer_option("--limit")
+          elsif value == "--max-value-bytes"
+            options[:max_value_bytes] = positive_integer_option("--max-value-bytes")
+          else
+            apply_path_option(options, value, command: "tail")
+          end
+        end
+
+        def tail_renderer(options)
+          encoder = console_text_encoder(options)
+          proc do |line, line_number|
+            write_encoded_record_line(
+              line,
+              line_number,
+              input_format: options.fetch(:format),
+              invalid: options.fetch(:invalid),
+              encoder: encoder
+            )
+          end
+        end
+
+        def tail_stdin(options, renderer)
+          limit = options.fetch(:limit)
+          return render_limited_stdin(limit, renderer) if limit
+
+          render_stream(@stdin.each_line, renderer)
+        end
+
+        def tail_file(options, renderer)
+          File.open(options.fetch(:path), "r") do |file|
+            line_number = render_file_snapshot(file, options, renderer)
+            follow_file(file, line_number, options, renderer) if options.fetch(:follow)
+          end
+        end
+
+        def render_file_snapshot(file, options, renderer)
+          entries = indexed_lines(file.each_line)
+          render_entries(limit_entries(entries, options.fetch(:limit)), renderer)
+          file.seek(0, IO::SEEK_END)
+          entries.last&.fetch(0) || 0
+        end
+
+        def follow_file(file, line_number, options, renderer)
+          loop do
+            if (line = file.gets)
+              line_number += 1
+              render_entries([[line_number, line]], renderer)
+            else
+              line_number = reset_follow_position(file) if file.stat.size < file.pos
+              sleep(options.fetch(:poll_interval))
+            end
+          end
+        end
+
+        def reset_follow_position(file)
+          file.seek(0)
+          0
+        end
+
+        def render_limited_stdin(limit, renderer)
+          entries = indexed_lines(@stdin.each_line)
+          render_entries(limit_entries(entries, limit), renderer)
+        end
+
+        def render_stream(lines, renderer)
+          line_number = 0
+          lines.each do |line|
+            line_number += 1
+            next if line.strip.empty?
+
+            renderer.call(line, line_number)
+          end
+        end
+
+        def limit_entries(entries, limit)
+          limit ? entries.last(limit) : entries
+        end
+
+        def render_entries(entries, renderer)
+          entries.each do |line_number, line|
+            renderer.call(line, line_number)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/cli/transcode.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    class CLI
+      class Transcode
+        include LineHelpers
+
+        DEFAULT_MAX_VALUE_BYTES = Serialization::TextEncoder::DEFAULT_MAX_VALUE_BYTES
+        FLAGS = {
+          "--color" => [:color, true],
+          "--no-color" => [:color, false],
+          "--plain" => %i[theme plain],
+          "--punk" => %i[theme punk],
+          "--skip-invalid" => %i[invalid skip],
+          "--raw-invalid" => %i[invalid raw]
+        }.freeze
+
+        def initialize(argv:, stdin:, stdout:)
+          @argv = argv
+          @stdin = stdin
+          @stdout = stdout
+        end
+
+        def call
+          options = transcode_options
+          encoder = encoder_for(options)
+          each_entry(options.fetch(:path)) do |line_number, line|
+            write_encoded_record_line(
+              line,
+              line_number,
+              input_format: options.fetch(:from),
+              invalid: options.fetch(:invalid),
+              encoder: encoder
+            )
+          end
+          0
+        end
+
+        private
+
+        def transcode_options
+          parse_command_options(default_transcode_options, command: "transcode") do |options, value|
+            apply_transcode_option(options, value)
+          end
+        end
+
+        def default_transcode_options
+          {
+            color: @stdout.respond_to?(:tty?) && @stdout.tty?,
+            from: :auto,
+            invalid: :fail,
+            max_value_bytes: DEFAULT_MAX_VALUE_BYTES,
+            path: nil,
+            theme: :plain,
+            to: :core
+          }
+        end
+
+        def apply_transcode_option(options, value)
+          if (assignment = FLAGS[value])
+            options[assignment.fetch(0)] = assignment.fetch(1)
+          elsif value.start_with?("--")
+            apply_named_option(options, value)
+          else
+            apply_path_option(options, value, command: "transcode")
+          end
+        end
+
+        def apply_named_option(options, value)
+          if value.start_with?("--from=")
+            options[:from] = value.delete_prefix("--from=").to_sym
+          elsif value.start_with?("--to=")
+            options[:to] = value.delete_prefix("--to=").to_sym
+          else
+            apply_separate_option(options, value)
+          end
+        end
+
+        def apply_separate_option(options, value)
+          case value
+          when "--from" then options[:from] = next_symbol_option("--from")
+          when "--to" then options[:to] = next_symbol_option("--to")
+          when "--theme" then options[:theme] = next_symbol_option("--theme")
+          when "--max-value-bytes" then options[:max_value_bytes] = positive_integer_option("--max-value-bytes")
+          else
+            apply_path_option(options, value, command: "transcode")
+          end
+        end
+
+        def encoder_for(options)
+          return console_text_encoder(options) if options.fetch(:to) == :console
+
+          ->(record) { LogFormats.encode(record, format: options.fetch(:to)) }
+        end
+
+        def each_entry(path, &)
+          return indexed_lines(@stdin.each_line).each(&) if path == "-"
+
+          File.open(path, "r") { |file| indexed_lines(file.each_line).each(&) }
+        end
+      end
+    end
+  end
+end

data/lib/julewire/core/cli.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    class CLI
+      INTERRUPTED_STATUS = 130
+
+      class << self
+        def call(argv: ARGV, stdin: $stdin, stdout: $stdout, stderr: $stderr)
+          new(argv: argv, stdin: stdin, stdout: stdout, stderr: stderr).call
+        end
+      end
+
+      def initialize(argv:, stdin:, stdout:, stderr:)
+        @argv = argv.dup
+        @stdin = stdin
+        @stdout = stdout
+        @stderr = stderr
+      end
+
+      def call
+        case command = @argv.shift
+        when "tail" then tail
+        when "transcode" then transcode
+        when "doctor" then doctor
+        when "-v", "--version", "version" then version
+        when nil, "-h", "--help", "help" then help
+        else
+          fail_with("unknown command #{command.inspect}")
+        end
+      rescue Interrupt
+        INTERRUPTED_STATUS
+      rescue ArgumentError, Errno::ENOENT => e
+        fail_with(e.message)
+      end
+
+      private
+
+      def tail
+        Tail.new(argv: @argv, stdin: @stdin, stdout: @stdout).call
+      end
+
+      def transcode
+        Transcode.new(argv: @argv, stdin: @stdin, stdout: @stdout).call
+      end
+
+      def doctor
+        Doctor.new(argv: @argv, stdout: @stdout).call
+      end
+
+      def version
+        @stdout.write("julewire #{Core::VERSION}\n")
+        0
+      end
+
+      def help
+        @stdout.write(<<~HELP)
+          Usage:
+            julewire tail [--follow|--once] [--format auto|core|NAME] [--skip-invalid|--raw-invalid] [--color|--no-color] [--theme plain|punk|--plain|--punk] [--limit N] [--max-value-bytes N] LOGFILE|-
+            julewire transcode [--from auto|core|NAME] [--to core|console|NAME] [--skip-invalid|--raw-invalid] [--color|--no-color] [--theme plain|punk|--plain|--punk] [--max-value-bytes N] LOGFILE|-
+            julewire doctor [--json|--text|--punk] [--color|--no-color]
+            julewire --version
+        HELP
+        0
+      end
+
+      def fail_with(message)
+        @stderr.write("julewire: #{message}\n")
+        1
+      end
+    end
+  end
+end

data/lib/julewire/core/configuration.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Core
+    class Configuration
+      ATTRIBUTES = %i[
+        destinations
+        emit_non_standard_exception_summaries
+        error_backtrace_lines
+        labels
+        level
+        on_drop
+        on_failure
+        pipeline_close_timeout
+        processors
+      ].freeze
+      REGISTRY_ATTRIBUTES = %i[destinations labels processors].freeze
+      SCALAR_ATTRIBUTES = (ATTRIBUTES - REGISTRY_ATTRIBUTES).freeze
+
+      attr_accessor(*SCALAR_ATTRIBUTES)
+      attr_reader(*REGISTRY_ATTRIBUTES)
+
+      def initialize(**options)
+        reject_unknown_options!(options)
+        @destinations = options.fetch(:destinations) { Destinations::Registry.new }
+        @emit_non_standard_exception_summaries = options.fetch(:emit_non_standard_exception_summaries, false)
+        @error_backtrace_lines = options.fetch(:error_backtrace_lines, Core::MAX_BACKTRACE_LINES)
+        @labels = options.fetch(:labels) { Fields::StaticLabels.new }
+        @level = options.fetch(:level, :debug)
+        @on_drop = options.fetch(:on_drop, nil)
+        @on_failure = options.fetch(:on_failure, nil)
+        @pipeline_close_timeout = options.fetch(:pipeline_close_timeout, 1)
+        @processors = options.fetch(:processors) { Processing::ProcessorRegistry.new }
+      end
+
+      def validate!
+        validate_contracts!
+        Validation.validate_non_negative_integer!(
+          error_backtrace_lines,
+          name: :error_backtrace_lines
+        )
+        Validation.validate_timeout!(
+          pipeline_close_timeout,
+          name: :pipeline_close_timeout
+        )
+        Records::Severity.normalize(level)
+        self
+      end
+
+      def snapshot
+        validate!
+        copy.tap do |configuration|
+          configuration.level = Records::Severity.normalize(level)
+          configuration.freeze
+        end
+      end
+
+      def copy
+        self.class.new(**copy_options)
+      end
+
+      def build_pipeline(invalid_severity_reporter: Diagnostics::InvalidSeverityReporter.counter)
+        Processing::Pipeline.new(configuration: self, invalid_severity_reporter: invalid_severity_reporter)
+      end
+
+      def freeze
+        destinations.freeze
+        labels.freeze
+        processors.freeze
+        super
+      end
+
+      private
+
+      def reject_unknown_options!(options)
+        Validation.validate_options!(options, ATTRIBUTES, name: :configuration)
+      end
+
+      def validate_contracts!
+        Validation.validate_callable!(on_drop, name: :on_drop, allow_nil: true)
+        Validation.validate_callable!(on_failure, name: :on_failure, allow_nil: true)
+      end
+
+      def copy_options
+        {
+          destinations: destinations.copy,
+          emit_non_standard_exception_summaries: emit_non_standard_exception_summaries,
+          error_backtrace_lines: error_backtrace_lines,
+          labels: labels.copy,
+          level: level,
+          on_drop: on_drop,
+          on_failure: on_failure,
+          pipeline_close_timeout: pipeline_close_timeout,
+          processors: processors.copy
+        }
+      end
+    end
+  end
+end