fluent-plugin-upi-logs 0.0.0.dev.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 81c8794ebbc766b3fe68a87536f2c5d77c200bf119ab874f50cf1b44f491e778
+  data.tar.gz: e1fb3041e1ae302702311e30b21a5fe05d83a43dd3c63c5b3176433789897784
+SHA512:
+  metadata.gz: f96e75e64d6e36c3295a5b4f79d57194dee4d837b8856a674962bc193863725f44ba143d9a80ce9e080d104fcc4de263974ce8c7939c49107521369459d5bb1c
+  data.tar.gz: 6bb353e42319388113a438bccfdecdb53cc649f704f61a5851deaaab37bdc199598bc224e2966672dcab2b27abc431f0e4786cdd30ec8694c3cde8ecb48cb8b9

data/lib/fluent/plugin/parser_upi_logs.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/parser'
+
+require 'caraml/upi/v1/prediction_log_pb'
+require 'caraml/upi/v1/router_log_pb'
+
+module Fluent
+  module Plugin
+    # Parser for UPI which supports prediction and router logs
+    class UpiParser < Fluent::Plugin::Parser
+      # Parser name should be identical to "parser_#{name}" to follow fluentd plugin convention
+      Fluent::Plugin.register_parser('upi_logs', self)
+
+      config_param :class_name, :string
+
+      def configure(conf)
+        super
+        if Google::Protobuf::DescriptorPool.generated_pool.lookup(@class_name).nil?
+          raise Fluent::ConfigError, "unrecognised class_name '#{class_name}'"
+        end
+
+        # Lookup will only work for proto that are declared by the require statements
+        @protobuf_descriptor = Google::Protobuf::DescriptorPool.generated_pool.lookup(@class_name).msgclass
+      end
+
+      def parse(binary)
+        record = @protobuf_descriptor.decode(binary)
+        time = Fluent::EventTime.now
+        # Record are returned in json format
+        yield time, record.to_json
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-upi-logs
+version: !ruby/object:Gem::Version
+  version: 0.0.0.dev.1
+platform: ruby
+authors:
+- Caraml Dev
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-01-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.26
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.26
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.6
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.7
+- !ruby/object:Gem::Dependency
+  name: caraml-upi-protos
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: google-protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: ruby-protocol-buffers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+description: |-
+  Fluentd parser custom plugin that can parse UPI logs (PredictionLog and RouterLog
+     - https://github.com/caraml-dev/universal-prediction-interface) into json
+email:
+- caraml-dev@gojek.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fluent/plugin/parser_upi_logs.rb
+homepage: https://github.com/caraml-dev/timber
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: Fluent parser plugin for UPI logs into JSON
+test_files: []