polyn 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b87ad4ec53f26f70e9c6bb82ee6605ef19a9fa45452f312533fc43db4b91d03d
-  data.tar.gz: ab02358b23de26f8db8c744d70d5d59e4d11ce0792c766e3de8b65af3acde143
+  metadata.gz: f3ccd094b68ccf93066ade4f7b3348e1048be916d1f220c4602abbf1884855f6
+  data.tar.gz: 06c13d39dbf4804d41a57754816338271ff2b1cdfdb4332cddfd46f07ecc3451
 SHA512:
-  metadata.gz: aea99bcf38e0a88f744eabb15340ab0c64761f91b052549fc933f48425e8a5d9d36bfec50ac9a8178b446056a95cd9b4f33e24226194f06fec9f16a77f483dba
-  data.tar.gz: 23bdb00776b3c3dfd4b07ed52b291fcb9842c61549a4ed58f9696357da1d292e29fb6365ea7eadac5bf9b0752fbd85fbf74b1fb54294400ba55e06292b813eef
+  metadata.gz: 2e7d51c4bea9a4f3910e0b209c34dd6281dea5df05b1bef6bcb580c824c65c5a20776486787a91e706472833ffbdb13be83bf7f30fbabdd8f5dd8ef95e4ff15b
+  data.tar.gz: 2b800e2e958a7e30c41720576b0c96b7f1f3f654bb08e6b97198aa3f2364095d17849ec2dbe080387496b2bcf40b35795b8051d92d48315c821d3d8d93cc7119

data/Gemfile

@@ -30,3 +30,9 @@ gem "timecop"
 gem "json_schemer"
 # EventMachine nats repo doesn't support jetstream, only nats-pure
 gem "nats-pure", "~> 2.0"
+gem "opentelemetry-api", "~> 1.1"
+
+group :test do
+  gem "opentelemetry-sdk", "~> 1.2"
+end
+

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    polyn (0.2.1)
+    polyn (0.3.0)
       json_schemer (~> 0.2)
       nats-pure (~> 2.0)
+      opentelemetry-api (~> 1.1)
       yard (~> 0.9)
 
 GEM
@@ -21,6 +22,18 @@ GEM
       regexp_parser (~> 2.0)
       uri_template (~> 0.7)
     nats-pure (2.1.2)
+    opentelemetry-api (1.1.0)
+    opentelemetry-common (0.19.6)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-registry (0.2.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-sdk (1.2.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.19.3)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.8.0)
+      opentelemetry-api (~> 1.0)
     parallel (1.21.0)
     parser (3.0.2.0)
       ast (~> 2.4.1)
@@ -75,6 +88,8 @@ PLATFORMS
 DEPENDENCIES
   json_schemer
   nats-pure (~> 2.0)
+  opentelemetry-api (~> 1.1)
+  opentelemetry-sdk (~> 1.2)
   polyn!
   rake (~> 13.0)
   rspec (~> 3.0)

data/README.md

@@ -78,15 +78,6 @@ Add `:source` to make the `source` of the event more specific
 polyn.publish("user.created.v1", { name: "Mary" }, source: "new.users")
 ```
 
-Add `:triggered_by` to add a triggering event to the `polyntrace`
-
-```ruby
-polyn = Polyn.connect(nats)
-
-event = Polyn::Event.new
-polyn.publish("user.created.v1", { name: "Mary" }, triggered_by: event)
-```
-
 You can also include options of `:header` and/or `:reply_to` to passthrough to NATS
 
 ### Consuming a Stream
@@ -165,6 +156,12 @@ Despite mocking some NATS functionality you will still need a running nats-serve
 When the tests start it will load all your schemas. The tests themselves will also use the running server to verify
 stream and consumer configuration information. This hybrid mocking approach is intended to give isolation and reliability while also ensuring correct integration.
 
+## Observability
+
+### Tracing
+
+Polyn uses [OpenTelemetry](https://opentelemetry.io/) to create distributed traces that will connect sent and received events in different services. Your application will need the [`opentelemetry-sdk` gem](https://opentelemetry.io/docs/instrumentation/ruby/getting-started/) installed to collect the trace information.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run

data/lib/cloud-event-schema.json

@@ -13,7 +13,7 @@
     "source": {
       "description": "Identifies the context in which an event happened.",
       "$ref": "#/definitions/sourcedef",
-      "examples" : [
+      "examples": [
         "https://github.com/cloudevents",
         "mailto:cncf-wg-serverless@lists.cncf.io",
         "urn:uuid:6e8bc430-9c3a-11d9-9669-0800200c9a66",
@@ -32,7 +32,7 @@
     "type": {
       "description": "Describes the type of event related to the originating occurrence.",
       "$ref": "#/definitions/typedef",
-      "examples" : [
+      "examples": [
         "com.github.pull_request.opened",
         "com.example.object.deleted.v2"
       ]
@@ -89,22 +89,14 @@
           "clientversion": "0.1.0"
         }
       ]
-    },
-    "polyntrace": {
-      "$ref": "#/definitions/polyntracedef",
-      "description": "Previous events that led to this one",
-      "examples": [
-        [
-          {
-            "type": "<topic>",
-            "time": "2018-04-05T17:31:00Z",
-            "id": "<uuid>"
-          }
-        ]
-      ]
     }
   },
-  "required": ["id", "source", "specversion", "type"],
+  "required": [
+    "id",
+    "source",
+    "specversion",
+    "type"
+  ],
   "definitions": {
     "iddef": {
       "type": "string",
@@ -124,28 +116,50 @@
       "minLength": 1
     },
     "datacontenttypedef": {
-      "type": ["string", "null"],
+      "type": [
+        "string",
+        "null"
+      ],
       "minLength": 1
     },
     "dataschemadef": {
-      "type": ["string", "null"],
+      "type": [
+        "string",
+        "null"
+      ],
       "format": "uri",
       "minLength": 1
     },
     "subjectdef": {
-      "type": ["string", "null"],
+      "type": [
+        "string",
+        "null"
+      ],
       "minLength": 1
     },
     "timedef": {
-      "type": ["string", "null"],
+      "type": [
+        "string",
+        "null"
+      ],
       "format": "date-time",
       "minLength": 1
     },
     "datadef": {
-      "type": ["object", "string", "number", "array", "boolean", "null"]
+      "type": [
+        "object",
+        "string",
+        "number",
+        "array",
+        "boolean",
+        "null"
+      ]
     },
     "data_base64def": {
-      "type": ["string", "null"],
+      "type": [
+        "string",
+        "null"
+      ],
       "contentEncoding": "base64"
     },
     "polyndatadef": {
@@ -161,27 +175,11 @@
           "type": "string"
         }
       },
-      "required": ["clientlang", "clientlangversion", "clientversion"]
-    },
-    "polyntracedef": {
-      "type" : "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "type": "string"
-          },
-          "time": {
-            "type": "string",
-            "format": "date-time"
-          },
-          "id"  : {
-            "type": "string",
-            "format": "uuid"
-          }
-        },
-        "required": ["type", "time", "id"]
-      }
+      "required": [
+        "clientlang",
+        "clientlangversion",
+        "clientversion"
+      ]
     }
   }
 }

data/lib/polyn/event.rb

@@ -76,7 +76,6 @@ module Polyn
       @time            = hash.fetch(:time, Time.now.utc.iso8601)
       @data            = hash.fetch(:data)
       @datacontenttype = hash.fetch(:datacontenttype, "application/json")
-      @polyntrace      = self.class.build_polyntrace(hash[:triggered_by])
       @polyndata       = {
         clientlang:        "ruby",
         clientlangversion: RUBY_VERSION,
@@ -93,7 +92,6 @@ module Polyn
         "time"            => time,
         "data"            => Utils::Hash.deep_stringify_keys(data),
         "datacontenttype" => datacontenttype,
-        "polyntrace"      => Utils::Hash.deep_stringify_keys(polyntrace),
         "polyndata"       => Utils::Hash.deep_stringify_keys(polyndata),
       }
     end
@@ -101,15 +99,20 @@ module Polyn
     ##
     # Get the Event `source` prefixed with reverse domain name
     def self.full_source(source = nil)
-      root = Polyn.configuration.source_root
-      name = Polyn::Naming.dot_to_colon("#{domain}:#{root}")
+      root    = Polyn.configuration.source_root
+      parts   = [domain, root]
+      combine = lambda do |items|
+        items.map { |part| Polyn::Naming.dot_to_colon(part) }.join(":")
+      end
+      name    = combine.call(parts)
 
       if source
         Polyn::Naming.validate_source_name!(source)
-        "#{name}:#{Polyn::Naming.dot_to_colon(source)}"
-      else
-        name
+        source = source.gsub(/(#{name}){1}:?/, "")
+        parts << source unless source.empty?
       end
+
+      combine.call(parts)
     end
 
     ##
@@ -119,18 +122,6 @@ module Polyn
       "#{domain}.#{Polyn::Naming.trim_domain_prefix(type)}"
     end
 
-    ##
-    # Use a triggering event to build the polyntrace of a new event
-    def self.build_polyntrace(triggered_by)
-      return [] unless triggered_by
-
-      triggered_by.polyntrace.concat([{
-        id:   triggered_by.id,
-        type: triggered_by.type,
-        time: triggered_by.time,
-      }])
-    end
-
     def self.domain
       Polyn.configuration.domain
     end

data/lib/polyn/nats.rb

@@ -8,6 +8,8 @@ module Polyn
       @nats = nats
     end
 
+    attr_reader :nats
+
     def publish(type, json, reply, **opts)
       @nats.publish(type, json, reply, **opts)
     end

data/lib/polyn/pull_subscriber.rb

@@ -11,8 +11,7 @@ module Polyn
     # is more than the `source_root`
     def initialize(fields)
       @nats          = fields.fetch(:nats)
-      @type          = fields.fetch(:type)
-      @type          = Polyn::Naming.trim_domain_prefix(@type)
+      @type          = Polyn::Naming.trim_domain_prefix(fields.fetch(:type))
       @consumer_name = Polyn::Naming.consumer_name(@type, fields[:source])
       @stream        = @nats.jetstream.find_stream_name_by_subject(@type)
       self.class.validate_consumer_exists!(@nats, @stream, @consumer_name)
@@ -39,18 +38,35 @@ module Polyn
     # @option params [Float] :timeout Duration of the fetch request before it expires.
     # @return [Array<NATS::Msg>]
     def fetch(batch = 1, params = {})
-      msgs = @psub.fetch(batch, params)
-      msgs.map do |msg|
-        msg   = Polyn::Nats::Msg.new(msg)
-        event = @serializer.deserialize(msg.data)
-        if event.is_a?(Polyn::Errors::Error)
-          msg.term
-          raise event
+      Polyn::Tracing.processing_span(@type) do |process_span|
+        msgs = @psub.fetch(batch, params)
+        msgs.map do |msg|
+          Polyn::Tracing.subscribe_span(@type, msg, links: [process_span]) do |span|
+            updated_msg = process_message(msg)
+            Polyn::Tracing.span_attributes(span,
+              nats:    @nats,
+              type:    @type,
+              event:   updated_msg.data,
+              payload: msg.data)
+            updated_msg
+          end
         end
+      end
+    end
 
-        msg.data = event
-        msg
+    private
+
+    def process_message(msg)
+      msg   = msg.clone
+      msg   = Polyn::Nats::Msg.new(msg)
+      event = @serializer.deserialize(msg.data)
+      if event.is_a?(Polyn::Errors::Error)
+        msg.term
+        raise event
       end
+
+      msg.data = event
+      msg
     end
   end
 end

data/lib/polyn/tracing.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Polyn
+  ##
+  # Methods to enable distributed tracing across services
+  # Attempts to follow OpenTelemetry conventions
+  # https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/messaging/
+  class Tracing
+    ##
+    # Tracer object to use to start a trace
+    def self.tracer
+      ::OpenTelemetry.tracer_provider.tracer("polyn", Polyn::VERSION)
+    end
+
+    ##
+    # Common attributes to add to a span involving an individual message
+    # https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/messaging/#messaging-attributes
+    def self.span_attributes(span, nats:, type:, event:, payload:)
+      span.add_attributes({
+        "messaging.system"                     => "NATS",
+        "messaging.destination"                => type,
+        "messaging.protocol"                   => "Polyn",
+        "messaging.url"                        => nats.uri.to_s,
+        "messaging.message_id"                 => event.id,
+        "messaging.message_payload_size_bytes" => payload.bytesize,
+      })
+    end
+
+    ##
+    # Uses the message header to extract trace information from the
+    # published message so the subscription handler can use it as
+    # the parent span. This will allow us to create a distributed
+    # trace between publications and subscriptions. It's expecting a
+    # `traceparent` header to be set on the message
+    # https://www.w3.org/TR/trace-context/#traceparent-header
+    def self.connect_span_with_received_message(msg, &block)
+      context = OpenTelemetry.propagation.extract(msg.header)
+      ::OpenTelemetry::Context.with_current(context, &block)
+    end
+
+    ##
+    # Add a `traceparent` header to the headers for a message so that the
+    # subscribers can be connected with it
+    # https://www.w3.org/TR/trace-context/#traceparent-header
+    def self.trace_header(headers = {})
+      ::OpenTelemetry.propagation.inject(headers)
+    end
+
+    ##
+    # Start a span for publishing an event
+    def self.publish_span(type, &block)
+      tracer.in_span("#{type} send", kind: "PRODUCER", &block)
+    end
+
+    ##
+    # Start a span for handling a received event
+    def self.subscribe_span(type, msg, links: nil, &block)
+      connect_span_with_received_message(msg) do
+        tracer.in_span("#{type} receive", kind: "CONSUMER", links: convert_links(links), &block)
+      end
+    end
+
+    ##
+    # Start a span to handle processing of batch messages
+    def self.processing_span(type, &block)
+      tracer.in_span("#{type} process", kind: "CONSUMER", &block)
+    end
+
+    class << self
+      private
+
+      def convert_links(links)
+        links.map { |link| create_span_link(link) } if links
+      end
+
+      def create_span_link(span)
+        ::OpenTelemetry::Trace::Link.new(span.context)
+      end
+    end
+  end
+end

data/lib/polyn/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Polyn
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/polyn.rb

@@ -21,6 +21,7 @@
 require "json_schemer"
 require "json"
 require "nats/client"
+require "opentelemetry"
 require "securerandom"
 
 require "polyn/configuration"
@@ -34,6 +35,8 @@ require "polyn/nats/jetstream/api/consumer_config"
 require "polyn/pull_subscriber"
 require "polyn/schema_store"
 require "polyn/serializers/json"
+require "polyn/testing/mock_nats"
+require "polyn/tracing"
 require "polyn/utils/utils"
 require "polyn/version"
 
@@ -77,27 +80,29 @@ module Polyn
     # @param type [String] The type of event
     # @param data [any] The data to include in the event
     # @option options [String] :source - information to specify the source of the event
-    # @option options [String] :triggered_by - The event that triggered this one.
     # Will use information from the event to build up the `polyntrace` data
     # @option options [String] :reply_to - Reply to a specific topic
     # @option options [String] :header - Headers to include in the message
     def publish(type, data, **opts)
-      event = Event.new({
-        type:         type,
-        source:       opts[:source],
-        data:         data,
-        triggered_by: opts[:triggered_by],
-      })
+      Polyn::Tracing.publish_span(type) do |span|
+        event = Event.new({
+          type:   type,
+          source: opts[:source],
+          data:   data,
+        })
 
-      # Ensure accidental message duplication doesn't happen
-      # https://docs.nats.io/using-nats/developer/develop_jetstream/model_deep_dive#message-deduplication
-      msg_id_header = { "Nats-Msg-Id" => event.id }
-      header        = opts.fetch(:header, {})
-      header        = msg_id_header.merge(header)
+        json = @serializer.serialize!(event)
 
-      json = @serializer.serialize!(event)
+        Polyn::Tracing.span_attributes(span,
+          nats:    @nats.nats,
+          type:    type,
+          event:   event,
+          payload: json)
 
-      @nats.publish(type, json, opts[:reply_to], header: header)
+        header = add_headers(opts.fetch(:header, {}), event)
+
+        @nats.publish(type, json, opts[:reply_to], header: header)
+      end
     end
 
     ## Create subscription which is dispatched asynchronously
@@ -110,9 +115,18 @@ module Polyn
     # @option options [String] :pending_bytes_limit
     def subscribe(type, opts = {}, &callback)
       @nats.subscribe(type, opts) do |msg|
-        event    = @serializer.deserialize!(msg.data)
-        msg.data = event
-        callback.call(msg)
+        Polyn::Tracing.subscribe_span(type, msg) do |span|
+          event    = @serializer.deserialize!(msg.data)
+
+          Polyn::Tracing.span_attributes(span,
+            nats:    @nats.nats,
+            type:    type,
+            event:   event,
+            payload: msg.data)
+
+          msg.data = event
+          callback.call(msg)
+        end
       end
     end
 
@@ -148,10 +162,17 @@ module Polyn
     # NATS connection class to use based on environment
     def nats_class
       if Polyn.configuration.polyn_env == "test"
-        Polyn::MockNats
+        Polyn::Testing::MockNats
       else
         Polyn::Nats
       end
     end
+
+    def add_headers(headers, event)
+      Polyn::Tracing.trace_header(headers)
+      # Ensure accidental message duplication doesn't happen
+      # https://docs.nats.io/using-nats/developer/develop_jetstream/model_deep_dive#message-deduplication
+      { "Nats-Msg-Id" => event.id }.merge(headers)
+    end
   end
 end

data/polyn.gemspec

@@ -48,5 +48,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "json_schemer", "~> 0.2"
   spec.add_dependency "nats-pure", "~> 2.0"
+  spec.add_dependency "opentelemetry-api", "~> 1.1"
   spec.add_dependency "yard", "~> 0.9"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polyn
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Jarod
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-09-09 00:00:00.000000000 Z
+date: 2022-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json_schemer
@@ -39,6 +39,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: opentelemetry-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -99,6 +113,7 @@ files:
 - lib/polyn/testing/mock_msg.rb
 - lib/polyn/testing/mock_nats.rb
 - lib/polyn/testing/mock_pull_subscription.rb
+- lib/polyn/tracing.rb
 - lib/polyn/utils/hash.rb
 - lib/polyn/utils/string.rb
 - lib/polyn/utils/utils.rb