streamforce 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c34310d1629ebc82065f442b4e4ef30eb91fef7f28aa42b01afbc062b6fd3a1e
-  data.tar.gz: 53ae2b66be382330501458097c322c88a70826400af18808ae5f1b062966b6dd
+  metadata.gz: 5d03ff265d884babb70b8db70637c9814a5819834ed83109b2b26537cb5c992b
+  data.tar.gz: b7b87b62347f92e9a27139536655ee95339e8a3b22bf2c1980db1eca00fa66e9
 SHA512:
-  metadata.gz: 4fb0fd547750fb0c26cc027407227bec6091e5a640431bbd26b31eac34866914aaa8fe9b2f0a99551b598aea6ae0f11de0f2ff54e85986fc2cd96e20c3e7a1af
-  data.tar.gz: 55b68bf69239eba0af1ceabfd225184c2ef8e838154273086614acbad4c685645f37bbb8ed6a60da04033f8cc965191a053a719a864815746ad06bfb868b30a3
+  metadata.gz: c4dc737d70d28c061ee3c89379b97274983c14008184acbe7c99139b59c3ed79b3ef79893ba3d486d3f142172d30461260237d171bdb7a230c6f92471769a605
+  data.tar.gz: 6a3bbd281947060861c34e5cceab6814cfad52a80e964c80ef4da4956a3c40287e1315ef05419c69b31e60ac2bd20b394336e85a087c8fd41e22691543f6f7d7

data/.rubocop.yml

@@ -1,8 +1,3 @@
-AllCops:
-  TargetRubyVersion: 3.0
-
-Style/StringLiterals:
-  EnforcedStyle: double_quotes
-
-Style/StringLiteralsInInterpolation:
-  EnforcedStyle: double_quotes
+# Omakase Ruby styling for Rails
+inherit_gem:
+  rubocop-rails-omakase: rubocop.yml

data/README.md

@@ -1,8 +1,76 @@
 # Streamforce
 
+In most cases, processing events received from the Salesforce Streaming API can be
+broken down into three very specific steps:
+
+1. Connecting to the Salesforce API and listening for messaging
+2. Ingesting received messages into some sort of internal event bus (e.g. RabbitMQ,
+   Kafka, Redis, etc)
+3. Processing the stored messages
+
+Streamforce aims to handle #1 and simplify the work done for #2.
+
+[Restforce](https://github.com/restforce/restforce) provides a simple API to connect
+to the Streaming API and consume messages:
+
+```ruby
+# Restforce uses faye as the underlying implementation for CometD.
+require 'faye'
+
+# Initialize a client with your username/password/oauth token/etc.
+client = Restforce.new(username: 'foo',
+                       password: 'bar',
+                       security_token: 'security token',
+                       client_id: 'client_id',
+                       client_secret: 'client_secret')
+
+EM.run do
+  # Subscribe to the PushTopic.
+  client.subscription '/topic/AllAccounts' do |message|
+    puts message.inspect
+  end
+end
+```
+
+However, the above code is usable in a production environment because:
+
+* The interactions with the Streaming API need to be logged using the correct severity
+  (e.g. handshakes should use `Logger::DEBUG` while subscription errors should use
+  `Logger::ERROR` for better visibility)
+* Replay IDs need to be stored using a persistent storage like Redis and not in-memory
+
+Streamforce comes with all the batteries included.
+
+## Usage
+
+A very simple client, which automatically connects based on the following environment
+variables:
+
+* `SALESFORCE_USERNAME`
+* `SALESFORCE_PASSWORD`
+* `SALESFORCE_SECURITY_TOKEN`
+* `SALESFORCE_CLIENT_ID`
+* `SALESFORCE_CLIENT_SECRET`
+* `REDIS_URL`
+
+```ruby
+require "streamforce"
+
+client = Streamforce::Client.new
+
+subscriptions = %w[
+  /topic/account-monitor
+  /event/AccountUpdated__e
+]
+
+client.subscribe(subscriptions) do |subscription, message|
+  # Your code
+end
+```
+
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/andreimaxim/streamforce.
+Bug reports and pull requests are welcome on GitHub at <https://github.com/andreimaxim/streamforce>.
 
 ## License
 

data/lib/streamforce/client.rb

@@ -1,79 +1,79 @@
 class Streamforce::Client
-  class << self
-    def host
-      ENV["SALESFORCE_HOST"]
-    end
-
-    def client_id
-      ENV["SALESFORCE_CLIENT_ID"]
-    end
-
-    def client_secret
-      ENV["SALESFORCE_CLIENT_SECRET"]
-    end
-
-    def username
-      ENV["SALESFORCE_USERNAME"]
-    end
-
-    def password
-      ENV["SALESFORCE_PASSWORD"]
-    end
-
-    def security_token
-      ENV["SALESFORCE_SECURITY_TOKEN"]
-    end
+  attr_reader :host, :username, :password, :client_id, :client_secret, :security_token,
+    :api_version
+
+  def initialize(opts = {})
+    @host           = opts.fetch(:host, ENV["SALESFORCE_HOST"])
+    @username       = opts.fetch(:username, ENV["SALESFORCE_USERNAME"])
+    @password       = opts.fetch(:password, ENV["SALESFORCE_PASSWORD"])
+    @client_id      = opts.fetch(:client_id, ENV["SALESFORCE_CLIENT_ID"])
+    @client_secret  = opts.fetch(:client_secret, ENV["SALESFORCE_CLIENT_SECRET"])
+    @security_token = opts.fetch(:security_token, ENV["SALESFORCE_SECURITY_TOKEN"])
+    @api_version    = opts.fetch(:api_version, ENV["SALESFORCE_API_VERSION"])
+
+    @logger = opts.fetch(:logger, Logger.new($stdout))
+    @logger.level = ENV.fetch("STREAMFORCE_LOG_LEVEL", Logger::INFO)
+  end
 
-    def version
-      ENV.fetch("SALESFORCE_API_VERSION", "61.0")
-    end
+  def subscribe(channels = [], &blk)
+    EM.run { subscribe_to_channels(faye, Array(channels), &blk) }
+  end
 
-    def authentication_url
-      URI.parse("https://#{host}/services/oauth2/token")
-    end
+  private
 
-    def authentication_params
-      {
-        grant_type: "password",
-        client_id: client_id,
-        client_secret: client_secret,
-        username: username,
-        password: "#{password}#{security_token}"
-      }
-    end
+  def instance_url
+    authentication["instance_url"]
   end
 
-  def set_authentication_credentials!
-    response = Net::HTTP.post_form(Streamforce::Client.authentication_url, Streamforce::Client.authentication_params)
-    credentials = JSON.parse(response.body)
+  def access_token
+    authentication["access_token"]
+  end
 
-    @access_token = credentials["access_token"]
-    @instance_url = "#{credentials["instance_url"]}/cometd/#{Streamforce::Client.version}"
+  def authentication_url
+    URI.parse("https://#{host}/services/oauth2/token")
   end
 
-  def subscribe(channels = [], &blk)
-    channels = Array(channels)
+  def authentication_params
+    {
+      grant_type: "password",
+      username: username,
+      password: "#{password}#{security_token}",
+      client_id: client_id,
+      client_secret: client_secret
+    }
+  end
 
-    set_authentication_credentials!
+  def authentication
+    @authentication ||= fetch_authentication_credentials
+  end
 
-    EM.run { subscribe_to_channels(faye, channels, &blk) }
+  def fetch_authentication_credentials
+    response = Net::HTTP.post_form(authentication_url, authentication_params)
+    JSON.parse(response.body)
   end
 
   def faye
-    @faye ||= Faye::Client.new(@instance_url).tap do |client|
-      client.set_header "Authorization", "OAuth #{@access_token}"
+    @faye ||= Faye::Client.new("#{instance_url}/cometd/#{api_version}").tap do |client|
+      client.set_header "Authorization", "OAuth #{access_token}"
 
       client.add_extension Streamforce::Extension::Replay.new
-      client.add_extension Streamforce::Extension::SubscriptionTracking.new
-      client.add_extension Streamforce::Extension::Logging.new
+      client.add_extension Streamforce::Extension::Logging.new(@logger)
     end
   end
 
   def subscribe_to_channels(client, channels, &blk)
     return if channels.empty?
 
-    client
-      .subscribe(channels.shift, &blk)
-      .callback { subscribe_to_channels(client, channels, &blk)}
+    # Subscribe to a single channel, otherwise Salesforce will return a 403 Unknown Client error
+    subscription = client.subscribe(channels.shift)
+
+    # Allow clients to receive [ channel, message ] block params
+    subscription.with_channel(&blk)
+
+    # Continue subscribing to the rest of the channels, regadless of the current subscription
+    # status
+    subscription
+      .callback { subscribe_to_channels(client, channels, &blk) }
+      .errback { subscribe_to_channels(client, channels, &blk) }
   end
 end

data/lib/streamforce/extension/logging.rb

@@ -1,18 +1,97 @@
 class Streamforce::Extension::Logging
-  def initialize(log_level = Logger::DEBUG)
-    @logger = Logger.new($stdout)
-    @logger.level = log_level
+  def initialize(logger)
+    @logger = logger
   end
 
-  def incoming(message, callback)
-    @logger.debug "Receving message: #{message.inspect}"
+  def incoming(payload, callback)
+    message = Streamforce::Message.new(payload)
+    log_incoming_message(message)
 
-    callback.call(message)
+    callback.call(payload)
   end
 
-  def outgoing(message, callback)
-    @logger.debug "Sending message: #{message.inspect}"
+  def outgoing(payload, callback)
+    message = Streamforce::Message.new(payload)
+    log_outgoing_message(message)
 
-    callback.call(message)
+    callback.call(payload)
+  end
+
+  def log_incoming_message(message)
+    if message.channel_type == "meta"
+      public_send("log_incoming_#{message.channel_name}", message)
+    else
+      @logger.debug "[#{message.channel_name}] #{message.data}"
+    end
+  end
+
+  def log_outgoing_message(message)
+    if message.channel_type == "meta"
+      public_send("log_outgoing_#{message.channel_name}", message)
+    else
+      @logger.debug "[#{message.channel_name}] #{message.data}"
+    end
+  end
+
+  def log_outgoing_handshake(message)
+    @logger.debug "[Client] Handshake requested..."
+  end
+
+  def log_incoming_handshake(message)
+    if message.success?
+      @logger.debug "[Server] Handshake accepted, assigning client_id=#{message.client_id}"
+    else
+      @logger.error "[Server] Connection was refused: #{message.error_message}"
+    end
+  end
+
+  def log_outgoing_connect(message)
+    debug message, "Sending connection request"
+  end
+
+  def log_incoming_connect(message)
+    if message.success?
+      debug message, "Connection successful!"
+    else
+      error message, "Connection failed: #{message.error_message}"
+    end
+  end
+
+  def log_outgoing_subscribe(message)
+    replay_id = message.replay_id
+
+    replay_info = if replay_id == -1
+      "for all new messages"
+    elsif replay_id == -2
+      "and requesting all stored messages"
+    else
+      "and requesting all messages newer than ##{replay_id}"
+    end
+
+    info message, "Subscribing to #{message.subscription} #{replay_info}"
+  end
+
+  def log_incoming_subscribe(message)
+    if message.success?
+      info message, "Successfully subscribed to #{message.subscription}"
+    else
+      error message, "Subscription for #{message.subscription} failed: #{message.error_message}"
+    end
+  end
+
+  def debug(message, text)
+    @logger.debug "[#{message.client_id}##{message.id}] #{text}"
+  end
+
+  def info(message, text)
+    @logger.info "[#{message.client_id}##{message.id}] #{text}"
+  end
+
+  def warn(message, text)
+    @logger.warn "[#{message.client_id}##{message.id}] #{text}"
+  end
+
+  def error(message, text)
+    @logger.error "[#{message.client_id}##{message.id}] #{text}"
   end
 end

data/lib/streamforce/extension/replay.rb

@@ -1,14 +1,36 @@
+require "redis"
+
 class Streamforce::Extension::Replay
   def initialize(log_level = Logger::INFO)
     @logger = Logger.new($stdout)
     @logger.level = log_level
+    @redis = Redis.new
   end
 
   def incoming(message, callback)
+    replay_id = message.dig "data", "event", "replayId"
+    channel   = message["channel"]
+
+    store(channel, replay_id)
     callback.call(message)
   end
 
   def outgoing(message, callback)
+    return callback.call(message) unless message["channel"] == "/meta/subscribe"
+
+    channel = message["subscription"]
+    message["ext"] = { "replay" => { channel => replay_id(channel).to_i } }
+
     callback.call(message)
   end
+
+  def store(channel, replay_id)
+    return if channel.nil? || replay_id.nil?
+
+    @redis.set channel, replay_id, ex: 86400
+  end
+
+  def replay_id(channel)
+    @redis.get(channel) || -1
+  end
 end

data/lib/streamforce/message.rb

@@ -0,0 +1,49 @@
+class Streamforce::Message
+  def initialize(payload)
+    @payload = payload
+  end
+
+  def success?
+    @payload["successful"]
+  end
+
+  def id
+    @payload["id"]
+  end
+
+  def replay_id
+    @payload.dig "ext", "replay", subscription
+  end
+
+  def channel
+    @payload["channel"]
+  end
+
+  def client_id
+    @payload["clientId"]
+  end
+
+  def channel_type
+    channel.split("/")[1]
+  end
+
+  def channel_name
+    channel.split("/")[2]
+  end
+
+  def data
+    @payload["data"]
+  end
+
+  def subscription
+    @payload["subscription"]
+  end
+
+  def subscription?
+    channel == "/meta/subscribe"
+  end
+
+  def error_message
+    @payload["error"]
+  end
+end

data/lib/streamforce/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Streamforce
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/lib/streamforce.rb

@@ -6,7 +6,6 @@ require "uri"
 require "net/http"
 require "logger"
 require "json"
-require "base64"
 
 require "faye"
 require "relaxed_cookiejar"

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: streamforce
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Andrei Maxim
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-12 00:00:00.000000000 Z
+date: 2024-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: base64
+  name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: zeitwerk
+  name: faye
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: faye
+  name: relaxed_cookiejar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: relaxed_cookiejar
+  name: redis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -66,7 +66,49 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rails-omakase
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
 email:
 - andrei@andreimaxim.ro
 executables: []
@@ -82,7 +124,7 @@ files:
 - lib/streamforce/client.rb
 - lib/streamforce/extension/logging.rb
 - lib/streamforce/extension/replay.rb
-- lib/streamforce/extension/subscription_tracking.rb
+- lib/streamforce/message.rb
 - lib/streamforce/version.rb
 homepage: https://github.com/andreimaxim/streamforce
 licenses:
@@ -91,7 +133,7 @@ metadata:
   homepage_uri: https://github.com/andreimaxim/streamforce
   source_code_uri: https://github.com/andreimaxim/streamforce
   changelog_uri: https://github.com/andreimaxim/streamforce/blob/main/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -99,7 +141,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.3.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -107,7 +149,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.5.9
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Small wrapper over the Salesforce Streaming API
 test_files: []

data/lib/streamforce/extension/subscription_tracking.rb

@@ -1,58 +0,0 @@
-class Streamforce::Extension::SubscriptionTracking
-  def initialize(log_level = Logger::DEBUG)
-    @logger = Logger.new($stdout)
-    @logger.level = log_level
-
-    @subscriptions = []
-  end
-
-  def incoming(message, callback)
-    if subscription?(message)
-      log_subscription_status(message)
-    else
-      log_subscription_payload(message)
-    end
-
-    callback.call(message)
-  end
-
-  def outgoing(message, callback)
-    @logger.debug "Requested subscription for #{message["subscription"]}" if subscription?(message)
-
-    callback.call(message)
-  end
-
-  def log_subscription_status(message)
-    subscription = message["subscription"]
-
-    if message["successful"]
-      @subscriptions << subscription
-
-      @logger.info "Subscription for #{subscription} was successful"
-    else
-      @logger.warn "Subscription for #{subscription} failed: #{message["error"]}"
-    end
-  end
-
-  def log_subscription_payload(message)
-    channel = message["channel"]
-    payload = message["data"].to_json
-
-    type = case channel.split("/")[1]
-           when "topic"
-             "PushTopic"
-           when "event"
-             "PlatformEvent"
-           else
-             "Unknown"
-           end
-
-    name = channel.split("/")[2]
-
-    @logger.info "[#{type}][#{name}]: #{payload}" if @subscriptions.include?(channel)
-  end
-
-  def subscription?(message)
-    message["channel"] == "/meta/subscribe"
-  end
-end