pug-sh 0.0.1

data/lib/pug/client.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "uri"
+require_relative "batcher"
+require_relative "connect"
+require_relative "errors"
+require_relative "event_builder"
+require_relative "log"
+require_relative "reads"
+require_relative "rpc"
+require_relative "validator"
+
+module Pug
+  class Client
+    DEFAULT_ENDPOINT = "https://api.pugs.dev"
+    DEFAULT_TIMEOUT = 10
+    RESERVED_PREFIX = "cookieless-"
+    TRACK_OPTIONS = %w[timestamp session_id location].freeze
+    IDENTIFY_OPTIONS = %w[anonymous_id device_id].freeze
+
+    class NotSerializable < StandardError; end
+    OMITTED = Object.new.freeze
+    private_constant :NotSerializable, :OMITTED
+
+    attr_reader :profiles, :activity, :insights
+
+    def initialize(api_key:, endpoint: nil, batch: nil, on_error: nil, logger: nil, timeout: nil)
+      unless api_key.is_a?(String) && !api_key.empty?
+        raise ArgumentError, "[Pug SDK] api_key is required and must be a non-empty String"
+      end
+      # A trailing newline from a secrets file would fail every request.
+      unless api_key.ascii_only? && api_key.match?(/\A[!-~]+\z/)
+        raise ArgumentError, "[Pug SDK] api_key must be printable ASCII without spaces (check for a trailing newline)"
+      end
+      unless api_key.start_with?("prv_")
+        raise ArgumentError, "[Pug SDK] api_key must be a private key (prv_…) for the server SDK"
+      end
+      raise ArgumentError, "[Pug SDK] on_error must be callable" unless on_error.nil? || on_error.respond_to?(:call)
+      unless on_error.nil? || takes_two?(on_error)
+        raise ArgumentError, "[Pug SDK] on_error must take two arguments, (error, events)"
+      end
+      raise ArgumentError, "[Pug SDK] batch must be a Hash" unless batch.nil? || batch.is_a?(Hash)
+      unless logger.nil? || %i[warn error].all? { |level| logger.respond_to?(level) }
+        raise ArgumentError, "[Pug SDK] logger must respond to warn and error, like a Logger (wrap an IO in Logger.new)"
+      end
+
+      endpoint = endpoint.to_s
+      endpoint = DEFAULT_ENDPOINT if endpoint.empty?
+      unless http_url?(endpoint)
+        raise ArgumentError, "[Pug SDK] endpoint must be an absolute http(s) URL, got #{endpoint.inspect}"
+      end
+
+      @log = Log.new(logger)
+      @connect = Connect.new(endpoint, api_key, timeout: resolve_timeout(timeout))
+      @rpc = RPC.new(@connect)
+      @events = EventBuilder.new(@log)
+      @batcher = Batcher.new(method(:send_batch), batch, on_error, @log)
+      @session_id = SecureRandom.uuid_v7
+      @closed = false
+      @profiles = Profiles.new(@rpc)
+      @activity = Activity.new(@rpc)
+      @insights = Insights.new(@rpc)
+    end
+
+    def track(distinct_id, kind, properties = OMITTED, options = nil, **keywords)
+      return @log.warn("track() called after close().") if @closed
+
+      distinct_id = id_string(distinct_id, "track", "distinct_id")
+      return if distinct_id.nil?
+      # The server rejects a whole batch over one such id, taking other users' events with it.
+      if distinct_id.start_with?(RESERVED_PREFIX)
+        return @log.error("track() distinct_id must not start with the reserved '#{RESERVED_PREFIX}' prefix.")
+      end
+      return @log.error("track() requires a non-empty event kind String.") unless kind?(kind)
+
+      properties, options = split_options(properties, options, keywords, TRACK_OPTIONS, "track")
+      return @log.error("track() properties must be a Hash.") unless properties.nil? || properties.is_a?(Hash)
+
+      session_id = options["session_id"]
+      return @log.error("track() session_id must be a String.") unless session_id.nil? || session_id.is_a?(String)
+
+      session_id = @session_id if session_id.nil? || session_id.empty?
+      event, bytesize = @events.build(kind, session_id, distinct_id, properties,
+                                      timestamp: options["timestamp"], location: options["location"])
+      @batcher.enqueue(event, bytesize) if event
+      nil
+    rescue StandardError => e
+      @log.error("Unexpected error in track(#{kind.inspect}): #{Log.describe(e)}")
+    end
+
+    def identify(external_id, traits = OMITTED, options = nil, **keywords)
+      return @log.warn("identify() called after close().") if @closed
+
+      external_id = id_string(external_id, "identify", "external_id")
+      return if external_id.nil?
+
+      traits, options = split_options(traits, options, keywords, IDENTIFY_OPTIONS, "identify")
+      return @log.error("identify() traits must be a Hash.") unless traits.nil? || traits.is_a?(Hash)
+
+      request = identify_request(external_id, traits, options)
+      return if request.nil?
+
+      violations = Validator.validate(request)
+      return @log.error("Invalid identify request: #{Validator.format(violations)}") unless violations.empty?
+
+      @rpc.unary(:identify, request)
+      nil
+    rescue NotSerializable => e
+      @log.error("identify() traits must be JSON-serializable: #{e.message}")
+    rescue StandardError => e
+      @log.error("Failed to identify #{external_id.inspect}: #{Log.describe(e)}")
+    end
+
+    def flush(timeout: nil)
+      @batcher.flush(timeout: wait_timeout(timeout, "flush"))
+    end
+
+    def close(timeout: nil)
+      @closed = true
+      @batcher.close(timeout: wait_timeout(timeout, "close"))
+    ensure
+      @connect.close
+    end
+
+    # The default would print the API key and every buffered event.
+    def inspect = "#<#{self.class.name}>"
+
+    private
+
+    def send_batch(events)
+      response = @rpc.unary(:batch_create, RPC::EventsV1::BatchCreateRequest.new(events: events))
+      # A refused event isn't an RPC error, so this is the only sign it was lost.
+      return unless response.dropped.positive?
+
+      @log.warn("Server dropped #{response.dropped} of #{events.size} event(s): #{response.dropped_by_reason.to_h}")
+    end
+
+    def identify_request(external_id, traits, options)
+      ids = IDENTIFY_OPTIONS.filter_map do |name|
+        value = options[name]
+        [name.to_sym, id_string(value, "identify", name, integers: false)] unless value.nil? || value == ""
+      end.to_h
+      return if ids.value?(nil)
+
+      request = { external_id: external_id, **ids }
+      request[:traits] = to_struct(traits) unless traits.nil?
+      RPC::SdkProfilesV1::IdentifyRequest.new(request)
+    end
+
+    def resolve_timeout(timeout)
+      return DEFAULT_TIMEOUT if timeout.nil?
+      return timeout if Connect.timeout?(timeout)
+
+      @log.warn("timeout must be a positive number of seconds; using the default #{DEFAULT_TIMEOUT}.")
+      DEFAULT_TIMEOUT
+    end
+
+    # A timeout that isn't a number waits without one, as close must still drain; one already
+    # past doesn't wait at all.
+    def wait_timeout(timeout, method)
+      return if timeout.nil? || timeout == Float::INFINITY
+      return [timeout.to_f, 0].max if timeout.is_a?(Numeric) && timeout.real? && timeout.finite?
+
+      @log.warn("#{method}() timeout must be a number of seconds; waiting without one.")
+      nil
+    end
+
+    # Lambdas and methods check their arity, so one taking fewer arguments would fail every call.
+    def takes_two?(callable)
+      callable = callable.method(:call) unless callable.is_a?(Proc) || callable.is_a?(Method)
+      return true if callable.is_a?(Proc) && !callable.lambda?
+
+      arity = callable.arity
+      arity.negative? ? -arity - 1 <= 2 : arity == 2
+    end
+
+    def http_url?(endpoint)
+      uri = URI.parse(endpoint)
+      uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
+    rescue URI::InvalidURIError
+      false
+    end
+
+    # An Integer passes as an id too, since a Rails record's id is one. Unlike property values,
+    # ids aren't repaired: a mangled id would be a different person.
+    def id_string(value, method, name, integers: true)
+      value = value.to_s if integers && value.is_a?(Integer)
+      unless value.is_a?(String) && !value.empty?
+        return @log.error("#{method}() requires a non-empty #{name} String#{" or Integer" if integers}.")
+      end
+
+      utf8 = if [Encoding::UTF_8, Encoding::BINARY, Encoding::US_ASCII].include?(value.encoding)
+               value.dup.force_encoding(Encoding::UTF_8)
+             else
+               value.encode(Encoding::UTF_8)
+             end
+      return utf8 if utf8.valid_encoding?
+
+      @log.error("#{method}() #{name} must be valid UTF-8.")
+    rescue EncodingError
+      @log.error("#{method}() #{name} must be valid UTF-8.")
+    end
+
+    def kind?(kind)
+      (kind.is_a?(String) || kind.is_a?(Symbol)) && !kind.empty?
+    end
+
+    # Braceless keywords arrive as one Hash, so track(id, kind, plan: "pro") and
+    # track(id, kind, timestamp: t) look alike. A keyword named after an option sets that option;
+    # any other is data, properties or traits, unless the data came first in an argument of its own.
+    def split_options(data, options, keywords, known, method)
+      named = keywords
+      if data.equal?(OMITTED)
+        named, data = keywords.partition { |key, _| known.include?(key.to_s) }.map(&:to_h)
+        data = nil if data.empty?
+      end
+      [data, options_hash(options, known, method).merge(options_hash(named, known, method))]
+    end
+
+    def options_hash(options, known, method)
+      return {} if options.nil?
+
+      unless options.is_a?(Hash)
+        @log.warn("Ignoring #{method}() options: expected a Hash, got #{options.class}")
+        return {}
+      end
+      options.each_with_object({}) do |(key, value), out|
+        if known.include?(key.to_s)
+          out[key.to_s] = value
+        else
+          @log.warn("Ignoring unknown #{method}() option #{key.inspect}")
+        end
+      end
+    end
+
+    # A JSON round trip accepts what JSON does (symbol keys, nested arrays) and refuses what a
+    # Struct can't carry, such as NaN. A Struct's numbers are doubles, so an integer past 2^53
+    # would arrive changed.
+    def to_struct(traits)
+      ready = EventBuilder.json_ready(traits, max_integer: EventBuilder::MAX_SAFE_FLOAT)
+      ::Google::Protobuf::Struct.from_hash(JSON.parse(JSON.generate(ready)))
+    rescue StandardError => e
+      raise NotSerializable, e.message
+    end
+  end
+end

data/lib/pug/connect.rb

@@ -0,0 +1,291 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "openssl"
+require "uri"
+require "zlib"
+require_relative "errors"
+require_relative "version"
+
+module Pug
+  # A Connect protocol client (https://connectrpc.com/docs/protocol) for the two call shapes the
+  # SDK uses: unary, and server streaming for profiles.list. Binary protobuf over HTTP/1.1, like
+  # sdk-node. Every failure surfaces as a Pug::Error carrying a Connect code.
+  class Connect
+    USER_AGENT = "pug-ruby/#{VERSION}".freeze
+    UNARY_CONTENT_TYPE = "application/proto"
+    STREAM_CONTENT_TYPE = "application/connect+proto"
+    CONTENT_TYPE = %r{\Aapplication/(?<stream>connect\+)?(?:(?<json>json)(?:; ?charset=utf-?8)?|(?<proto>proto))\z}i
+    END_STREAM = 0b10
+    COMPRESSED = 0b01
+
+    # Carries an exception raised by the caller's block past the transport's rescue clauses, which
+    # would otherwise mistake, say, the caller's IOError for a network failure. Whoever passed
+    # the block re-raises the original.
+    class Passthrough < Exception # rubocop:disable Lint/InheritException
+      attr_reader :error
+
+      def initialize(error)
+        @error = error
+        super(error.message)
+      end
+    end
+
+    CODES = %i[canceled unknown invalid_argument deadline_exceeded not_found already_exists permission_denied
+               resource_exhausted failed_precondition aborted out_of_range unimplemented internal unavailable
+               data_loss unauthenticated].to_h { |code| [code.to_s, code] }.freeze
+
+    # https://connectrpc.com/docs/protocol/#http-to-error-code: 400 is internal and 404
+    # unimplemented on purpose.
+    HTTP_CODES = { 400 => :internal, 401 => :unauthenticated, 403 => :permission_denied, 404 => :unimplemented,
+                   429 => :unavailable, 502 => :unavailable, 503 => :unavailable, 504 => :unavailable }.freeze
+
+    def self.timeout?(value)
+      value.is_a?(Numeric) && value.real? && value.positive? && value.finite?
+    end
+
+    def initialize(endpoint, api_key, timeout:)
+      @uri = URI(endpoint)
+      @base_path = @uri.path.to_s.chomp("/")
+      @api_key = api_key
+      @timeout = timeout
+      @pool = Pool.new(@uri)
+    end
+
+    def unary(procedure, request, response_class)
+      response = call(procedure, request.class.encode(request), UNARY_CONTENT_TYPE, @timeout) do |http, post|
+        http.request(post)
+      end
+      raise unary_error(response) unless response.code == "200"
+
+      check_content_type(response, stream: false)
+      decode(response_class, response.body.to_s)
+    end
+
+    # The deadline covers the whole stream, including the time the caller's block takes.
+    def server_stream(procedure, request, response_class, timeout: nil)
+      timeout ||= @timeout
+      body = envelope(0, request.class.encode(request))
+      deadline = monotonic + timeout
+      call(procedure, body, STREAM_CONTENT_TYPE, timeout) do |http, post|
+        http.request(post) do |response|
+          raise http_error(response) unless response.code == "200"
+
+          check_content_type(response, stream: true)
+          check_stream_encoding(response)
+          read_stream(http, response, response_class, deadline) do |message|
+            yield message
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            raise Passthrough, e
+          end
+        end
+      end
+      nil
+    end
+
+    def close
+      @pool.close
+    end
+
+    # The default would print the API key.
+    def inspect = "#<#{self.class.name}>"
+
+    private
+
+    def call(procedure, body, content_type, timeout)
+      post = Net::HTTP::Post.new("#{@base_path}/#{procedure}")
+      post["Content-Type"] = content_type
+      post["Connect-Protocol-Version"] = "1"
+      post["Connect-Timeout-Ms"] = (timeout * 1000).ceil.to_s
+      post["User-Agent"] = USER_AGENT
+      post["X-Api-Key"] = @api_key
+      post.body = body
+      @pool.with(timeout) { |http| yield http, post }
+    rescue Timeout::Error => e
+      raise Error.new("request timed out: #{e.message}", code: :deadline_exceeded)
+    rescue SocketError, SystemCallError, IOError, OpenSSL::SSL::SSLError => e
+      raise Error.new("#{e.class}: #{e.message}", code: :unavailable)
+    rescue Net::HTTPBadResponse, Net::ProtocolError, Zlib::Error => e
+      raise Error.new("#{e.class}: #{e.message}", code: :internal)
+    end
+
+    def read_stream(http, response, response_class, deadline)
+      buffer = "".b
+      ended = false
+      response.read_body do |chunk|
+        buffer << chunk
+        while buffer.bytesize >= 5
+          flags, length = buffer.unpack("CN")
+          break if buffer.bytesize < 5 + length
+
+          data = buffer.byteslice(5, length)
+          buffer = buffer.byteslice((5 + length)..)
+          if ended
+            raise Error.new("protocol error: received extra message after EndStreamResponse", code: :invalid_argument)
+          end
+          if flags.anybits?(COMPRESSED)
+            raise Error.new("protocol error: received a compressed message it did not ask for", code: :internal)
+          end
+
+          if flags.anybits?(END_STREAM)
+            end_stream(data)
+            ended = true
+          else
+            yield decode(response_class, data)
+          end
+        end
+        next if ended
+
+        remaining = deadline - monotonic
+        raise Error.new("request timed out", code: :deadline_exceeded) unless remaining.positive?
+
+        http.read_timeout = remaining
+      end
+      raise Error.new("protocol error: premature end of stream", code: :data_loss) unless buffer.empty?
+      raise Error.new("protocol error: missing EndStreamResponse", code: :invalid_argument) unless ended
+    end
+
+    def end_stream(data)
+      invalid = Error.new("invalid end stream", code: :unknown)
+      parsed = parse_json(data)
+      raise invalid unless parsed.is_a?(Hash)
+      raise wire_error(parsed["error"], invalid) unless parsed["error"].nil?
+    end
+
+    def unary_error(response)
+      fallback = http_error(response)
+      match = CONTENT_TYPE.match(response["Content-Type"].to_s)
+      return fallback unless match && match[:json]
+
+      wire_error(parse_json(response.body.to_s), fallback)
+    end
+
+    def wire_error(json, fallback)
+      return fallback unless json.is_a?(Hash)
+
+      message = json["message"]
+      return fallback unless message.nil? || message.is_a?(String)
+
+      Error.new(message.to_s, code: CODES.fetch(json["code"].to_s, fallback.code))
+    end
+
+    def parse_json(data)
+      JSON.parse(data.dup.force_encoding(Encoding::UTF_8))
+    rescue JSON::ParserError, EncodingError
+      nil
+    end
+
+    def check_content_type(response, stream:)
+      type = response["Content-Type"].to_s
+      match = CONTENT_TYPE.match(type)
+      return if match && match[:proto] && !match[:stream] == !stream
+
+      raise Error.new("unsupported content type #{type}", code: match ? :internal : :unknown)
+    end
+
+    def check_stream_encoding(response)
+      encoding = response["Connect-Content-Encoding"]
+      return if encoding.nil? || encoding.casecmp?("identity")
+
+      raise Error.new("unsupported response encoding \"#{encoding}\"", code: :internal)
+    end
+
+    def decode(response_class, data)
+      response_class.decode(data)
+    rescue ::Google::Protobuf::ParseError => e
+      raise Error.new("failed to decode #{response_class.descriptor.name}: #{e.message}", code: :internal)
+    end
+
+    def envelope(flags, data)
+      [flags, data.bytesize].pack("CN") + data
+    end
+
+    # Net::HTTP doesn't follow redirects, so one means the endpoint is misconfigured, as a 404
+    # does: unimplemented, which nothing retries. A proxy may have answered instead of a Connect
+    # server, so the status gets the redirect target or the start of the body.
+    def http_error(response)
+      status = response.code.to_i
+      redirect = (300..399).cover?(status)
+      detail = redirect ? response["Location"]&.then { |url| "redirected to #{url}" } : snippet(response.body)
+      code = redirect ? :unimplemented : HTTP_CODES.fetch(status, :unknown)
+      Error.new(["HTTP #{status}", detail].compact.join(": "), code: code)
+    end
+
+    def snippet(body)
+      text = body.to_s.byteslice(0, 512).force_encoding(Encoding::UTF_8).scrub("").gsub(/\s+/, " ").strip
+      text[0, 120] unless text.empty?
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    # Keep-alive connections, one per concurrent caller. Net::HTTP sessions aren't thread-safe,
+    # so each is checked out by one thread at a time.
+    class Pool
+      MAX_IDLE = 4
+
+      def initialize(uri)
+        @uri = uri
+        @idle = []
+        @lock = Mutex.new
+        @pid = Process.pid
+      end
+
+      def with(timeout)
+        http = checkout
+        http.open_timeout = timeout
+        http.read_timeout = timeout
+        http.write_timeout = timeout
+        http.start unless http.started?
+        result = yield http
+        checkin(http)
+        http = nil
+        result
+      ensure
+        discard(http) if http
+      end
+
+      def close
+        connections = @lock.synchronize do
+          forked
+          @idle.slice!(0..)
+        end
+        connections.each { |http| discard(http) }
+      end
+
+      private
+
+      def checkout
+        @lock.synchronize do
+          forked
+          @idle.pop
+        end || Net::HTTP.new(@uri.hostname, @uri.port).tap { |http| http.use_ssl = @uri.scheme == "https" }
+      end
+
+      def checkin(http)
+        keep = @lock.synchronize do
+          forked
+          @idle.size < MAX_IDLE && @idle.push(http)
+        end
+        discard(http) unless keep
+      end
+
+      # A forked child must not reuse, or even close, its parent's sockets: a TLS close_notify
+      # would be written into the parent's live connection. They are dropped instead.
+      def forked
+        return if @pid == Process.pid
+
+        @pid = Process.pid
+        @idle = []
+      end
+
+      def discard(http)
+        http.finish if http.started?
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/pug/countries.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Pug
+  # Mirrors internal/geo/countries.go in the pug server. The server stores $country unvalidated
+  # and only the choropleth filters it, so a bad code becomes a permanent rollup dimension that
+  # is simply missing from the map. "XX", Cloudflare's unknown, passes a two-letter check.
+  COUNTRY_CODES = %w[
+    AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ BA BB BD BE BF BG BH BI BJ
+    BL BM BN BO BQ BR BS BT BV BW BY BZ CA CC CD CF CG CH CI CK CL CM CN CO CR
+    CU CV CW CX CY CZ DE DJ DK DM DO DZ EC EE EG EH ER ES ET FI FJ FK FM FO FR
+    GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY HK HM HN HR HT HU
+    ID IE IL IM IN IO IQ IR IS IT JE JM JO JP KE KG KH KI KM KN KP KR KW KY KZ
+    LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF MG MH MK ML MM MN MO MP MQ
+    MR MS MT MU MV MW MX MY MZ NA NC NE NF NG NI NL NO NP NR NU NZ OM PA PE PF
+    PG PH PK PL PM PN PR PS PT PW PY QA RE RO RS RU RW SA SB SC SD SE SG SH SI
+    SJ SK SL SM SN SO SR SS ST SV SX SY SZ TC TD TF TG TH TJ TK TL TM TN TO TR
+    TT TV TW TZ UA UG UM US UY UZ VA VC VE VG VI VN VU WF WS XK YE YT ZA ZM ZW
+  ].to_set.freeze
+end

data/lib/pug/errors.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Pug
+  class Error < StandardError
+    attr_reader :code
+
+    def initialize(message = nil, code: nil)
+      @code = code
+      super(code ? [code, message].reject { |part| part.to_s.empty? }.join(": ") : message)
+    end
+  end
+
+  class ClosedError < Error; end
+
+  class QueueOverflowError < Error; end
+end