x_api_scraper 0.1.0

data/lib/xapi_scraper/utils/forms.rb

@@ -0,0 +1,232 @@
+# Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+
+# typed: true
+# frozen_string_literal: true
+
+require "cgi"
+require "date"
+require "base64"
+require "sorbet-runtime"
+
+module XapiScraper
+  module Utils
+    extend T::Sig
+
+    sig do
+      params(
+        field_name: String,
+        explode: T::Boolean,
+        obj: Object,
+        delimiter: String,
+        get_field_name_lambda: T.proc.params(obj_field: ::Crystalline::MetadataFields::Field).returns(String)
+      )
+        .returns(T::Hash[String, T::Array[String]])
+    end
+    def self._populate_form(field_name, explode, obj, delimiter, &get_field_name_lambda)
+      params = {}
+
+      return params if obj.nil?
+
+      if obj.respond_to?(:fields)
+        items = []
+        T.unsafe(obj).fields.each do |obj_field|
+          obj_field_name = get_field_name_lambda.call(obj_field)
+          next if obj_field_name == ""
+
+          val = obj.send(obj_field.name.to_sym)
+          next if val.nil?
+
+          if explode
+            params[obj_field_name] = [val_to_string(val)]
+          else
+            items.append("#{obj_field_name}#{delimiter}#{val_to_string(val)}")
+          end
+        end
+
+        params[field_name] = [items.join(delimiter)] if !items.empty?
+      elsif obj.is_a?(Hash)
+        items = []
+        obj.each do |key, value|
+          next if value.nil?
+
+          if explode
+            params[key] = val_to_string(value)
+          else
+            items.append("#{key}#{delimiter}#{val_to_string(value)}")
+          end
+        end
+
+        params[field_name] = [items.join(delimiter)] if !items.empty?
+
+      elsif obj.is_a?(Array)
+        items = []
+        obj.each do |value|
+          next if value.nil?
+
+          if explode
+            params[field_name] = [] if !params.key?(field_name)
+
+            params[field_name].append(val_to_string(value))
+          else
+            items.append(val_to_string(value))
+          end
+        end
+
+        params[field_name] = items.map(&:to_s).join(delimiter) if !items.empty?
+
+      else
+        params[field_name] = val_to_string(obj)
+      end
+
+      params
+    end
+
+    sig {
+      params(media_type: String, request: Object).returns(
+        [String, Object, T::Array[T::Array[T.any(T::Array[T.nilable(String)], String)]]]
+      )
+    }
+    def self.serialize_multipart_form(media_type, request)
+      form = []
+      T.unsafe(request).fields.each do |field|
+        val = request.send(field.name)
+        next if val.nil?
+
+        field_metadata = field.metadata[:multipart_form]
+        next if field_metadata.nil?
+
+        if field_metadata[:file] == true
+          field_name = field_metadata[:field_name]
+
+          # Handle arrays of files
+          if val.is_a?(Array)
+            val.each do |file_obj|
+              file_name = T.let("", String)
+              content = nil
+
+              T.must(file_obj).fields.each do |file_field|
+                file_metadata = file_field.metadata[:multipart_form]
+                next if file_metadata.nil?
+
+                if file_metadata[:content] == true
+                  content = T.let(file_obj.send(file_field.name), String)
+                else
+                  file_name = T.let(file_obj.send(file_field.name), String)
+                end
+              end
+
+              if T.unsafe(file_name) == "" || T.unsafe(content).nil?
+                raise StandardError, "invalid multipart/form-data file"
+              end
+
+              form.append([field_name, [file_name, content]])
+            end
+          else
+            # Handle single file
+            file_name = T.let("", String)
+            content = nil
+
+            T.must(val).fields.each do |file_field|
+              file_metadata = file_field.metadata[:multipart_form]
+              next if file_metadata.nil?
+
+              if file_metadata[:content] == true
+                content = T.let(val.send(file_field.name), String)
+              else
+                file_name = T.let(val.send(file_field.name), String)
+              end
+            end
+
+            if T.unsafe(file_name) == "" || T.unsafe(content).nil?
+              raise StandardError, "invalid multipart/form-data file"
+            end
+
+            form.append([field_name, [file_name, content]])
+          end
+        elsif field_metadata[:json] == true
+          to_append = [
+            field_metadata.fetch(:field_name, field.name),
+            [
+              nil,
+              ::Crystalline.to_json(val),
+              "application/json"
+            ]
+          ]
+          form.append(to_append)
+        else
+          field_name = field_metadata.fetch(
+            :field_name,
+            field.name
+          )
+          if val.is_a?(Array)
+            val.each do |value|
+              next if value.nil?
+
+              form.append(
+                [field_name, [nil, val_to_string(value)]]
+              )
+            end
+          else
+            form.append([field_name, [nil, val_to_string(val)]])
+          end
+        end
+      end
+
+      [media_type, nil, form]
+    end
+
+    sig do
+      params(data: T.any(Object, T::Hash[String, String]))
+        .returns(T::Hash[T.any(String, Symbol), Object])
+    end
+    def self.serialize_form_data(data)
+      get_form_field_name = lambda do |obj_field|
+        obj_param_metadata = obj_field.metadata[:form]
+
+        return T.let({}, T::Hash[T.any(String, Symbol), Object]) if obj_param_metadata.nil?
+
+        return obj_param_metadata.fetch(:field_name, obj_field.name)
+      end
+
+      form = T.let({}, T::Hash[T.any(String, Symbol), Object])
+      if data.respond_to?(:fields)
+        T.unsafe(data).fields.each do |field|
+          val = data.send(field.name)
+          next if val.nil?
+
+          metadata = field.metadata[:form]
+
+          next if metadata.nil?
+
+          field_name = metadata.fetch(:field_name, field.name)
+
+          if metadata[:json]
+            form[field_name] = ::Crystalline.to_json(val)
+          else
+            if metadata.fetch(:style, "form") == "form"
+              form = form.merge(
+                _populate_form(
+                  field_name,
+                  metadata.fetch(:explode, true),
+                  val,
+                  ",",
+                  &get_form_field_name
+                )
+              )
+            else
+              raise StandardError, "Invalid form style for field #{field.name}"
+            end
+          end
+        end
+      elsif data.is_a?(Hash)
+        data.each do |key, value|
+          form[key] = [val_to_string(value)]
+        end
+      else
+        raise StandardError, "Invalid request body type #{data.class}"
+      end
+
+      form
+    end
+  end
+end

data/lib/xapi_scraper/utils/headers.rb

@@ -0,0 +1,82 @@
+# Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+
+# typed: true
+# frozen_string_literal: true
+
+require "cgi"
+require "date"
+require "base64"
+require "sorbet-runtime"
+
+module XapiScraper
+  module Utils
+    extend T::Sig
+
+    sig {
+      params(headers_params: Object, gbls: T.nilable(T::Hash[Symbol, T::Hash[Symbol, T::Hash[Symbol, Object]]])).returns(
+        T::Hash[Symbol, String]
+      )
+    }
+    def self.get_headers(headers_params, gbls = nil)
+      return {} if headers_params.nil?
+
+      headers = {}
+      T.unsafe(headers_params).fields.each do |f|
+        metadata = f.metadata[:header]
+        next if metadata.nil?
+
+        value = _populate_from_globals(f.name, headers_params&.send(f.name), "header", gbls)
+        value = _serialize_header(metadata.fetch(:explode, false), value)
+        headers[metadata.fetch(:field_name, f.name)] = value if !T.must(value).empty?
+      end
+
+      headers
+    end
+
+    sig { params(explode: T::Boolean, obj: Object).returns(T.nilable(String)) }
+    def self._serialize_header(explode, obj)
+      return "" if obj.nil?
+
+      if obj.respond_to?(:fields)
+        items = []
+        T.unsafe(obj).fields.each do |obj_field|
+          obj_param_metadata = obj_field.metadata[:header]
+          next if obj_param_metadata.nil?
+
+          obj_field_name = obj_param_metadata.fetch(:field_name, obj_field.name)
+          next if obj_field_name == ""
+
+          val = obj.send(obj_field.name)
+          next if val.nil?
+
+          if explode
+            items.append("#{obj_field_name}=#{val_to_string(val)}")
+          else
+            items.append(obj_field_name)
+            items.append(val_to_string(val))
+          end
+        end
+
+        items.join(",") if !items.empty?
+      elsif obj.is_a?(Hash)
+        items = []
+        obj.each do |key, value|
+          next if value.nil?
+
+          if explode
+            items.append("#{key}=#{val_to_string(value)}")
+          else
+            items.append(key)
+            items.append(val_to_string(value))
+          end
+        end
+
+        items.join(",") if !items.empty?
+      elsif obj.is_a?(Array)
+        items = obj.filter { |v| !v.nil? }.map { |v| val_to_string(v) }.join(",")
+      else
+        val_to_string(obj)
+      end
+    end
+  end
+end

data/lib/xapi_scraper/utils/query_params.rb

@@ -0,0 +1,167 @@
+# Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+
+# typed: true
+# frozen_string_literal: true
+
+require "date"
+require "base64"
+require "sorbet-runtime"
+
+module XapiScraper
+  module Utils
+    extend T::Sig
+
+    sig {
+      params(
+        clazz: Object,
+        query_params: Object,
+        url_override: T.nilable(::String),
+        gbls: T.nilable(T::Hash[Symbol, T::Hash[Symbol, T::Hash[Symbol, Object]]])
+      )
+        .returns(T::Hash[T.any(String, Symbol), T::Array[String]])
+    }
+    def self.get_query_params(clazz, query_params, url_override = nil, gbls = nil)
+      parsed_params = T.let({}, T::Hash[T.any(String, Symbol), T::Array[String]])
+      if !url_override.nil?
+        parsed_url = URI.parse(url_override)
+        URI.decode_www_form(parsed_url.query.to_s).each do |key, value|
+          parsed_params[key] ||= []
+          T.must(parsed_params[key]) << value
+        end
+      end
+
+      params = T.let({}, T::Hash[T.any(String, Symbol), T::Array[String]])
+      T.unsafe(clazz).fields.each do |f|
+        request_metadata = f.metadata[:request]
+        next if !request_metadata.nil?
+
+        metadata = f.metadata[:query_param]
+        next if metadata.nil?
+
+        param_name = f.name
+        value = query_params&.send(param_name.to_sym)
+        value = _populate_from_globals(param_name, value, "queryParam", gbls)
+
+        f_name = metadata[:field_name]
+        serialization = metadata.fetch(:serialization, "")
+        if serialization != ""
+          params = params.merge(
+            _get_serialized_params(
+              metadata,
+              f_name,
+              value
+            )
+          )
+        else
+          style = metadata.fetch(:style, "form")
+          case style
+          when "deepObject"
+            params = params.merge(
+              _get_deep_object_query_params(
+                metadata,
+                f_name,
+                value
+              )
+            )
+          when "form"
+            params = params.merge(
+              _get_delimited_query_params(
+                metadata,
+                f_name,
+                value,
+                ","
+              )
+            )
+          when "pipeDelimited"
+            params = params.merge(
+              _get_delimited_query_params(
+                metadata,
+                f_name,
+                value,
+                "|"
+              )
+            )
+          else
+            raise StandardError, "not yet implemented"
+          end
+        end
+      end
+
+      params.merge(parsed_params)
+    end
+
+    sig do
+      params(metadata: T::Hash[Symbol, String], field_name: String, obj: Object)
+        .returns(T::Hash[T.any(String, Symbol), T::Array[String]])
+    end
+    def self._get_deep_object_query_params(metadata, field_name, obj)
+      params = T.let({}, T::Hash[T.any(String, Symbol), T::Array[String]])
+
+      return params if obj.nil?
+
+      if obj.respond_to?(:fields)
+        T.unsafe(obj).fields.each do |obj_field|
+          obj_param_metadata = obj_field.metadata[:query_param]
+          next if obj_param_metadata.nil?
+
+          val = obj.send(obj_field.name)
+          next if val.nil?
+
+          key = "#{metadata.fetch(:field_name, field_name)}[#{obj_param_metadata.fetch(:field_name, obj_field.name)}]"
+          if val.is_a?(Array)
+            val.each do |v|
+              next if v.nil?
+
+              params[key] = [] if !params.include?(key)
+
+              T.must(params[key]) << val_to_string(v)
+            end
+          else
+            params[key] = [val_to_string(val)]
+          end
+        end
+      elsif obj.is_a?(Hash)
+        obj.each do |key, value|
+          next if value.nil?
+
+          param_key = "#{metadata.fetch(:field_name, field_name)}[#{key}]"
+          if value.is_a?(Array)
+            value.each do |val|
+              next if val.nil?
+
+              params[param_key] = [] if !params.include?(param_key)
+
+              T.must(params[param_key]).append(val_to_string(val))
+            end
+          else
+            params[param_key] = [val_to_string(value)]
+          end
+        end
+      end
+
+      params
+    end
+
+    sig do
+      params(metadata: T::Hash[Symbol, String], field_name: String, obj: Object, delimiter: String)
+        .returns(T::Hash[T.any(String, Symbol), T::Array[String]])
+    end
+    def self._get_delimited_query_params(metadata, field_name, obj, delimiter)
+      get_query_param_field_name = lambda do |obj_field|
+        obj_param_metadata = obj_field.metadata[:query_param]
+
+        return {} if obj_param_metadata.nil?
+
+        return obj_param_metadata.fetch(:field_name, obj_field.name)
+      end
+
+      _populate_form(
+        field_name,
+        T.cast(metadata.fetch(:explode, true), T::Boolean),
+        obj,
+        delimiter,
+        &get_query_param_field_name
+      )
+    end
+  end
+end

data/lib/xapi_scraper/utils/request_bodies.rb

@@ -0,0 +1,79 @@
+# Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+
+# typed: true
+# frozen_string_literal: true
+
+require "cgi"
+require "date"
+require "base64"
+require "sorbet-runtime"
+
+module XapiScraper
+  module Utils
+    extend T::Sig
+
+    SERIALIZATION_METHOD_TO_CONTENT_TYPE = {
+      'json': "application/json",
+      'form': "application/x-www-form-urlencoded",
+      'multipart': "multipart/form-data",
+      'raw': "application/octet-stream",
+      'string': "text/plain"
+    }.freeze
+
+    sig do
+      params(
+        request: Object,
+        nullable: T::Boolean,
+        optional: T::Boolean,
+        request_field_name: Symbol,
+        serialization_method: Symbol
+      )
+        .returns([String, Object, T.nilable(T::Array[T::Array[T.any(T::Array[T.nilable(String)], String)]])])
+    end
+    def self.serialize_request_body(request, nullable, optional, request_field_name, serialization_method)
+      return ["", nil, nil] if request.nil? && !nullable && optional
+
+      if !request.respond_to?(:fields) || !request.respond_to?(request_field_name)
+        return serialize_content_type(SERIALIZATION_METHOD_TO_CONTENT_TYPE[serialization_method], request)
+      end
+
+      request_metadata = T.let(nil, T.nilable(T::Array[T::Array[T.any(T::Array[T.nilable(String)], String)]]))
+      T.unsafe(request).fields.each do |f|
+        if f.name == request_field_name
+          request_metadata = f.metadata[:request]
+          break
+        end
+      end
+
+      if request_metadata.nil?
+        # Field name collision: model has a field matching request_field_name but without :request metadata.
+        # Serialize the entire object as the request body.
+        return serialize_content_type(SERIALIZATION_METHOD_TO_CONTENT_TYPE[serialization_method], request)
+      end
+
+      request_val = request.send(request_field_name)
+      return ["", nil, nil] if request_val.nil?
+
+      serialize_content_type(
+        request_metadata.fetch(:media_type, "application/octet-stream"),
+        request_val
+      )
+    end
+
+    sig do
+      params(media_type: String, request: Object)
+        .returns([String, Object, T.nilable(T::Array[T::Array[T.any(T::Array[T.nilable(String)], String)]])])
+    end
+    def self.serialize_content_type(media_type, request)
+      if media_type.match("^(application|text)\\/([^+]+\\+)*json.*")
+        return media_type, ::Crystalline.to_json(request), nil
+      end
+
+      return serialize_multipart_form(media_type, request) if media_type.match("^multipart\\/.*")
+      return media_type, serialize_form_data(request), nil if media_type.match("^application\\/x-www-form-urlencoded.*")
+      return media_type, request, nil if request.is_a?(String) || request.is_a?(Array)
+
+      raise StandardError, "invalid request body type #{request.class} for mediaType #{media_type}"
+    end
+  end
+end

data/lib/xapi_scraper/utils/retries.rb

@@ -0,0 +1,139 @@
+# Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+
+# typed: true
+# frozen_string_literal: true
+
+require "faraday/retry"
+require "time"
+require "sorbet-runtime"
+
+module XapiScraper
+  module Utils
+    class BackoffStrategy
+      extend T::Sig
+
+      sig { returns(T.nilable(::Float)) }
+      attr_accessor :exponent
+
+      sig { returns(T.nilable(::Integer)) }
+      attr_accessor :initial_interval
+
+      sig { returns(T.nilable(::Integer)) }
+      attr_accessor :max_elapsed_time
+
+      sig { returns(T.nilable(::Integer)) }
+      attr_accessor :max_interval
+
+      sig do
+        params(
+          exponent: T.nilable(::Float),
+          initial_interval: T.nilable(::Integer),
+          max_elapsed_time: T.nilable(::Integer),
+          max_interval: T.nilable(::Integer)
+        )
+          .void
+      end
+      def initialize(exponent: nil, initial_interval: nil, max_elapsed_time: nil, max_interval: nil)
+        @exponent = T.let(exponent, T.nilable(::Float))
+        @initial_interval = T.let(initial_interval, T.nilable(::Integer))
+        @max_elapsed_time = T.let(max_elapsed_time, T.nilable(::Integer))
+        @max_interval = T.let(max_interval, T.nilable(::Integer))
+      end
+    end
+
+    class RetryConfig
+      extend T::Sig
+
+      sig { returns(T.nilable(BackoffStrategy)) }
+      attr_accessor :backoff
+
+      sig { returns(T.nilable(T::Boolean)) }
+      attr_accessor :retry_connection_errors
+
+      sig { returns(T.nilable(::String)) }
+      attr_accessor :strategy
+
+      sig do
+        params(
+          backoff: T.nilable(BackoffStrategy),
+          retry_connection_errors: T.nilable(T::Boolean),
+          strategy: T.nilable(::String)
+        )
+          .void
+      end
+      def initialize(backoff: nil, retry_connection_errors: nil, strategy: nil)
+        @backoff = T.let(backoff, T.nilable(BackoffStrategy))
+        @retry_connection_errors = T.let(retry_connection_errors, T.nilable(T::Boolean))
+        @strategy = T.let(strategy, T.nilable(::String))
+      end
+
+      sig { params(initial_time: ::Time).returns(::Hash) }
+      def to_faraday_retry_options(initial_time:)
+        retry_options = {
+          # must overwrite default max of 2 retries and it must be positive
+          max: 1_000_000_000,
+          # ensure all HTTP methods are retried, especially via retry_if
+          methods: []
+        }
+
+        if @retry_connection_errors
+          retry_options[:exceptions] = Faraday::Retry::Middleware::DEFAULT_EXCEPTIONS + [Faraday::ConnectionFailed]
+        end
+
+        if @strategy == "backoff" && @backoff
+          retry_options[:backoff_factor] = @backoff.exponent unless @backoff.exponent.nil?
+          retry_options[:interval] = (@backoff.initial_interval.to_f / 1000) unless @backoff.initial_interval.nil?
+          retry_options[:max_interval] = (@backoff.max_interval.to_f / 1000) unless @backoff.max_interval.nil?
+
+          unless @backoff.max_elapsed_time.nil?
+            stop_time = initial_time + (@backoff.max_elapsed_time.to_f / 1000)
+            retry_options[:retry_if] = -> (_env, _exc) { Time.now < stop_time }
+          end
+
+        end
+
+        retry_options
+      end
+    end
+
+    class RetryMiddleware < Faraday::Retry::Middleware
+      extend T::Sig
+
+      sig { params(retries: T.untyped, env: T.untyped).returns(T.untyped) }
+      def calculate_sleep_amount(retries, env)
+        retry_after = [calculate_retry_after(env), calculate_rate_limit_reset(env)].compact.max
+        retry_interval = calculate_retry_interval(retries)
+        return retry_interval if retry_after.nil?
+
+        # unlike the stock middleware, cap server delays at max_interval rather
+        # than abandoning the retry
+        [[retry_after, @options.max_interval].min, retry_interval].max
+      end
+
+      private
+
+      sig { params(env: T.untyped).returns(T.untyped) }
+      def calculate_retry_after(env)
+        response_headers = env[:response_headers]
+        return unless response_headers
+
+        ms_value = response_headers["retry-after-ms"]
+        return ms_value.to_f / 1000 if ms_value&.match?(/\A\s*\d+\s*\z/)
+
+        value = response_headers["retry-after"]
+        return if value.nil? || value.strip.empty?
+        return value.to_f if value.match?(/\A\s*\d+\s*\z/)
+
+        begin
+          Time.httpdate(value) - Time.now
+        rescue ArgumentError
+          begin
+            Time.iso8601(value) - Time.now
+          rescue ArgumentError
+            nil
+          end
+        end
+      end
+    end
+  end
+end