files.com 1.1.813 → 1.1.815

data/lib/files.com/transfer/options.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Files
+  module Transfer
+    # Lets another thread stop an adaptive transfer. cancel and pause stop new requests at once; the transfer then
+    # waits for the requests already in flight, releases the memory and connections it holds and raises
+    # CancelledError or PausedError. The first of the two wins, and a Control stays stopped: resume with a new one.
+    class Control
+      def initialize
+        @lock = Mutex.new
+        @state = :running
+        @listeners = []
+      end
+
+      def cancel
+        change(:cancelled)
+      end
+
+      def pause
+        change(:paused)
+      end
+
+      def cancelled?
+        @state == :cancelled
+      end
+
+      def paused?
+        @state == :paused
+      end
+
+      # :running, :cancelled or :paused.
+      attr_reader :state
+
+      # A listener's wake is called after each change, so jobs waiting on this control notice it at once.
+      def subscribe(listener)
+        @lock.synchronize { @listeners << listener }
+      end
+
+      def unsubscribe(listener)
+        @lock.synchronize { @listeners.delete(listener) }
+      end
+
+      private def change(state)
+        listeners = @lock.synchronize do
+          return false unless @state == :running
+
+          @state = state
+          @listeners.dup
+        end
+        listeners.each(&:wake)
+        true
+      end
+    end
+
+    # What an adaptive transfer may use. The transfer: option takes these as a Hash, true for all the defaults, an
+    # Options or a Job that several transfers share.
+    #
+    # concurrency:: :automatic (the default) tunes the requests in flight from what the link sustains; an Integer from
+    #               1 to 256 keeps exactly that many.
+    # startup:: With :automatic, :default starts an upload to cloud storage with one request and grows while each
+    #           step pays off; :aggressive starts at 32. Downloads and direct destinations ignore it.
+    # max_concurrency:: With :automatic, the most requests in flight, from 1 to 256. The default is 64 for cloud
+    #                   storage and downloads and 16 for direct destinations, which it can only lower.
+    # buffer_bytes:: The most bytes held in memory at once for parts waiting to be sent or written, at least 65,536.
+    #                The default is 256 MiB.
+    # max_files:: How many files a shared Job transfers at once; more wait their turn. The default is 4.
+    # control:: A Control that cancels or pauses the transfer.
+    class Options
+      MAX_CONCURRENCY = 256
+      DEFAULT_BUFFER_BYTES = 256 * 1024 * 1024
+      MIN_BUFFER_BYTES = 64 * 1024
+      DEFAULT_MAX_FILES = 4
+      KEYS = %i[concurrency startup max_concurrency buffer_bytes max_files control].freeze
+      STARTUPS = %i[default aggressive].freeze
+
+      attr_reader :concurrency, :startup, :max_concurrency, :buffer_bytes, :max_files, :control
+
+      def self.from(value)
+        case value
+        when Options then value
+        when true then new
+        when Hash
+          unknown = value.keys - KEYS
+          raise InvalidParameterError, "Bad parameter: transfer does not take #{unknown.map(&:inspect).join(', ')}" unless unknown.empty?
+
+          new(**value)
+        else
+          raise InvalidParameterError, "Bad parameter: transfer must be true, a Hash of transfer options, a Files::Transfer::Options or a Files::Transfer::Job"
+        end
+      end
+
+      def initialize(concurrency: :automatic, startup: :default, max_concurrency: nil, buffer_bytes: DEFAULT_BUFFER_BYTES, max_files: DEFAULT_MAX_FILES, control: nil)
+        @concurrency = concurrency
+        @startup = startup
+        @max_concurrency = max_concurrency
+        @buffer_bytes = buffer_bytes
+        @max_files = max_files
+        @control = control
+        validate
+        freeze
+      end
+
+      def automatic?
+        @concurrency == :automatic
+      end
+
+      def to_h
+        KEYS.to_h { |key| [ key, public_send(key) ] }
+      end
+
+      private def validate
+        raise InvalidParameterError, "Bad parameter: transfer concurrency must be :automatic or an Integer from 1 to #{MAX_CONCURRENCY}" unless automatic? || concurrency_count?(@concurrency)
+        raise InvalidParameterError, "Bad parameter: transfer startup must be :default or :aggressive" unless STARTUPS.include?(@startup)
+        raise InvalidParameterError, "Bad parameter: transfer startup applies only to automatic concurrency" if !automatic? && @startup != :default
+        raise InvalidParameterError, "Bad parameter: transfer max_concurrency must be an Integer from 1 to #{MAX_CONCURRENCY}, with automatic concurrency" unless @max_concurrency.nil? || (automatic? && concurrency_count?(@max_concurrency))
+        raise InvalidParameterError, "Bad parameter: transfer buffer_bytes must be an Integer of at least #{MIN_BUFFER_BYTES}" unless @buffer_bytes.is_a?(Integer) && @buffer_bytes >= MIN_BUFFER_BYTES
+        raise InvalidParameterError, "Bad parameter: transfer max_files must be a positive Integer" unless @max_files.is_a?(Integer) && @max_files.positive?
+        raise InvalidParameterError, "Bad parameter: transfer control must be a Files::Transfer::Control" unless @control.nil? || @control.is_a?(Control)
+      end
+
+      private def concurrency_count?(value)
+        value.is_a?(Integer) && value.between?(1, MAX_CONCURRENCY)
+      end
+    end
+  end
+end

data/lib/files.com/transfer/policy.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Files
+  module Transfer
+    # The numbers a Controller works within for one kind of destination. Every count is capped at max, so a caller's
+    # max_concurrency, or a fixed concurrency, always dominates.
+    class Policy
+      MIB = 1024 * 1024
+      # A warm-start window at least this fast ends an upload's warm start at the start concurrency.
+      FAST_LINK_BYTES_PER_SECOND = 96 * MIB
+      # A window is judged only when its requests in flight averaged at least this share of the limit.
+      ELIGIBLE_OCCUPANCY = 0.75
+      # A failure, a throttle and a growing queue shrink the limit by these shares, once per epoch.
+      FAILURE_SHRINK = 0.35
+      THROTTLE_SHRINK = 0.10
+      LATENCY_SHRINK = 0.08
+      # A probe is kept when a window's throughput beats the one before it by this share, and given up after this many
+      # windows that do not; the limit then holds for HOLD_WINDOWS windows.
+      PROBE_GAIN = 0.01
+      PROBE_WINDOWS = 2
+      HOLD_WINDOWS = 1
+      # After a rejected warm-start probe, the windows to rest before probing again by one.
+      GROWTH_INTERVAL_WINDOWS = 1
+      WARM_REJECTIONS = 2
+      # Seconds after a throttle, or its Retry-After when longer, during which completions are not measured.
+      THROTTLE_COOLDOWN = 1.0
+      DOWNLOAD_RANGE_BYTES = 8 * MIB
+      # Parts asked for in one begin_upload.
+      UPLOAD_BATCH_PARTS = 5
+      # The most requests cloud storage and downloads have in flight, and a job across all its transfers, unless the
+      # caller's max_concurrency or fixed concurrency says otherwise.
+      DEFAULT_MAX = 64
+
+      attr_reader :name, :initial, :start, :min, :soft, :max, :window_attempts, :growth_step, :queue_suppress, :queue_backoff, :latency_floor, :rejected_floor
+
+      # Cloud storage, such as S3. :default starts at one request and warms up toward 32; :aggressive starts at 32.
+      def self.storage_upload(startup: :default, max_concurrency: nil)
+        new(name: :storage, initial: startup == :aggressive ? 32 : 1, start: 32, min: 4, soft: 32, max: max_concurrency || DEFAULT_MAX,
+            growth_step: 2, queue_suppress: 20, queue_backoff: 34, latency_floor: 12, rejected_floor: 12, warm: startup == :default
+        )
+      end
+
+      # A destination the API reports a direct connection for, such as an agent. Its queue thresholds are cloud
+      # storage's 20 and 34, scaled to its maximum of 16.
+      def self.direct_upload(max_concurrency: nil)
+        new(name: :direct, initial: 4, start: 4, min: 1, soft: 16, max: [ 16, max_concurrency || 16 ].min,
+            growth_step: 1, queue_suppress: 5, queue_backoff: 9, latency_floor: 1, rejected_floor: 1
+        )
+      end
+
+      def self.download(max_concurrency: nil)
+        new(name: :download, initial: 32, start: 32, min: 4, soft: 32, max: max_concurrency || DEFAULT_MAX,
+            growth_step: 2, queue_suppress: 20, queue_backoff: 34, latency_floor: 12, rejected_floor: 12
+        )
+      end
+
+      def self.fixed(concurrency)
+        new(name: :fixed, initial: concurrency, start: concurrency, min: concurrency, soft: concurrency, max: concurrency,
+            growth_step: 0, queue_suppress: concurrency, queue_backoff: concurrency, latency_floor: concurrency, rejected_floor: concurrency, fixed: true
+        )
+      end
+
+      def initialize(name:, initial:, start:, min:, soft:, max:, growth_step:, queue_suppress:, queue_backoff:, latency_floor:, rejected_floor:,
+                     window_attempts: 16, warm: false, fixed: false)
+        @name = name
+        @max = max
+        @initial = [ initial, max ].min
+        @start = [ start, max ].min
+        @min = [ min, max ].min
+        @soft = [ soft, max ].min
+        @window_attempts = window_attempts
+        @growth_step = growth_step
+        @queue_suppress = queue_suppress
+        @queue_backoff = queue_backoff
+        @latency_floor = [ latency_floor, max ].min
+        @rejected_floor = [ rejected_floor, max ].min
+        @warm = warm && @initial < @start
+        @fixed = fixed
+        freeze
+      end
+
+      def warm?
+        @warm
+      end
+
+      def fixed?
+        @fixed
+      end
+    end
+  end
+end

data/lib/files.com/transfer/storage_retry.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Files
+  module Transfer
+    # What a storage response means for the request that got it, and when a signed URL stops working.
+    #
+    # 429 and 503 are throttles, retried after their whole Retry-After (seconds or an HTTP date) or at once without
+    # one; a Retry-After too large to schedule refuses the request instead. 500, 502, 504, a network failure and S3's
+    # RequestTimeout error (a 400 whose whole body, of at most 64 KiB, is a complete, DTD-free XML document whose
+    # unnamespaced root <Error> has a direct <Code>RequestTimeout</Code>) are retried after 100 ms. Throttles hold the
+    # destination back for every request of the job (Holds). Anything else is refused: 501, the other 4xx,
+    # 3xx (redirects are never followed) and a success that does not acknowledge the part. Each retry is a new
+    # admission, within the caller's Files.max_network_retries, and never for a part whose instructions say
+    # retry_parts: false.
+    module StorageRetry
+      THROTTLE_STATUSES = [ 429, 503 ].freeze
+      RETRY_STATUSES = [ 500, 502, 504 ].freeze
+      RETRY_DELAY = 0.1
+      # Seconds before a signed URL's expiry at which it is renewed rather than used.
+      EXPIRY_MARGIN = 5
+      FRONTEND_HEADER = "x-files-frontend-app"
+      # The most of a 400's body judged, which bounds the XML checked.
+      XML_MAX_BYTES = 64 * 1024
+      # An XML declaration, accepted only at the very start of the body.
+      XML_DECLARATION = /\A<\?xml\s+version\s*=\s*(["'])1\.[0-9]+\1(?:\s+encoding\s*=\s*(["'])[A-Za-z][A-Za-z0-9._-]*\2)?(?:\s+standalone\s*=\s*(["'])(?:yes|no)\3)?\s*\?>/
+      # A predefined entity or a character reference: no other entity can be defined, since a DTD is never accepted.
+      XML_REFERENCE = '&(?:amp|lt|gt|quot|apos|#[0-9]+|#x\h+);'
+      XML_VALUE = %{(?:"(?:[^"<&]|#{XML_REFERENCE})*"|'(?:[^'<&]|#{XML_REFERENCE})*')}.freeze
+      # One attribute: its name, with any prefix, and its quoted value.
+      XML_ATTRIBUTE = /\s+([A-Za-z_][\w.:-]*)\s*=\s*(#{XML_VALUE})/
+      # A start, end or empty-element tag, with its attributes, or character data. A comment, processing instruction,
+      # CDATA section or DTD matches nothing, so a body holding one is refused, as is a prefixed element name.
+      XML_TOKEN = /\G(?:<(\/?)([A-Za-z_][\w.-]*)((?:\s+[A-Za-z_][\w.:-]*\s*=\s*#{XML_VALUE})*)\s*(\/?)>|((?:[^<&]|#{XML_REFERENCE})+))/
+      XML_CHARACTER_REFERENCE = /&#(x\h+|[0-9]+);/
+      # Characters XML never allows: the C0 controls other than tab, newline and carriage return.
+      XML_FORBIDDEN = /[\x00-\x08\x0B\x0C\x0E-\x1F]/n
+      SIGNATURE_PARAMS = [ %w[x-amz-date x-amz-expires], %w[x-files-date x-files-expires], %w[x-goog-date x-goog-expires] ].freeze
+
+      # [:throttle, seconds], [:retry, seconds] or [:refuse] for a response that is not a success. headers are
+      # lowercase names, each with its values. truncated says the body is only the first part of a longer one, which a
+      # transport keeps no more of: such a 400 is never taken for RequestTimeout, since its whole document was not seen.
+      def self.classify(status, headers, body, truncated: false)
+        if THROTTLE_STATUSES.include?(status)
+          seconds = retry_after(headers.fetch("retry-after", []).first)
+          return seconds.finite? ? [ :throttle, seconds ] : [ :refuse ]
+        end
+        return [ :retry, RETRY_DELAY ] if RETRY_STATUSES.include?(status)
+        return [ :retry, RETRY_DELAY ] if status == 400 && !truncated && request_timeout?(body)
+
+        [ :refuse ]
+      end
+
+      # Seconds to wait, never negative and never shortened. Infinity when the value cannot be scheduled.
+      def self.retry_after(value, now: Time.now)
+        return 0.0 if value.nil?
+
+        text = value.strip
+        seconds = text.match?(/\A\d+\z/) ? text.to_f : Time.httpdate(text) - now
+        seconds.nan? ? Float::INFINITY : [ seconds, 0.0 ].max
+      rescue ArgumentError
+        0.0
+      end
+
+      # Whether body is exactly S3's RequestTimeout error: a well-formed XML document of at most 64 KiB in UTF-8, with
+      # no DTD, comment, processing instruction (but an XML declaration at its start), CDATA section or entity but the
+      # predefined ones, no namespace (neither an xmlns declaration nor a prefix on any element or attribute), no
+      # repeated attribute and no character XML disallows, whose root is <Error> with a single direct <Code> child
+      # holding only RequestTimeout. A nested or repeated Code, another code, or anything else is not.
+      def self.request_timeout?(body)
+        return false unless body.is_a?(String) && body.bytesize <= XML_MAX_BYTES
+
+        text = body.b
+        utf8 = text.dup.force_encoding(Encoding::UTF_8)
+        return false unless utf8.valid_encoding? && !text.match?(XML_FORBIDDEN) && !utf8.match?(/[￾￿]/)
+
+        text = text.sub(XML_DECLARATION, "").strip
+        stack = []
+        codes = []
+        rooted = false
+        position = 0
+        while position < text.bytesize
+          token = XML_TOKEN.match(text, position)
+          return false unless token&.begin(0) == position
+
+          position = token.end(0)
+          closing, name, attributes, empty, content = token.captures
+          if content
+            return false unless (stack.any? || content.strip.empty?) && valid_xml_text?(content)
+
+            codes[-1] << content if stack.size == 2 && stack.last == "Code" && codes[-1]
+          elsif closing == "/"
+            return false unless attributes.empty? && empty.empty? && stack.pop == name
+          else
+            return false unless valid_xml_attributes?(attributes)
+            return false if stack.empty? && (rooted || name != "Error")
+
+            rooted = true if stack.empty?
+            codes[-1] = nil if stack.size == 2 && stack.last == "Code"
+            codes << +"" if stack.size == 1 && name == "Code"
+            stack << name unless empty == "/"
+          end
+        end
+        rooted && stack.empty? && codes == [ "RequestTimeout" ]
+      end
+
+      # A tag's attributes: none in a namespace (no xmlns declaration, no prefix), none named twice, and each value well
+      # formed.
+      def self.valid_xml_attributes?(attributes)
+        names = []
+        attributes.scan(XML_ATTRIBUTE) do |name, value|
+          return false if name == "xmlns" || name.include?(":") || names.include?(name) || !valid_xml_text?(value)
+
+          names << name
+        end
+        true
+      end
+
+      # Character data or an attribute value: without "]]>", and with character references only to characters XML
+      # allows.
+      def self.valid_xml_text?(text)
+        !text.include?("]]>") && text.scan(XML_CHARACTER_REFERENCE).all? do |(digits)|
+          code = digits.start_with?("x") ? digits[1..].to_i(16) : digits.to_i
+          [ 0x9, 0xA, 0xD ].include?(code) || code.between?(0x20, 0xD7FF) || code.between?(0xE000, 0xFFFD) || code.between?(0x10000, 0x10FFFF)
+        end
+      end
+
+      def self.frontend?(headers)
+        headers.key?(FRONTEND_HEADER)
+      end
+
+      # The earliest of the API's expires and the expiry the signed URL states: S3's X-Amz-Date plus X-Amz-Expires,
+      # the same X-Files- and X-Goog- parameters, or an Azure SAS's se. nil when neither says.
+      def self.expires_at(url, api_expires = nil)
+        times = [ api_time(api_expires) ]
+        # Names compare without case, and the last of the parameters with a name wins, in the order the URL gives them.
+        query = {}
+        ::URI.decode_www_form(::URI.parse(url).query.to_s).each { |name, value| query[name.downcase] = value }
+        SIGNATURE_PARAMS.each do |date, lifetime|
+          signed = compact_time(query[date])
+          times << (signed + Integer(query[lifetime], 10)) if signed && query[lifetime].to_s.match?(/\A\d+\z/)
+        end
+        times << api_time(query["se"])
+        times.compact.min
+      rescue ArgumentError, ::URI::InvalidURIError
+        api_time(api_expires)
+      end
+
+      def self.expired?(expires_at, now: Time.now)
+        !expires_at.nil? && now >= expires_at - EXPIRY_MARGIN
+      end
+
+      def self.compact_time(text)
+        match = text.to_s.match(/\A(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z\z/)
+        match && Time.utc(*match.captures.map(&:to_i))
+      end
+
+      def self.api_time(value)
+        case value
+        when Time then value
+        when String then Time.iso8601(value)
+        end
+      rescue ArgumentError
+        nil
+      end
+    end
+  end
+end