withruntime 0.1.0

data/lib/withruntime/tar.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "zlib"
+
+module WithRuntime
+  # A small ustar writer and reader for directory uploads, downloads and build contexts.
+  module Tar
+    module_function
+
+    def header(name, size, mode, type, mtime = 0)
+      bytes = name.b
+      prefix = "".b
+      if bytes.bytesize > 100
+        split = name.rindex("/", 155)
+        raise ArgumentError, "Path too long for a tar archive: #{name}" if split.nil? || split <= 0 || name[(split + 1)..].bytesize > 100
+
+        prefix = name[0, split].b
+        bytes = name[(split + 1)..].b
+      end
+      block = "\0".b * 512
+      put = ->(offset, value, length) { block[offset, [length, value.bytesize].min] = value.b[0, length] }
+      octal = ->(value, length) { "#{value.to_s(8).rjust(length - 1, "0")}\0" }
+      put.call(0, bytes, 100)
+      put.call(100, octal.call(mode & 0o7777, 8), 8)
+      put.call(108, octal.call(0, 8), 8)
+      put.call(116, octal.call(0, 8), 8)
+      put.call(124, octal.call(size, 12), 12)
+      put.call(136, octal.call(mtime, 12), 12)
+      put.call(148, " " * 8, 8)
+      put.call(156, type, 1)
+      put.call(257, "ustar\u000000", 8)
+      put.call(345, prefix, 155)
+      put.call(148, "#{block.sum(64).to_s(8).rjust(6, "0")}\0 ", 8)
+      block
+    end
+
+    def pad(size) = "\0".b * ((512 - (size % 512)) % 512)
+
+    def gzip(data)
+      out = StringIO.new("".b)
+      writer = Zlib::GzipWriter.new(out, 6)
+      writer.mtime = 0
+      writer.write(data)
+      writer.close
+      out.string
+    end
+
+    # A directory as a gzipped tar: directories and regular files; links stay behind.
+    def pack_directory(root)
+      tar = "".b
+      base = File.expand_path(root)
+      Dir.glob("**/*", File::FNM_DOTMATCH, base: base).sort.each do |relative|
+        next if relative.split("/").any? { |part| [".", ".."].include?(part) }
+
+        full = File.join(base, relative)
+        stat = File.lstat(full)
+        if stat.directory?
+          tar << header("#{relative}/", 0, stat.mode, "5", stat.mtime.to_i)
+        elsif stat.file?
+          data = File.binread(full)
+          tar << header(relative, data.bytesize, stat.mode, "0", stat.mtime.to_i) << data << pad(data.bytesize)
+        end
+      end
+      gzip(tar << ("\0".b * 1024))
+    end
+
+    # Unpacks a gzipped tar into target, refusing any entry that would land outside it, and links.
+    def unpack(archive, target)
+      data = Zlib::GzipReader.new(StringIO.new(archive)).read.b
+      root = File.expand_path(target)
+      FileUtils.mkdir_p(root)
+      field = ->(start, length) { data[start, length].split("\0", 2).first.to_s.force_encoding("UTF-8") }
+      offset = 0
+      long_name = nil
+      while offset + 512 <= data.bytesize
+        break if data[offset, 512].count("\0") == 512
+
+        size = field.call(offset + 124, 12).strip.then { |text| text.empty? ? 0 : text.to_i(8) }
+        type = data[offset + 156]
+        prefix = field.call(offset + 345, 155)
+        name = long_name || (prefix.empty? ? field.call(offset, 100) : "#{prefix}/#{field.call(offset, 100)}")
+        long_name = nil
+        mode = field.call(offset + 100, 8).strip.then { |text| text.empty? ? 0o644 : text.to_i(8) }
+        body = data[offset + 512, size]
+        offset += 512 + (((size + 511) / 512) * 512)
+        if type == "L"
+          long_name = body.split("\0", 2).first.force_encoding("UTF-8")
+          next
+        end
+        name = name.delete_prefix("./")
+        next if name.empty? || name == "." || name == "./"
+
+        destination = File.expand_path(name, root)
+        raise IOError, "Refusing an archive entry outside the target: #{name}" unless destination.start_with?("#{root}/")
+
+        case type
+        when "5" then FileUtils.mkdir_p(destination)
+        when "0", "\0", "7"
+          FileUtils.mkdir_p(File.dirname(destination))
+          File.binwrite(destination, body)
+          File.chmod(mode & 0o777, destination)
+        end
+      end
+    end
+  end
+end

data/lib/withruntime/transport.rb

@@ -0,0 +1,362 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "securerandom"
+require "uri"
+
+module WithRuntime
+  # One connection pool, one retry policy and one error shape for every
+  # product: the same rules as the JavaScript, Python, Go and Java SDKs. Writes
+  # carry an idempotency key, made per call and kept across the client's own
+  # retries; transport failures, 429, 502, 503 and 504 are retried with backoff
+  # and jitter; a create waits out a full trial, quota or region.
+  class Transport
+    class OpeningCancelled < IOError; end
+    class Cancellation
+      def initialize
+        @lock = Mutex.new
+        @cancelled = false
+        @action = nil
+        @changed = ConditionVariable.new
+      end
+      def cancelled? = @lock.synchronize { @cancelled }
+      def register(&action)
+        run = @lock.synchronize { @action = action; @cancelled }
+        action.call if run
+      end
+      def wait(seconds)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds
+        @lock.synchronize do
+          until @cancelled
+            left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            return true if left <= 0
+            @changed.wait(@lock, left)
+          end
+        end
+        false
+      end
+      def cancel
+        action = @lock.synchronize { @cancelled = true; @changed.broadcast; @action }
+        action&.call
+      end
+    end
+    NETWORK_ERRORS = [IOError, SystemCallError, SocketError, Timeout::Error, OpenSSL::SSL::SSLError,
+                      Net::HTTPBadResponse, Net::ProtocolError, EOFError].freeze
+
+    attr_reader :base_url, :api_key, :timeout, :max_retries, :wait_for_capacity
+
+    def initialize(api_key:, base_url:, timeout:, max_retries:, max_connections:, wait_for_capacity:)
+      @api_key = api_key
+      @base_url = base_url
+      @uri = URI(base_url)
+      @timeout = timeout
+      @max_retries = max_retries
+      @wait_for_capacity = wait_for_capacity
+      @limit = [1, max_connections].max
+      @idle = []
+      @open = 0
+      @lock = Mutex.new
+      @freed = ConditionVariable.new
+      @client = "sdk-ruby/#{VERSION}"
+    end
+
+    # Percent-encodes one path segment, as encodeURIComponent does.
+    def self.segment(value)
+      value.to_s.b.gsub(/[^A-Za-z0-9\-_.!~*'()]/) { |c| format("%%%02X", c.ord) }
+    end
+
+    # The value of a JSON call: a Hash, an Array or nil.
+    def json(method, path, query: nil, body: nil, **options)
+      with_connection(options) do |http|
+        response = send_call(http, method, path, query: query, body: body, **options)
+        text = response.body.to_s
+        text.strip.empty? ? nil : JSON.parse(text)
+      rescue JSON::ParserError => e
+        raise Error.new("Unexpected answer from #{method} #{path}: #{e.message}", code: "unexpected_answer")
+      end
+    end
+
+    def bytes(method, path, query: nil, body: nil, **options)
+      with_connection(options) do |http|
+        send_call(http, method, path, query: query, body: body, **options).body.to_s.b
+      end
+    end
+
+    # Newline-delimited JSON events as they arrive, one Hash per event, on a
+    # connection of their own.
+    def events(method, path, query: nil, body: nil, cancel: nil, **options, &block)
+      return enum_for(:events, method, path, query: query, body: body, cancel: cancel, **options) unless block
+
+      cancellation = Cancellation.new
+      cancel&.register { cancellation.cancel }
+      http = connect
+      begin
+        buffer = +""
+        send_call(http, method, path, query: query, body: body, accept: "application/x-ndjson", cancel: cancellation, **options) do |chunk|
+          buffer << chunk
+          while (newline = buffer.index("\n"))
+            line = buffer.slice!(0, newline + 1).strip
+            yield JSON.parse(line) unless line.empty?
+          end
+        end
+        yield JSON.parse(buffer) unless buffer.strip.empty?
+      rescue JSON::ParserError => e
+        raise Error.new("Unexpected stream line: #{e.message}", code: "unexpected_answer")
+      ensure
+        cancellation.cancel
+        begin
+          http.finish if http.started?
+        rescue IOError
+          nil
+        end
+      end
+    end
+
+    def connection_error(timed_out, write, key, cause = nil)
+      if timed_out
+        ConnectionError.new("The call was cancelled or ran past its deadline.", code: "timeout",
+                            hint: "Allow a longer timeout for a long call, or retry with the same idempotency key.",
+                            idempotency_key: key)
+      else
+        message = "No answer from Runtime at #{base_url}."
+        message += " The change may have happened; retrying with the same idempotency key is safe." if write
+        message += " (#{cause.class}: #{cause.message})" if cause
+        ConnectionError.new(message, code: "connection_error",
+                                     hint: "Check the network, HTTPS_PROXY, and RUNTIME_API_URL if you set it.",
+                                     idempotency_key: key)
+      end
+    end
+
+    private
+
+    # HTTPS_PROXY (or HTTP_PROXY for an http origin) and NO_PROXY, read the
+    # way curl reads them. Net::HTTP's own :ENV reads http_proxy even for HTTPS.
+    def self.proxy_for(uri, env = ENV)
+      names = uri.scheme == "https" ? %w[HTTPS_PROXY https_proxy] : %w[HTTP_PROXY http_proxy]
+      value = names.map { |name| env[name] }.find { |found| found && !found.strip.empty? }
+      return nil unless value
+
+      # URI's matcher knows neither curl's "*" (no proxy at all) nor "*.example.com".
+      exempt = (env["NO_PROXY"] || env["no_proxy"]).to_s.split(",").map(&:strip).reject(&:empty?)
+      return nil if exempt.include?("*")
+      exempt = exempt.map { |entry| entry.delete_prefix("*") }.join(",")
+      return nil if !exempt.empty? && !URI::Generic.use_proxy?(uri.host, uri.host, uri.port, exempt)
+
+      value = "http://#{value}" unless value.include?("://")
+      URI(value.strip)
+    end
+
+    def connect
+      proxy = Transport.proxy_for(@uri)
+      http = if proxy
+               Net::HTTP.new(@uri.host, @uri.port, proxy.host, proxy.port,
+                             proxy.user && URI.decode_www_form_component(proxy.user),
+                             proxy.password && URI.decode_www_form_component(proxy.password))
+             else
+               Net::HTTP.new(@uri.host, @uri.port, nil)
+             end
+      http.use_ssl = @uri.scheme == "https"
+      http.max_retries = 0
+      http.open_timeout = 30
+      http.keep_alive_timeout = 30
+      http
+    end
+
+    # Net::HTTP keeps its socket local until TCP, proxy CONNECT and TLS finish.
+    # Keep that opening alone in an owned worker: raising an IOError there runs
+    # Net::HTTP's connect rescue and closes its local socket. Never interrupt the
+    # caller's event block, and join before returning so cancellation leaks no
+    # opening thread or socket.
+    def open_connection(http, cancel)
+      return if http.started?
+      return http.start unless cancel
+
+      opened = false
+      gate = Queue.new
+      worker = Thread.new do
+        Thread.current.report_on_exception = false
+        gate.pop
+        http.start
+      end
+      worker.report_on_exception = false
+      cancel.register { worker.raise(OpeningCancelled, "Stream connection cancelled.") if worker.alive? }
+      gate.push(true)
+      worker.value
+      opened = true
+    ensure
+      if worker&.alive?
+        worker.raise(OpeningCancelled, "Stream connection cancelled.")
+        begin
+          worker.join
+        rescue *NETWORK_ERRORS
+          nil
+        end
+      end
+      # connect can assign @socket immediately before do_start marks the
+      # session started. An interruption between those lines skips connect's
+      # rescue, so finish/started? alone cannot clean up this failed opening.
+      if worker && !opened
+        begin
+          http.instance_variable_get(:@socket)&.close
+        rescue IOError, SystemCallError
+          nil
+        end
+      end
+    end
+
+    def with_connection(_options)
+      http = @lock.synchronize do
+        loop do
+          break @idle.pop unless @idle.empty?
+          if @open < @limit
+            @open += 1
+            break connect
+          end
+          @freed.wait(@lock)
+        end
+      end
+      healthy = false
+      begin
+        result = yield http
+        healthy = true
+        result
+      ensure
+        @lock.synchronize do
+          if healthy
+            @idle.push(http)
+          else
+            http.finish if http.started?
+            @open -= 1
+          end
+          @freed.signal
+        end
+      end
+    end
+
+    def backoff(attempt) = [8.0, 0.25 * (2**[attempt, 10].min)].min * (0.5 + rand)
+
+    # 0.5 s, 1 s, 2 s, 4 s, then every 8 s, jittered, so a queue of CI jobs spreads out.
+    def room_backoff(attempt) = [8.0, 0.5 * (2**[attempt, 10].min)].min * (0.75 + (rand * 0.5))
+
+    def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+    def target(path, query)
+      pairs = []
+      (query || {}).each do |key, value|
+        next if value.nil?
+
+        Array(value).each { |item| pairs << [key.to_s, item.to_s] }
+      end
+      pairs.empty? ? path : "#{path}?#{URI.encode_www_form(pairs)}"
+    end
+
+    # Sends a call, retrying as the rules say, and returns the response once it
+    # succeeded. With a block, the body is streamed to it.
+    def send_call(http, method, path, query: nil, body: nil, raw: nil, accept: "application/json", wait: nil,
+                  key: nil, no_retry: false, room: 0, timeout: nil, cancel: nil, &stream)
+      write = method != "GET"
+      key = write ? (key || SecureRandom.uuid) : nil
+      deadline = now + (timeout || @timeout) + room
+      room_until = now + room
+      room_attempt = 0
+      payload = raw || (body.nil? ? nil : JSON.generate(body))
+      attempt = 0
+      loop do
+        raise connection_error(true, write, key) if cancel&.cancelled?
+        left = deadline - now
+        raise connection_error(true, write, key) if left <= 0
+
+        request = Net::HTTPGenericRequest.new(method, !payload.nil?, true, target(path, query))
+        request["Authorization"] = "Bearer #{api_key}"
+        request["Accept"] = accept
+        request["X-Runtime-Client"] = @client
+        request["User-Agent"] = "withruntime-ruby/#{VERSION}"
+        request["Idempotency-Key"] = key if key
+        request["Prefer"] = "wait=#{[120, wait].min}" if wait&.positive?
+        if payload
+          request["Content-Type"] = raw ? "application/octet-stream" : "application/json"
+          request.body = payload
+        end
+        http.read_timeout = left
+        http.write_timeout = left if http.respond_to?(:write_timeout=)
+        response = nil
+        begin
+          open_connection(http, cancel)
+          cancel&.register do
+            http.read_timeout = 0.001
+            http.write_timeout = 0.001
+            http.open_timeout = 0.001
+            begin
+              http.instance_variable_get(:@socket)&.io&.close
+            rescue IOError, SystemCallError
+              nil
+            end
+          end
+          raise connection_error(true, write, key) if cancel&.cancelled?
+          if stream
+            http.request(request) do |answer|
+              response = answer
+              answer.read_body(&stream) if answer.code.to_i < 300
+            end
+          else
+            response = http.request(request)
+          end
+          status = response.code.to_i
+          if status >= 300 && status < 400
+            raise Net::HTTPBadResponse, "Runtime answered a redirect (#{status})"
+          end
+        rescue *NETWORK_ERRORS => e
+          http.finish if http.started?
+          late = e.is_a?(Timeout::Error) || now >= deadline || cancel&.cancelled?
+          raise connection_error(late, write, key, e) if late || no_retry || attempt >= max_retries
+
+          pause(backoff(attempt), deadline, write, key, nil, cancel)
+          attempt += 1
+          next
+        end
+        return response if status < 300
+
+        text = response.body.to_s
+        parsed = begin
+          JSON.parse(text)
+        rescue JSON::ParserError
+          { "error" => { "message" => text.strip.empty? ? nil : text.strip[0, 500] }.compact }
+        end
+        failure = Error.from(status, parsed, key)
+        if !no_retry && room.positive? && Error::WAITS_FOR_ROOM.include?(failure.code) && failure.details["field"] != "count"
+          room_left = room_until - now
+          raise failure if room_left <= 0
+
+          wait_for = failure.retry_after || room_backoff(room_attempt)
+          room_attempt += 1
+          pause([room_left, wait_for].min, deadline, write, key, failure, cancel)
+          next
+        end
+        retryable = [429, 502, 503, 504].include?(status) && !Error::DELIBERATE.include?(failure.code)
+        raise failure if no_retry || !retryable || attempt >= max_retries
+
+        wait_for = failure.retry_after
+        if wait_for.nil?
+          header = Float(response["Retry-After"], exception: false)
+          wait_for = header&.positive? ? header : backoff(attempt)
+        end
+        pause([30.0, wait_for].min * (0.9 + (rand * 0.2)), deadline, write, key, failure, cancel)
+        attempt += 1
+      end
+    end
+
+    def pause(seconds, deadline, write, key, failure = nil, cancel = nil)
+      if now + seconds >= deadline
+        raise failure if failure
+
+        raise connection_error(true, write, key)
+      end
+      if cancel
+        raise connection_error(true, write, key) unless cancel.wait(seconds)
+      else
+        sleep(seconds)
+      end
+    end
+  end
+end