carthage_remote_cache 0.0.8 → 0.0.11

data/lib/commands/upload_command.rb

@@ -1,10 +1,10 @@
-require 'concurrent'
+require "concurrent"
 
 class UploadCommand
   def self.new_with_defaults(options)
     shell = ShellWrapper.new
     config = Configuration.new(shell)
-    networking = Networking.new(config)
+    networking = Networking.new(config, options[:is_retry_enabled])
     api = API.new(shell, config, networking, options)
 
     UploadCommand.new(
@@ -62,7 +62,7 @@ class UploadCommand
   def upload(carthage_dependency)
     version_file = carthage_dependency.new_version_file
 
-    if @api.version_file_matches_server?(carthage_dependency, version_file)
+    if @api.version_file_matches_server?(carthage_dependency, version_file, nil)
       $LOG.debug("Version file #{version_file.path} matches server version, skipping upload")
       @mutex.synchronize do
         @number_of_skipped_archives += version_file.number_of_frameworks
@@ -72,9 +72,9 @@ class UploadCommand
 
     @networking.upload_version_file(carthage_dependency)
 
-    version_file.frameworks_by_platform.each do |platform, framework_names|
-      for framework_name in framework_names
-        archive_size = @api.create_and_upload_archive(carthage_dependency, framework_name, platform)
+    version_file.frameworks_by_platform.each do |platform, frameworks|
+      for framework in frameworks
+        archive_size = @api.create_and_upload_archive(carthage_dependency, framework, platform)
         @mutex.synchronize do
           @number_of_uploaded_archives += 1
           @total_archive_size += archive_size

data/lib/commands/verify_command.rb

@@ -2,7 +2,7 @@ class VerifyCommand
   def self.new_with_defaults(options)
     shell = ShellWrapper.new
     config = Configuration.new(shell)
-    networking = Networking.new(config)
+    networking = Networking.new(config, options[:is_retry_enabled])
     api = API.new(shell, config, networking, options)
 
     VerifyCommand.new(api: api)

data/lib/commands.rb

@@ -1,8 +1,8 @@
 lib = File.expand_path("..", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
-require 'commands/download_command'
-require 'commands/init_command'
-require 'commands/server_command'
-require 'commands/upload_command'
-require 'commands/verify_command'
+require "commands/download_command"
+require "commands/init_command"
+require "commands/server_command"
+require "commands/upload_command"
+require "commands/verify_command"

data/lib/configuration.rb

@@ -1,4 +1,4 @@
-require 'uri'
+require "uri"
 
 class Configuration
   class UserConfig

data/lib/constants.rb

@@ -1,12 +1,14 @@
-CARTHAGE_DIR = 'Carthage'
-CARTHAGE_BUILD_DIR = File.join(CARTHAGE_DIR, 'Build')
-CARTFILE_RESOLVED = 'Cartfile.resolved'
-CARTRCFILE = 'Cartrcfile'
+CARTHAGE_DIR = "Carthage"
+CARTHAGE_BUILD_DIR = File.join(CARTHAGE_DIR, "Build")
+CARTFILE_RESOLVED = "Cartfile.resolved"
+CARTRCFILE = "Cartrcfile"
 THREAD_POOL_SIZE = 8
 
 SERVER_DEFAULT_PORT = 9292
-SERVER_CACHE_DIR = File.join(Dir.home, '.carthagerc_server')
+SERVER_CACHE_DIR = File.join(Dir.home, ".carthagerc_server")
 
 ARCHIVE_CHECKSUM_HEADER_REST_CLIENT = :archive_checksum
-ARCHIVE_CHECKSUM_HEADER_SINATRA_IN = 'HTTP_ARCHIVE_CHECKSUM'
-ARCHIVE_CHECKSUM_HEADER_SINATRA_OUT = 'Archive-Checksum'
+ARCHIVE_CHECKSUM_HEADER_SINATRA_IN = "HTTP_ARCHIVE_CHECKSUM"
+ARCHIVE_CHECKSUM_HEADER_SINATRA_OUT = "Archive-Checksum"
+
+PLATFORMS = [:iOS, :macOS, :tvOS, :watchOS]

data/lib/crc32.rb

@@ -1,5 +1,5 @@
-require 'digest'
-require 'zlib'
+require "digest"
+require "zlib"
 
 class Digest::CRC32 < Digest::Class
   include Digest::Instance
@@ -17,6 +17,6 @@ class Digest::CRC32 < Digest::Class
   end
 
   def finish
-    [@crc32].pack('N')
+    [@crc32].pack("N")
   end
 end

data/lib/errors.rb

@@ -51,7 +51,7 @@ class FrameworkValidationError < AppError
   end
 
   def to_s
-    header = ['Framework', CARTHAGE_BUILD_DIR, CARTFILE_RESOLVED]
+    header = ["Framework", CARTHAGE_BUILD_DIR, CARTFILE_RESOLVED]
     rows = @errors.map { |e| [e.framework_name, e.build_version, e.cartfile_resolved_version] }
     table = Table.new(header, rows)
     <<~EOS
@@ -69,3 +69,14 @@ end
 class MissingFrameworkDirectoryError < AppError; end
 
 class ServerVersionMismatchError < AppError; end
+
+class PlatformMismatchError < AppError
+  def initialize(platform)
+    @platform = platform
+  end
+
+  def to_s
+    platforms = PLATFORMS.map(&:to_s).join(", ")
+    "Platform '#{@platform}' doesn't match any of: #{platforms}"
+  end
+end

data/lib/framework.rb

@@ -0,0 +1,19 @@
+# Example data:
+# - "name" : "Sentry",
+# - "linking" : "dynamic"
+class Framework
+  attr_reader :name
+
+  def self.parse(json)
+    Framework.new(json["name"], json["linking"])
+  end
+
+  def initialize(name, linking)
+    @name = name
+    @linking = linking
+  end
+
+  def make_archive(platform)
+    FrameworkCarthageArchive.new(name, platform)
+  end
+end

data/lib/framework_carthage_archive.rb

@@ -0,0 +1,57 @@
+require "fileutils"
+
+class FrameworkCarthageArchive < CarthageArchive
+
+  # Aggregate following files:
+  # - Carthage/Build/iOS/Alamofire.framework
+  # - Carthage/Build/iOS/Alamofire.framework/Alamofire
+  # - Carthage/Build/iOS/618BEB79-4C7F-3692-B140-131FB983AC5E.bcsymbolmap
+  # into Alamofire-iOS.zip
+  def compress_archive(shell, carthage_build_dir = CARTHAGE_BUILD_DIR)
+    $LOG.debug("Archiving #{@framework_name} for #{@platform}")
+
+    platform_path = File.join(carthage_build_dir, platform_to_carthage_dir_string(@platform))
+    framework_path = File.join(platform_path, "#{@framework_name}.framework")
+    raise MissingFrameworkDirectoryError.new, "Archive can't be created, no framework directory at #{framework_path}" unless Dir.exist?(framework_path)
+
+    # It's very likely, that binary releases don't contain DSYMs.
+    dsym_path = File.join(platform_path, "#{@framework_name}.framework.dSYM")
+    unless File.exist?(dsym_path)
+      $LOG.error("DSYM File #{dsym_path} not found, continuing")
+      dsym_path = nil
+    end
+
+    binary_path = File.join(framework_path, @framework_name)
+    raise AppError.new, "Binary #{binary_path} is missing, failed to read .bcsymbolmap files" unless File.exist?(binary_path)
+
+    bcsymbolmap_paths = find_bcsymbolmap_paths(shell, platform_path, binary_path)
+
+    input_paths = []
+    input_paths << framework_path
+    input_paths << dsym_path unless dsym_path.nil?
+    input_paths += bcsymbolmap_paths
+
+    $LOG.debug("Adding > #{input_paths.inspect}")
+
+    delete_archive
+    shell.archive(input_paths, @archive_path)
+    $LOG.debug("Created #{@archive_path} archive, file size: #{formatted_archive_size}")
+  end
+
+  private
+
+  def find_bcsymbolmap_paths(shell, platform_path, binary_path)
+    raw_dwarfdump = shell.dwarfdump(binary_path)
+    uuids = parse_uuids(raw_dwarfdump)
+    bcsymbolmap_paths = uuids.map { |uuid| File.join(platform_path, "#{uuid}.bcsymbolmap") }.select { |path| File.exist?(path) }
+    bcsymbolmap_paths
+  end
+
+  # Example dwarfdump link:
+  # UUID: 618BEB79-4C7F-3692-B140-131FB983AC5E (i386) Carthage/Build/iOS/CocoaLumberjackSwift.framework/CocoaLumberjackSwift
+  def parse_uuids(raw_dwarfdump)
+    lines = raw_dwarfdump.split("\n")
+    uuids = lines.map { |line| line[/^UUID: ([A-Z0-9\-]+)\s+\(.*$/, 1] }
+    uuids.compact
+  end
+end

data/lib/log.rb

@@ -1,4 +1,4 @@
-require 'logger'
+require "logger"
 
 $LOG = Logger.new(STDOUT)
 $LOG.level = Logger::INFO

data/lib/networking.rb

@@ -1,9 +1,10 @@
-require 'rest-client'
-require 'uri'
+require "rest-client"
+require "uri"
 
 class Networking
-  def initialize(config)
+  def initialize(config, is_retry_enabled)
     @config = config
+    @is_retry_enabled = is_retry_enabled
   end
 
   # Version
@@ -11,11 +12,13 @@ class Networking
   def get_server_version
     url = new_version_url
     $LOG.debug("Fetching server version from #{url}")
-    server_version = RestClient.get(url) do |response, request, result|
-      if response.code == 200
-        response.strip
-      else
-        raise AppError.new, "Failed to read server version from #{url}, response:\n  #{response[0...300]}"
+    server_version = perform_network_request do
+      RestClient.get(url) do |response, request, result|
+        if response.code == 200
+          response.strip
+        else
+          raise AppError.new, "Failed to read server version from #{url}, response:\n  #{response[0...300]}"
+        end
       end
     end
     server_version
@@ -24,15 +27,22 @@ class Networking
   # Version Files
 
   # @return VersionFile or nil
-  def download_version_file(carthage_dependency)
+  def download_version_file(carthage_dependency, platforms)
     url = new_version_file_url(carthage_dependency)
-    $LOG.debug("Downloading version file from #{url}")
-    version_file = RestClient.get(url) do |response, request, result|
-      if response.code == 200
-        File.write(carthage_dependency.version_filename, response.to_s)
-        VersionFile.new(carthage_dependency.version_filename)
-      else
-        nil
+    params = {}
+    unless platforms.nil?
+      params[:platform] = platforms.map(&:to_s).join(",")
+    end
+
+    version_file = perform_network_request do
+      $LOG.debug("Downloading version file from #{url}, params: #{params}")
+      RestClient.get(url, { params: params }) do |response, request, result|
+        if response.code == 200
+          File.write(carthage_dependency.version_filename, response.to_s)
+          VersionFile.new(carthage_dependency.version_filename)
+        else
+          nil
+        end
       end
     end
     version_file
@@ -41,10 +51,12 @@ class Networking
   # @raise AppError on upload failure
   def upload_version_file(carthage_dependency)
     url = new_version_file_url(carthage_dependency)
-    $LOG.debug("Uploading #{carthage_dependency.version_filename}")
-    RestClient.post(url, :version_file => File.new(carthage_dependency.version_filepath)) do |response, request, result|
-      unless response.code == 200
-        raise AppError.new, "Version file upload #{carthage_dependency.version_filename} failed, response:\n  #{response[0..300]}"
+    perform_network_request do
+      $LOG.debug("Uploading #{carthage_dependency.version_filename}")
+      RestClient.post(url, :version_file => File.new(carthage_dependency.version_filepath)) do |response, request, result|
+        unless response.code == 200
+          raise AppError.new, "Version file upload #{carthage_dependency.version_filename} failed, response:\n  #{response[0..300]}"
+        end
       end
     end
   end
@@ -52,16 +64,18 @@ class Networking
   #  Archives
 
   # @return Hash with CarthageArchive and checksum or nil
-  def download_framework_archive(carthage_dependency, framework_name, platform)
-    url = new_framework_url(carthage_dependency, framework_name, platform)
-    $LOG.debug("Downloading framework from #{url}")
-    archive = RestClient.get(url) do |response, request, result|
-      if response.code == 200
-        archive = CarthageArchive.new(framework_name, platform)
-        File.write(archive.archive_path, response.to_s)
-        {:archive => archive, :checksum => response.headers[ARCHIVE_CHECKSUM_HEADER_REST_CLIENT]}
-      else
-        nil
+  def download_framework_archive(carthage_dependency, framework, platform)
+    url = new_framework_url(carthage_dependency, framework.name, platform)
+    archive = perform_network_request do
+      $LOG.debug("Downloading framework from #{url}")
+      RestClient.get(url) do |response, request, result|
+        if response.code == 200
+          archive = framework.make_archive(platform)
+          File.write(archive.archive_path, response.to_s)
+          { :archive => archive, :checksum => response.headers[ARCHIVE_CHECKSUM_HEADER_REST_CLIENT] }
+        else
+          nil
+        end
       end
     end
     archive
@@ -70,12 +84,14 @@ class Networking
   # @raise AppError when upload fails
   def upload_framework_archive(zipfile_name, carthage_dependency, framework_name, platform, checksum)
     url = new_framework_url(carthage_dependency, framework_name, platform)
-    params = {:framework_file => File.new(zipfile_name)}
-    headers = {ARCHIVE_CHECKSUM_HEADER_REST_CLIENT => checksum}
-    $LOG.debug("Uploading framework to #{url}, headers: #{headers}")
-    RestClient.post(url, params, headers) do |response, request, result|
-      unless response.code == 200
-        raise AppError.new, "Framework upload #{zipfile_name} failed, response:\n  #{response[0..300]}"
+    params = { :framework_file => File.new(zipfile_name) }
+    headers = { ARCHIVE_CHECKSUM_HEADER_REST_CLIENT => checksum }
+    perform_network_request do
+      $LOG.debug("Uploading framework to #{url}, headers: #{headers}")
+      RestClient.post(url, params, headers) do |response, request, result|
+        unless response.code == 200
+          raise AppError.new, "Framework upload #{zipfile_name} failed, response:\n  #{response[0..300]}"
+        end
       end
     end
   end
@@ -83,12 +99,12 @@ class Networking
   private
 
   def new_version_url
-    new_server_url(['version'])
+    new_server_url(["version"])
   end
 
   def new_version_file_url(carthage_dependency)
     new_server_url([
-      'versions',
+      "versions",
       @config.xcodebuild_version,
       @config.swift_version,
       carthage_dependency.guessed_framework_basename,
@@ -99,7 +115,7 @@ class Networking
 
   def new_framework_url(carthage_dependency, framework_name, platform)
     new_server_url([
-      'frameworks',
+      "frameworks",
       @config.xcodebuild_version,
       @config.swift_version,
       carthage_dependency.guessed_framework_basename,
@@ -115,13 +131,37 @@ class Networking
       :scheme => @config.server_uri.scheme,
       :host => @config.server_uri.host,
       :port => @config.server_uri.port,
-      :path => '/' + sanitized_path_slices.join('/'),
+      :path => "/" + sanitized_path_slices.join("/"),
     )
     uri.to_s
   end
 
   # Mangle identifiers for URL paths.
   def sanitized(input)
-    input.gsub(/\//, '_')
+    input.gsub(/\//, "_")
+  end
+
+  def perform_network_request
+    if @is_retry_enabled
+      retries_remaining = 3
+      sleep_time_seconds = 5
+      begin
+        result = yield
+      rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError => e
+        if retries_remaining > 0
+          $LOG.warn("Network request failed - remaining retries: #{retries_remaining}, sleeping for: #{sleep_time_seconds}s, error: #{e.message}")
+          sleep(sleep_time_seconds)
+          retries_remaining -= 1
+          sleep_time_seconds *= 3
+          retry
+        else
+          raise e
+        end
+      end
+      result
+    else
+      yield
+    end
   end
 end

data/lib/server/config.ru

@@ -1,6 +1,6 @@
-require 'rubygems'
-require 'bundler'
+require "rubygems"
+require "bundler"
 Bundler.require
 
-require './server_app.rb'
+require "./server_app.rb"
 run Sinatra::Application

data/lib/server/server_app.rb

@@ -1,14 +1,21 @@
-require 'sinatra'
-require 'fileutils'
-require 'carthage_remote_cache'
-
-get '/' do
-  "Welcome to carthage_remote_cache"
+require "json"
+require "sinatra"
+require "fileutils"
+require "carthage_remote_cache"
+
+get "/" do
+  <<-eos
+    <html>
+      <p>Welcome to <strong>carthage_remote_cache</strong> (#{VERSION})</p>
+      <p>To browse cache contents visit <a href="/browser/">/browser/</a></p>
+    </html>
+  eos
 end
 
-version_path = '/version'
-versions_path = '/versions/:xcodebuild_version/:swift_version/:dependency_name/:version/:version_filename'
-frameworks_path = '/frameworks/:xcodebuild_version/:swift_version/:dependency_name/:version/:framework_name/:platform'
+version_path = "/version"
+versions_path = "/versions/:xcodebuild_version/:swift_version/:dependency_name/:version/:version_filename"
+frameworks_path = "/frameworks/:xcodebuild_version/:swift_version/:dependency_name/:version/:framework_name/:platform"
+browser_path = "/browser/*"
 
 get version_path do
   status(200)
@@ -16,13 +23,25 @@ get version_path do
 end
 
 get versions_path do
+  if params.key?(:platform)
+    begin
+      platforms = platform_to_symbols(params[:platform])
+    rescue AppError => e
+      status(400)
+      return JSON.pretty_generate({ "error" => e.message })
+    end
+  else
+    platforms = PLATFORMS
+  end
+
   dirname = params_to_framework_dir(params)
   filename = params[:version_filename]
   filepath = File.join(dirname, filename)
 
   if File.exist?(filepath)
     status(200)
-    send_file(filepath)
+    version_file = VersionFile.new(filepath, platforms)
+    JSON.pretty_generate(version_file.json)
   else
     status(404)
   end
@@ -39,7 +58,7 @@ post versions_path do
   File.delete(target_filename) if File.exist?(target_filename)
 
   $LOG.info("Writing: #{target_filename}")
-  File.open(target_filename, 'wb') do |target_file|
+  File.open(target_filename, "wb") do |target_file|
     target_file.write(source_file.read)
   end
 
@@ -75,7 +94,7 @@ post frameworks_path do
   File.delete(target_filename) if File.exist?(target_filename)
 
   $LOG.info("Writing: #{target_filename}")
-  File.open(target_filename, 'wb') do |target_file|
+  File.open(target_filename, "wb") do |target_file|
     target_file.write(source_file.read)
   end
 
@@ -91,6 +110,44 @@ post frameworks_path do
   end
 end
 
+# Full blown file browser.
+get browser_path do
+  url_path = "/" + params["splat"][0]
+  path = File.join(SERVER_CACHE_DIR, url_path)
+
+  if File.file?(path)
+    status(200)
+    send_file(path)
+  else
+    html = "<html>"
+
+    # Current directory
+    html += "<h2>#{url_path}</h2>"
+
+    # ".." link
+    if url_path != "/"
+      parent = File.dirname(url_path)
+      parent += "/" if parent != "/"
+      html += "<p><a href=\"/browser#{parent}\">..</a></p>"
+    end
+
+    # Child links
+    for name in Dir.children(path).select { |name| name != ".DS_Store" }.sort
+      child_path = File.join(path, name)
+      html += "<p>"
+      if File.file?(child_path)
+        html += "<a href=\"#{name}\">#{name}</a> #{format_file_size(File.size(child_path))}"
+      else
+        html += "<a href=\"#{name}/\">#{name}/</a>"
+      end
+      html += " <span style=\"color:#777\">#{File.ctime(child_path).to_s}</span>"
+      html += "</p>"
+    end
+
+    html
+  end
+end
+
 private
 
 def params_to_framework_dir(params)

data/lib/shell_wrapper.rb

@@ -1,4 +1,4 @@
-require 'open3'
+require "open3"
 
 class ShellWrapper
   # @return

data/lib/table.rb

@@ -32,8 +32,8 @@ class Table
   end
 
   def separator_line
-    dashes = @column_sizes.map { |size| '-' * size }
-    '+' + dashes.join('+') + '+'
+    dashes = @column_sizes.map { |size| "-" * size }
+    "+" + dashes.join("+") + "+"
   end
 
   def header_line
@@ -41,7 +41,7 @@ class Table
       column_size = @column_sizes[index] - 1
       " %-#{column_size}.#{column_size}s" % column
     end
-    '|' + columns.join('|') + '|'
+    "|" + columns.join("|") + "|"
   end
 
   def row_line(row)
@@ -53,6 +53,6 @@ class Table
         "%#{column_size}.#{column_size}s " % column
       end
     end
-    '|' + columns.join('|') + '|'
+    "|" + columns.join("|") + "|"
   end
 end

data/lib/utils.rb

@@ -15,7 +15,7 @@ end
 def quote(input)
   if input.is_a? String
     if input.empty?
-      ''
+      ""
     else
       '"' + input + '"'
     end
@@ -23,7 +23,7 @@ def quote(input)
     input
       .map { |e| quote(e) }
       .select { |e| !e.empty? }
-      .join(' ')
+      .join(" ")
   else
     raise AppError.new, "Unsupported type #{input}"
   end
@@ -32,13 +32,13 @@ end
 def platform_to_api_string(platform)
   case platform
   when :iOS
-    'iOS'
+    "iOS"
   when :macOS
-    'macOS'
+    "macOS"
   when :tvOS
-    'tvOS'
+    "tvOS"
   when :watchOS
-    'watchOS'
+    "watchOS"
   else
     raise AppError.new, "Unrecognized platform #{platform.inspect}"
   end
@@ -47,20 +47,34 @@ end
 def platform_to_carthage_dir_string(platform)
   case platform
   when :iOS
-    'iOS'
+    "iOS"
   when :macOS
-    'Mac'
+    "Mac"
   when :tvOS
-    'tvOS'
+    "tvOS"
   when :watchOS
-    'watchOS'
+    "watchOS"
   else
     raise AppError.new, "Unrecognized platform #{platform.inspect}"
   end
 end
 
+def platform_to_symbols(string)
+  platforms = string.split(",").map(&:to_sym)
+  for platform in platforms
+    if !PLATFORMS.include?(platform)
+      raise PlatformMismatchError.new(platform)
+    end
+  end
+  platforms
+end
+
 # @return string in "x.y MB" format
 def format_file_size(bytes)
-  megabytes = bytes / 1000.0 / 1000.0
-  "#{megabytes.round(1)} MB"
+  if bytes == 0
+    "0.0 MB"
+  else
+    megabytes = [0.1, bytes / 1000.0 / 1000.0].max
+    "#{megabytes.round(1)} MB"
+  end
 end

data/lib/version.rb

@@ -1 +1 @@
-VERSION = '0.0.8'
+VERSION = "0.0.11"