azure-blob 0.4.1 → 0.5.0

data/lib/azure_blob/client.rb

@@ -1,21 +1,48 @@
 # frozen_string_literal: true
 
-require_relative "signer"
 require_relative "block_list"
 require_relative "blob_list"
 require_relative "blob"
 require_relative "http"
+require_relative "shared_key_signer"
+require_relative "entra_id_signer"
 require "time"
 require "base64"
 
 module AzureBlob
+  # AzureBlob Client class. You interact with the Azure Blob api
+  # through an instance of this class.
   class Client
-    def initialize(account_name:, access_key:, container:)
+    def initialize(account_name:, access_key:, container:, **options)
       @account_name = account_name
       @container = container
-      @signer = Signer.new(account_name:, access_key:)
+
+      @signer = !access_key.nil? && !access_key.empty?  ?
+        AzureBlob::SharedKeySigner.new(account_name:, access_key:) :
+        AzureBlob::EntraIdSigner.new(account_name:, **options.slice(:principal_id))
     end
 
+    # Create a blob of type block. Will automatically split the the blob in multiple block and send the blob in pieces (blocks) if the blob is too big.
+    #
+    # When the blob is small enough this method will send the blob through {Put Blob}[https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob]
+    #
+    # If the blob is too big, the blob is split in blocks sent through a series of {Put Block}[https://learn.microsoft.com/en-us/rest/api/storageservices/put-block] requests
+    # followed by a {Put Block List}[https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list] to commit the block list.
+    #
+    # Takes a key (path), the content (String or IO object), and options.
+    #
+    # Options:
+    #
+    # [+:content_type+]
+    #   Will be saved on the blob in Azure.
+    # [+:content_disposition+]
+    #   Will be saved on the blob in Azure.
+    # [+:content_md5+]
+    #   Will ensure integrity of the upload. The checksum must be a base64 digest. Can be produced with +OpenSSL::Digest::MD5.base64digest+.
+    #   The checksum is only checked on a single upload! To verify checksum when uploading multiple blocks, call directly put_blob_block with
+    #   a checksum for each block, then commit the blocks with commit_blob_blocks.
+    # [+:block_size+]
+    #   Block size in bytes, can be used to force the method to split the upload in smaller chunk. Defaults to +AzureBlob::DEFAULT_BLOCK_SIZE+ and cannot be bigger than +AzureBlob::MAX_UPLOAD_SIZE+
     def create_block_blob(key, content, options = {})
       if content.size > (options[:block_size] || DEFAULT_BLOCK_SIZE)
         put_blob_multiple(key, content, **options)
@@ -24,6 +51,18 @@ module AzureBlob
       end
     end
 
+    # Returns the full or partial content of the blob
+    #
+    # Calls to the {Get Blob}[https://learn.microsoft.com/en-us/rest/api/storageservices/get-blob] endpoint.
+    #
+    # Takes a key (path) and options.
+    #
+    # Options:
+    #
+    # [+:start+]
+    #   Starting point in bytes
+    # [+:end+]
+    #   Ending point in bytes
     def get_blob(key, options = {})
       uri = generate_uri("#{container}/#{key}")
 
@@ -34,6 +73,15 @@ module AzureBlob
       Http.new(uri, headers, signer:).get
     end
 
+    # Delete a blob
+    #
+    # Calls to {Delete Blob}[https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob]
+    #
+    # Takes a key (path) and options.
+    #
+    # Options:
+    # [+:delete_snapshots+]
+    #   Sets the value of the x-ms-delete-snapshots header. Default to +include+
     def delete_blob(key, options = {})
       uri = generate_uri("#{container}/#{key}")
 
@@ -44,11 +92,28 @@ module AzureBlob
       Http.new(uri, headers, signer:).delete
     end
 
+    # Delete all blobs prefixed by the given prefix.
+    #
+    # Calls to {List blobs}[https://learn.microsoft.com/en-us/rest/api/storageservices/list-blobs]
+    # followed to a series of calls to {Delete Blob}[https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob]
+    #
+    # Takes a prefix and options
+    #
+    # Look delete_blob for the list of options.
     def delete_prefix(prefix, options = {})
       results = list_blobs(prefix:)
       results.each { |key| delete_blob(key) }
     end
 
+    # Returns a BlobList containing a list of keys (paths)
+    #
+    # Calls to {List blobs}[https://learn.microsoft.com/en-us/rest/api/storageservices/list-blobs]
+    #
+    # Options:
+    # [+:prefix+]
+    #   Prefix of the blobs to be listed. Defaults to listing everything in the container.
+    # [:+max_results+]
+    #   Maximum number of results to return per page.
     def list_blobs(options = {})
       uri = generate_uri(container)
       query = {
@@ -69,6 +134,11 @@ module AzureBlob
       BlobList.new(fetcher)
     end
 
+    # Returns a Blob object without the content.
+    #
+    # Calls to {Get Blob Properties}[https://learn.microsoft.com/en-us/rest/api/storageservices/get-blob-properties]
+    #
+    # This can be used to see if the blob exist or obtain metada such as content type, disposition, checksum or Azure custom metadata.
     def get_blob_properties(key, options = {})
       uri = generate_uri("#{container}/#{key}")
 
@@ -77,16 +147,37 @@ module AzureBlob
       Blob.new(response)
     end
 
+    # Return a URI object to a resource in the container. Takes a path.
+    #
+    # Example: +generate_uri("#{container}/#{key}")+
     def generate_uri(path)
       URI.parse(URI::DEFAULT_PARSER.escape(File.join(host, path)))
     end
 
+    # Returns an SAS signed URI
+    #
+    # Takes a
+    # - key (path)
+    # - A permission string (+"r"+, +"rw"+)
+    # - expiry as a UTC iso8601 time string
+    # - options
     def signed_uri(key, permissions:, expiry:, **options)
       uri = generate_uri("#{container}/#{key}")
       uri.query = signer.sas_token(uri, permissions:, expiry:, **options)
       uri
     end
 
+    # Creates a Blob of type append.
+    #
+    # Calls to {Put Blob}[https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob]
+    #
+    # You are expected to append blocks to the blob with append_blob_block after creating the blob.
+    # Options:
+    #
+    # [+:content_type+]
+    #   Will be saved on the blob in Azure.
+    # [+:content_disposition+]
+    #   Will be saved on the blob in Azure.
     def create_append_blob(key, options = {})
       uri = generate_uri("#{container}/#{key}")
 
@@ -101,6 +192,15 @@ module AzureBlob
       Http.new(uri, headers, metadata: options[:metadata], signer:).put(nil)
     end
 
+    # Append a block to an Append Blob
+    #
+    # Calls to {Append Block}[https://learn.microsoft.com/en-us/rest/api/storageservices/append-block]
+    #
+    # Options:
+    #
+    # [+:content_md5+]
+    #   Will ensure integrity of the upload. The checksum must be a base64 digest. Can be produced with +OpenSSL::Digest::MD5.base64digest+.
+    #   The checksum must be the checksum of the block not the blob.
     def append_blob_block(key, content, options = {})
       uri = generate_uri("#{container}/#{key}")
       uri.query = URI.encode_www_form(comp: "appendblock")
@@ -114,6 +214,16 @@ module AzureBlob
       Http.new(uri, headers, signer:).put(content)
     end
 
+    # Uploads a block to a blob.
+    #
+    # Calls to {Put Block}[https://learn.microsoft.com/en-us/rest/api/storageservices/put-block]
+    #
+    # Returns the id of the block. Required to commit the list of blocks to a blob.
+    #
+    # Options:
+    #
+    # [+:content_md5+]
+    #   Must be the checksum for the block not the blob. The checksum must be a base64 digest. Can be produced with +OpenSSL::Digest::MD5.base64digest+.
     def put_blob_block(key, index, content, options = {})
       block_id = generate_block_id(index)
       uri = generate_uri("#{container}/#{key}")
@@ -130,6 +240,17 @@ module AzureBlob
       block_id
     end
 
+    # Commits the list of blocks to a blob.
+    #
+    # Calls to {Put Block List}[https://learn.microsoft.com/en-us/rest/api/storageservices/put-block-list]
+    #
+    # Takes a key (path) and an array of block ids
+    #
+    # Options:
+    #
+    # [+:content_md5+]
+    #   This is the checksum for the whole blob. The checksum is saved on the blob, but it is not validated!
+    #   Add a checksum for each block if you want Azure to validate integrity.
     def commit_blob_blocks(key, block_ids, options = {})
       block_list = BlockList.new(block_ids)
       content = block_list.to_s
@@ -139,7 +260,7 @@ module AzureBlob
       headers = {
         "Content-Length": content.size,
         "Content-Type": options[:content_type],
-        "Content-MD5": options[:content_md5],
+        "x-ms-blob-content-md5": options[:content_md5],
         "x-ms-blob-content-disposition": options[:content_disposition],
       }
 
@@ -171,7 +292,7 @@ module AzureBlob
         "x-ms-blob-type": "BlockBlob",
         "Content-Length": content.size,
         "Content-Type": options[:content_type],
-        "Content-MD5": options[:content_md5],
+        "x-ms-blob-content-md5": options[:content_md5],
         "x-ms-blob-content-disposition": options[:content_disposition],
       }
 

data/lib/azure_blob/entra_id_signer.rb

@@ -0,0 +1,115 @@
+require "base64"
+require "openssl"
+require "net/http"
+require "rexml/document"
+
+require_relative "canonicalized_resource"
+require_relative "identity_token"
+
+require_relative "user_delegation_key"
+
+module AzureBlob
+  class EntraIdSigner # :nodoc:
+    attr_reader :token
+    attr_reader :account_name
+
+    def initialize(account_name:, principal_id: nil)
+      @token = AzureBlob::IdentityToken.new(principal_id:)
+      @account_name = account_name
+    end
+
+    def authorization_header(uri:, verb:, headers: {})
+      "Bearer #{token}"
+    end
+
+    def sas_token(uri, options = {})
+      to_sign = [
+        options[:permissions],
+        options[:start],
+        options[:expiry],
+        CanonicalizedResource.new(uri, account_name, url_safe: false, service_name: :blob),
+        delegation_key.signed_oid,
+        delegation_key.signed_tid,
+        delegation_key.signed_start,
+        delegation_key.signed_expiry,
+        delegation_key.signed_service,
+        delegation_key.signed_version,
+        nil,
+        nil,
+        nil,
+        options[:ip],
+        options[:protocol],
+        SAS::Version,
+        SAS::Resources::Blob,
+        nil,
+        nil,
+        nil,
+        options[:content_disposition],
+        nil,
+        nil,
+        options[:content_type],
+      ].join("\n")
+
+      query = {
+        SAS::Fields::Permissions => options[:permissions],
+        SAS::Fields::Start => options[:start],
+        SAS::Fields::Expiry => options[:expiry],
+
+        SAS::Fields::SignedObjectId => delegation_key.signed_oid,
+        SAS::Fields::SignedTenantId => delegation_key.signed_tid,
+        SAS::Fields::SignedKeyStartTime => delegation_key.signed_start,
+        SAS::Fields::SignedKeyExpiryTime => delegation_key.signed_expiry,
+        SAS::Fields::SignedKeyService => delegation_key.signed_service,
+        SAS::Fields::Signedkeyversion => delegation_key.signed_version,
+
+
+        SAS::Fields::SignedIp => options[:ip],
+        SAS::Fields::SignedProtocol => options[:protocol],
+        SAS::Fields::Version => SAS::Version,
+        SAS::Fields::Resource => SAS::Resources::Blob,
+
+        SAS::Fields::Disposition => options[:content_disposition],
+        SAS::Fields::Type => options[:content_type],
+        SAS::Fields::Signature => sign(to_sign, key: delegation_key.to_s),
+
+      }.reject { |_, value| value.nil? }
+
+      URI.encode_www_form(**query)
+    end
+
+    private
+
+    def delegation_key
+      @delegation_key ||= UserDelegationKey.new(account_name:, signer: self)
+    end
+
+    def sign(body, key:)
+      Base64.strict_encode64(OpenSSL::HMAC.digest("sha256", key, body))
+    end
+
+    module SAS # :nodoc:
+      Version = "2024-05-04"
+      module Fields # :nodoc:
+        Permissions = :sp
+        Version = :sv
+        Start = :st
+        Expiry = :se
+        Resource = :sr
+        Signature = :sig
+        Disposition = :rscd
+        Type = :rsct
+        SignedObjectId = :skoid
+        SignedTenantId = :sktid
+        SignedKeyStartTime = :skt
+        SignedKeyExpiryTime = :ske
+        SignedKeyService = :sks
+        Signedkeyversion = :skv
+        SignedIp = :sip
+        SignedProtocol = :spr
+      end
+      module Resources # :nodoc:
+        Blob = :b
+      end
+    end
+  end
+end

data/lib/azure_blob/http.rb

@@ -6,8 +6,15 @@ require "net/http"
 require "rexml"
 
 module AzureBlob
-  class Http
-    class Error < AzureBlob::Error; end
+  class Http # :nodoc:
+    class Error < AzureBlob::Error
+      attr_reader :body, :status
+      def initialize(body: nil, status: nil)
+        @body = body
+        @status = status
+        super(body)
+      end
+    end
     class FileNotFoundError < Error; end
     class ForbidenError < Error; end
     class IntegrityError < Error; end
@@ -44,6 +51,15 @@ module AzureBlob
       true
     end
 
+    def post(content)
+      sign_request("POST") if signer
+      @response = http.start do |http|
+        http.post(uri, content, headers)
+      end
+      raise_error  unless success?
+      response.body
+    end
+
     def head
       sign_request("HEAD") if signer
       @response = http.start do |http|
@@ -91,7 +107,7 @@ module AzureBlob
     end
 
     def raise_error
-      raise error_from_response.new(@response.body)
+      raise error_from_response.new(body: @response.body, status: @response.code&.to_i)
     end
 
     def status

data/lib/azure_blob/identity_token.rb

@@ -0,0 +1,65 @@
+require "json"
+
+module AzureBlob
+  class IdentityToken
+    RESOURCE_URI = "https://storage.azure.com/"
+    EXPIRATION_BUFFER = 600 # 10 minutes
+
+    IDENTITY_ENDPOINT = ENV["IDENTITY_ENDPOINT"] || "http://169.254.169.254/metadata/identity/oauth2/token"
+    API_VERSION = ENV["IDENTITY_ENDPOINT"] ? "2019-08-01" : "2018-02-01"
+
+    def initialize(principal_id: nil)
+      @identity_uri = URI.parse(IDENTITY_ENDPOINT)
+      params = {
+        'api-version': API_VERSION,
+        resource: RESOURCE_URI,
+      }
+      params[:principal_id] = principal_id if principal_id
+      @identity_uri.query = URI.encode_www_form(params)
+    end
+
+    def to_s
+      refresh if expired?
+      token
+    end
+
+    private
+
+    def expired?
+      token.nil? || Time.now >= (expiration - EXPIRATION_BUFFER)
+    end
+
+    def refresh
+      headers =  { "Metadata" => "true" }
+      headers["X-IDENTITY-HEADER"] = ENV["IDENTITY_HEADER"] if ENV["IDENTITY_HEADER"]
+
+      attempt = 0
+      begin
+        attempt += 1
+        response = JSON.parse(AzureBlob::Http.new(identity_uri, headers).get)
+      rescue AzureBlob::Http::Error => error
+        if should_retry?(error, attempt)
+          attempt = 1 if error.status == 410
+          delay = exponential_backoff(error, attempt)
+          Kernel.sleep(delay)
+          retry
+        end
+        raise
+      end
+      @token = response["access_token"]
+      @expiration = Time.at(response["expires_on"].to_i)
+    end
+
+    def should_retry?(error, attempt)
+      is_500 = error.status/500 == 1
+      (is_500 || [ 404, 408, 410, 429 ].include?(error.status)) && attempt < 5
+    end
+
+    def exponential_backoff(error, attempt)
+      EXPONENTIAL_BACKOFF[attempt -1] || raise(AzureBlob::Error.new("Exponential backoff out of bounds!"))
+    end
+    EXPONENTIAL_BACKOFF = [ 2, 6, 14, 30 ]
+
+    attr_reader :identity_uri, :expiration, :token
+  end
+end

data/lib/azure_blob/metadata.rb

@@ -1,5 +1,5 @@
 module AzureBlob
-  class Metadata
+  class Metadata # :nodoc:
     def initialize(metadata = nil)
       @metadata = metadata || {}
       @headers = @metadata.map do |key, value|

data/lib/azure_blob/{signer.rb → shared_key_signer.rb}

@@ -6,7 +6,7 @@ require_relative "canonicalized_headers"
 require_relative "canonicalized_resource"
 
 module AzureBlob
-  class Signer
+  class SharedKeySigner # :nodoc:
     def initialize(account_name:, access_key:)
       @account_name = account_name
       @access_key = Base64.decode64(access_key)
@@ -71,21 +71,21 @@ module AzureBlob
       URI.encode_www_form(**query)
     end
 
+    private
+
     def sign(body)
       Base64.strict_encode64(OpenSSL::HMAC.digest("sha256", access_key, body))
     end
 
-    private
-
     def sanitize_headers(headers)
       headers = headers.dup
       headers[:"Content-Length"] = nil if headers[:"Content-Length"].to_i == 0
       headers
     end
 
-    module SAS
+    module SAS # :nodoc:
       Version = "2024-05-04"
-      module Fields
+      module Fields # :nodoc:
         Permissions = :sp
         Version = :sv
         Expiry = :se
@@ -94,7 +94,7 @@ module AzureBlob
         Disposition = :rscd
         Type = :rsct
       end
-      module Resources
+      module Resources # :nodoc:
         Blob = :b
       end
     end

data/lib/azure_blob/user_delegation_key.rb

@@ -0,0 +1,67 @@
+require_relative "http"
+
+module AzureBlob
+  class UserDelegationKey # :nodoc:
+    EXPIRATION = 25200 # 7 hours
+    EXPIRATION_BUFFER = 3600 # 1 hours
+    def initialize(account_name:, signer:)
+      @uri = URI.parse(
+        "https://#{account_name}.blob.core.windows.net/?restype=service&comp=userdelegationkey"
+      )
+
+      @signer = signer
+
+      refresh
+    end
+
+    def to_s
+      user_delegation_key
+    end
+
+    def refresh
+      return unless expired?
+      now = Time.now.utc
+
+
+      start = now.iso8601
+      @expiration = (now + EXPIRATION)
+      expiry = @expiration.iso8601
+
+      content = <<-XML.gsub!(/[[:space:]]+/, " ").strip!
+        <?xml version="1.0" encoding="utf-8"?>
+        <KeyInfo>
+            <Start>#{start}</Start>
+            <Expiry>#{expiry}</Expiry>
+        </KeyInfo>
+      XML
+
+      response  = Http.new(uri, signer:).post(content)
+
+      doc = REXML::Document.new(response)
+
+      @signed_oid  = doc.get_elements("/UserDelegationKey/SignedOid").first.get_text.to_s
+      @signed_tid = doc.get_elements("/UserDelegationKey/SignedTid").first.get_text.to_s
+      @signed_start = doc.get_elements("/UserDelegationKey/SignedStart").first.get_text.to_s
+      @signed_expiry = doc.get_elements("/UserDelegationKey/SignedExpiry").first.get_text.to_s
+      @signed_service = doc.get_elements("/UserDelegationKey/SignedService").first.get_text.to_s
+      @signed_version = doc.get_elements("/UserDelegationKey/SignedVersion").first.get_text.to_s
+      @user_delegation_key = Base64.decode64(doc.get_elements("/UserDelegationKey/Value").first.get_text.to_s)
+    end
+
+    attr_reader :signed_oid,
+      :signed_tid,
+      :signed_start,
+      :signed_expiry,
+      :signed_service,
+      :signed_version,
+      :user_delegation_key
+
+    private
+
+    def expired?
+      expiration.nil? || Time.now >= (expiration - EXPIRATION_BUFFER)
+    end
+
+    attr_reader :uri, :user_delegation_key, :signer, :expiration
+  end
+end

data/lib/azure_blob/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AzureBlob
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end