azure-blob 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20f2d1c40dbc979782d017b025e5369f21e8ec9c1de02e50ef030648b8fd557b
-  data.tar.gz: d818c311fb5a55db068d1b6d7e8d112d0fc44aa5f862144ec789ce3e3cd97a99
+  metadata.gz: 94edcaaaa7717925c1fb8238b9e7420949b26c98c90344cbf54930d5e0ff3b19
+  data.tar.gz: aae153da753212659590ed4a3ae029c86cf7ed66b4f924dac1a5f3a5ff3b2e05
 SHA512:
-  metadata.gz: 9f58536a8295aa300c0be73a4654e2a888a51db9c7acc0bb793297c3878c39947aef7bf25c4231ab30e2ee764bd65a3f5c5863d0a2b5b555bf9c7bd976abeb13
-  data.tar.gz: ae0f1e93eaff5c7f196fa78b6cf230f6be2a3c05cb4ba9635eb19828539cf381c8c23415453678ea19e790d860b65180260e9175ad1c825cb904047b09893047
+  metadata.gz: 102c3d5fd08761199413e96bd8e4bae8a84ac5478d7bb1202e232fbff346703f7cb3b8f344553983b607dbacec56942452ce18aa0afc184a4c994d31dc94118c
+  data.tar.gz: 646636874dd381757595f82999dd0b7b5647a740512c8e2baf86bdc2045e579cdf78354a4690c746452280efb7ff92cbf9c49c95775ccbcf6a690eb0ae05131f

data/.terraform.lock.hcl

@@ -0,0 +1,22 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/azurerm" {
+  version     = "3.113.0"
+  constraints = "~> 3.0"
+  hashes = [
+    "h1:eEUtt0lrLdpVaF6FiDq8BGQPgEcykmhj0aNIL7hTOGw=",
+    "zh:12479f5664288943400447b55e50df675c28ae82ad8d373cc2e5682f3a3411f0",
+    "zh:1b42a14e80e568429d3b55fed753ca3ef0df9dcdfa107890d7264599c020940f",
+    "zh:381be6ca617f848de3baa3985a6e1788e91a803afe04a3c5c727453528b6310d",
+    "zh:3e70e2e07b6db1c363de3e5d0ca47f27fc956473df03329c7d2e54d3ac29176b",
+    "zh:87c7633aeaa828098c6055da9e67d4acaf4b46748b6b3f0267e105e55f05de25",
+    "zh:8d0d98226901f874770dd5220d4701a12ae8bd586994615aa7dcba12b9736bec",
+    "zh:9fd913acd42a60c3a90a18ce803567ef861db8779a59aacced91f2cbd86de9d9",
+    "zh:b6f3f7ae0a055437fb36c139af9bb3135e7f4dad172157ae1eb0177dc74d703f",
+    "zh:b927027ba2bf40d34e03d742fd2b6c5299023b5ab8e6f05e50aac76a46ad1094",
+    "zh:ceb5187b9d2a439f4e48944f3ffeeeaf47a03dbe6f3325ea1775bf659ce0aa88",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+    "zh:fb9d78dfeca7489bffca9b1a1f3abee7f16dbbcba31388aea1102062c1d6dce8",
+  ]
+}

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 ## [Unreleased]
 
-## [0.1.0] - 2024-04-05
+## [0.5.0] 2024-09-09
 
-- Initial release
+- Added support for Managed Identities (Entra ID)
+
+## [0.4.2] 2024-06-06
+
+- Documentation
+- Fix an issue with integrity check on multi block upload
+
+
+## [0.4.1] 2024-05-27
+
+First working release.
+
+- Re-implemented the required parts of the azure-storage-blob API to make Active Storage work.
+- Extracted the AzureStorage adapter from Rails.

data/README.md

@@ -2,41 +2,113 @@
 
 This gem was built to replace azure-storage-blob (deprecated) in Active Storage, but was written to be Rails agnostic.
 
+## Active Storage
+
+### Migration
+To migrate from azure-storage-blob to azure-blob:
+
+1. Replace `azure-storage-blob` in your Gemfile with `azure-blob`
+2. Run `bundle install`
+3. Change the `AzureStorage` service to `AzureBlob`  in your Active Storage config (`config/storage.yml`)
+4. Restart or deploy the app.
+
+### Managed Identity (Entra ID)
+
+AzureBlob supports managed identities on :
+- Azure VM
+- App Service
+- Azure Functions (Untested but should work)
+- Azure Containers (Untested but should work)
+
+AKS support will likely require more work. Contributions are welcome.
+
+To authenticate through managed identities instead of a shared key, omit `storage_access_key` from your `storage.yml` file.
+
+It is recommended to add the identity's `principal_id` to the config.
+
+ActiveStorage config example:
+
+```
+prod:
+  service: AzureBlob
+  container: container_name
+  storage_account_name: account_name
+  principal_id: 71b34410-4c50-451d-b456-95ead1b18cce
+```
+
+## Standalone
+
+Instantiate a client with your account name, an access key and the container name:
+
+```ruby
+client = AzureBlob::Client.new(
+      account_name: @account_name,
+      access_key: @access_key,
+      container: @container,
+    )
+
+path = "some/new/file"
+
+# Upload
+client.create_block_blob(path, "Hello world!")
+
+# Download
+client.get_blob(path) #=> "Hello world!"
+
+# Delete
+client.delete_blob(path)
+```
+
+For the full list of methods: https://www.rubydoc.info/gems/azure-blob/AzureBlob/Client
+
 ## Contributing
 
-### dev environment
+### Dev environment
 
-Ensure your version of Ruby fit the minimum version in `azure-blob.gemspec`
+A dev environment is supplied through Nix with [devenv](https://devenv.sh/).
 
-and setup those Env variables:
+1. Install [devenv](https://devenv.sh/).
+2. Enter the dev environment by cd into the repo and running `devenv shell` (or `direnv allow` if you are a direnv user).
+3. Log into azure CLI with `az login`
+4. `terraform init`
+5. `terraform apply` This will generate the necessary infrastructure on azure.
+6. Generate devenv.local.nix with your private key and container information: `generate-env-file`
+7. If you are using direnv, the environment will reload automatically. If not, exit the shell and reopen it by hitting <C-d> and running `devenv shell` again.
 
-- `AZURE_ACCOUNT_NAME`
-- `AZURE_ACCESS_KEY`
-- `AZURE_PRIVATE_CONTAINER`
-- `AZURE_PUBLIC_CONTAINER`
+#### Entra ID
 
+To test with Entra ID, the `AZURE_ACCESS_KEY` environment variable must be unset and the code must be ran or proxied through a VPS with the proper roles.
 
-A dev environment setup is also supplied through Nix with [devenv](https://devenv.sh/).
+For cost saving, the terraform variable `create_vm` is false by default.
+To create the VPS, Create a var file `var.tfvars` containing:
 
-To use the Nix environment:
-1- install [devenv](https://devenv.sh/)
-2- Copy `devenv.local.nix.example` to `devenv.local.nix`
-3- Insert your azure credentials into `devenv.local.nix`
-4- Start the shell with `devenv shell` or with [direnv](https://direnv.net/).
+```
+create_vm = true
+```
+and re-apply terraform: `terraform apply -var-file=var.tfvars`.
 
-### Tests
+This will create the VPS and required managed identities.
 
-`bin/rake test`.
+`bin/rake test_azure_vm` and `bin/rake test_app_service` will establish a VPN connection to the VM or App service container and run the test suite. You might be prompted for a sudo password when the VPN starts (sshuttle).
 
-# Active Storage
+After you are done, run terraform again without the var file (`terraform apply`) to destroy the VPS and App service application.
 
-## Migration
-To migrate from azure-storage-blob to azure-blob:
+#### Cleanup
+
+Some tests copied over from Rails don't clean after themselves. A rake task is provided to empty your containers and keep cost low: `bin/rake flush_test_container`
+
+#### Run without devenv/nix
 
-1- Replace `azure-storage-blob` in your Gemfile with `azure-blob`
-2- Run `bundle install`
-3- change the `AzureStorage` service to `AzureBlob`  in your Active Storage config (`config/storage.yml`)
-4- Restart or deploy the app.
+If you prefer not using devenv/nix:
+
+Ensure your version of Ruby fit the minimum version in `azure-blob.gemspec`
+
+and setup those Env variables:
+
+- `AZURE_ACCOUNT_NAME`
+- `AZURE_ACCESS_KEY`
+- `AZURE_PRIVATE_CONTAINER`
+- `AZURE_PUBLIC_CONTAINER`
 
 ## License
 

data/Rakefile

@@ -2,7 +2,57 @@
 
 require "bundler/gem_tasks"
 require "minitest/test_task"
+require "azure_blob"
+require_relative "test/support/app_service_vpn"
+require_relative "test/support/azure_vm_vpn"
 
-Minitest::TestTask.create
+Minitest::TestTask.create(:test_rails) do
+  self.test_globs = [ "test/rails/**/test_*.rb",
+                     "test/rails/**/*_test.rb", ]
+end
+
+Minitest::TestTask.create(:test_client) do
+  self.test_globs = [ "test/client/**/test_*.rb",
+                     "test/client/**/*_test.rb", ]
+end
 
 task default: %i[test]
+
+task :test do
+  Rake::Task["test_client"].execute
+  Rake::Task["test_rails"].execute
+end
+
+task :test_app_service do |t|
+  vpn = AppServiceVpn.new
+  ENV["IDENTITY_ENDPOINT"] = vpn.endpoint
+  ENV["IDENTITY_HEADER"] = vpn.header
+  Rake::Task["test_entra_id"].execute
+ensure
+  vpn.kill
+end
+
+task :test_azure_vm do |t|
+  vpn = AzureVmVpn.new
+  Rake::Task["test_entra_id"].execute
+ensure
+  vpn.kill
+end
+
+task :test_entra_id do |t|
+  ENV["AZURE_ACCESS_KEY"] = nil
+  Rake::Task["test"].execute
+end
+
+task :flush_test_container do |t|
+  AzureBlob::Client.new(
+    account_name: ENV["AZURE_ACCOUNT_NAME"],
+    access_key: ENV["AZURE_ACCESS_KEY"],
+    container: ENV["AZURE_PRIVATE_CONTAINER"],
+  ).delete_prefix ""
+  AzureBlob::Client.new(
+    account_name: ENV["AZURE_ACCOUNT_NAME"],
+    access_key: ENV["AZURE_ACCESS_KEY"],
+    container: ENV["AZURE_PUBLIC_CONTAINER"],
+  ).delete_prefix ""
+end

data/azure-blob.gemspec

@@ -9,13 +9,14 @@ Gem::Specification.new do |spec|
   spec.email = [ "joe@dupuis.io" ]
 
   spec.summary = "Azure blob client"
-  spec.homepage = "https://github.com/JoeDupuis/azure-blob"
+  spec.homepage = "https://github.com/testdouble/azure-blob"
   spec.license = "MIT"
   spec.required_ruby_version = ">= 3.1"
 
+  spec.metadata["rubygems_mfa_required"] = "true"
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = spec.homepage
-  spec.metadata["changelog_uri"] = "https://github.com/JoeDupuis/azure-blob/blob/main/CHANGELOG.md"
+  spec.metadata["changelog_uri"] = "https://github.com/testdouble/azure-blob/blob/main/CHANGELOG.md"
 
   spec.add_dependency "rexml"
 

data/devenv.lock

@@ -108,16 +108,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1715542476,
+        "lastModified": 1725001927,
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "44072e24566c5bcc0b7aa9178a0104f4cfffab19",
-        "treeHash": "3f9021e4c33de6fe59b88ac8c3019fc49136dc2a",
+        "rev": "6e99f2a27d600612004fbd2c3282d614bfee6421",
+        "treeHash": "1e85443cc9f0ba302df2cf61cacb8014943e2d19",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -129,11 +129,11 @@
         "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1713939467,
+        "lastModified": 1724737223,
         "owner": "bobvanderlinden",
         "repo": "nixpkgs-ruby",
-        "rev": "c1ba161adf31119cfdbb24489766a7bcd4dbe881",
-        "treeHash": "0d32620317b29f94d6718684f030dd2fc2f30cb2",
+        "rev": "175b5867babcbc471b94be9fd5576f2973bbdb6d",
+        "treeHash": "2fe3404ac0eeb7bcb7ac7b5f5f8b9b6a7e460147",
         "type": "github"
       },
       "original": {
@@ -160,16 +160,16 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1715542476,
+        "lastModified": 1725001927,
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "44072e24566c5bcc0b7aa9178a0104f4cfffab19",
-        "treeHash": "3f9021e4c33de6fe59b88ac8c3019fc49136dc2a",
+        "rev": "6e99f2a27d600612004fbd2c3282d614bfee6421",
+        "treeHash": "1e85443cc9f0ba302df2cf61cacb8014943e2d19",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.11",
+        "ref": "nixos-24.05",
         "repo": "nixpkgs",
         "type": "github"
       }

data/devenv.nix

@@ -1,12 +1,36 @@
-{ pkgs, ... }:
+{ pkgs, config, ... }:
 
 {
+  env = {
+    LD_LIBRARY_PATH = "${config.devenv.profile}/lib";
+  };
+
   packages = with pkgs; [
     git
     libyaml
+    terraform
+    azure-cli
+    glib
+    vips
+    sshuttle
+    sshpass
+    rsync
   ];
 
-
   languages.ruby.enable = true;
-  languages.ruby.version = "3.1.5";
+  languages.ruby.version = "3.1.6";
+
+  scripts.generate-env-file.exec = ''
+    terraform output -raw devenv_local_nix > devenv.local.nix
+  '';
+
+  scripts.proxy-vps.exec = ''
+    exec sshuttle -e "ssh -o CheckHostIP=no -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" -r "$(terraform output --raw vm_username)@$(terraform output --raw vm_ip)" 0/0
+  '';
+
+  scripts.start-app-service-ssh.exec = ''
+      resource_group=$(terraform output --raw "resource_group")
+      app_name=$(terraform output --raw "app_service_app_name")
+      exec az webapp create-remote-connection --resource-group $resource_group --name $app_name
+  '';
 }

data/devenv.yaml

@@ -1,6 +1,6 @@
 allowUnfree: true
 inputs:
   nixpkgs:
-    url: github:NixOS/nixpkgs/nixos-23.11
+    url: github:NixOS/nixpkgs/nixos-24.05
   nixpkgs-ruby:
     url: github:bobvanderlinden/nixpkgs-ruby

data/input.tf

@@ -0,0 +1,44 @@
+variable "location" {
+  type    = string
+  default = "westus2"
+}
+
+variable "prefix" {
+  type    = string
+  default = "azure-blob"
+}
+
+variable "storage_account_name" {
+  type    = string
+  default = "azureblobrubygemdev"
+}
+
+variable "create_vm" {
+  type    = bool
+  default = false
+}
+
+variable "vm_size" {
+  type    = string
+  default = "Standard_B2s"
+}
+
+variable "vm_username" {
+  type    = string
+  default = "azureblob"
+}
+
+variable "vm_password" {
+  type    = string
+  default = "qwe123QWE!@#"
+}
+
+variable "create_app_service" {
+  type    = bool
+  default = false
+}
+
+variable "ssh_key" {
+  type    = string
+  default = ""
+}

data/lib/active_storage/service/azure_blob_service.rb

@@ -25,17 +25,17 @@
 require "active_support/core_ext/numeric/bytes"
 require "active_storage/service"
 
-require 'azure_blob'
+require "azure_blob"
 
 module ActiveStorage
-  # = Active Storage \Azure Storage \Service
+  # = Active Storage \Azure Blob \Service
   #
   # Wraps the Microsoft Azure Storage Blob Service as an Active Storage service.
-  # See ActiveStorage::Service for the generic API documentation that applies to all services.
+  # See {ActiveStorage::Service}[https://api.rubyonrails.org/classes/ActiveStorage/Service.html] for more details.
   class Service::AzureBlobService < Service
     attr_reader :client, :container, :signer
 
-    def initialize(storage_account_name:, storage_access_key:, container:, public: false, **options)
+    def initialize(storage_account_name:, storage_access_key: nil, container:, public: false, **options)
       @container = container
       @public = public
       @client = AzureBlob::Client.new(

data/lib/azure_blob/blob.rb

@@ -1,7 +1,13 @@
 # frozen_string_literal: true
 
 module AzureBlob
+  # AzureBlob::Blob holds the metada for a given Blob.
   class Blob
+    # You should not instanciate this object directly,
+    # but obtain one when calling relevant methods of AzureBlob::Client.
+    #
+    # Expects a Net::HTTPResponse object from a
+    # HEAD or GET request to a blob uri.
     def initialize(response)
       @response = response
     end
@@ -26,6 +32,7 @@ module AzureBlob
       response.code == "200"
     end
 
+    # Returns the custom Azure metada tagged on the blob.
     def metadata
       @metadata || response
         .to_hash

data/lib/azure_blob/blob_list.rb

@@ -3,10 +3,28 @@
 require "rexml"
 
 module AzureBlob
+  # Enumerator class to lazily iterate over a list of Blob keys.
   class BlobList
     include REXML
     include Enumerable
 
+    # You should not instanciate this object directly,
+    # but obtain one when calling relevant methods of AzureBlob::Client.
+    #
+    # Expects a callable object that takes an Azure API page marker as an
+    # argument and returns the raw body response of a call to the list blob endpoint.
+    #
+    # Example:
+    #
+    #   fetcher = ->(marker) do
+    #     uri.query = URI.encode_www_form(
+    #       marker: marker,
+    #       ...
+    #     )
+    #     response = Http.new(uri, signer:).get
+    #   end
+    #   AzureBlob::BlobList.new(fetcher)
+    #
     def initialize(fetcher)
       @fetcher = fetcher
     end

data/lib/azure_blob/block_list.rb

@@ -3,7 +3,9 @@
 require "rexml"
 
 module AzureBlob
-  class BlockList
+  class BlockList # :nodoc:
+    # Internal
+    # BlockList builds the XML list of blocks to commit to a blob
     include REXML
     def initialize(blocks)
       @blocks = blocks

data/lib/azure_blob/canonicalized_headers.rb

@@ -1,5 +1,5 @@
 module AzureBlob
-  class CanonicalizedHeaders
+  class CanonicalizedHeaders # :nodoc:
     STANDARD_HEADERS = [
       :"x-ms-version",
     ]

data/lib/azure_blob/canonicalized_resource.rb

@@ -1,7 +1,7 @@
 require "cgi"
 
 module AzureBlob
-  class CanonicalizedResource
+  class CanonicalizedResource # :nodoc:
     def initialize(uri, account_name, service_name: nil, url_safe: true)
       # This next line is needed because CanonicalizedResource
       # need to be escaped for auhthorization headers, but not SAS tokens