infrawrench-bulk_instances 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bccbd61e8ed4801a2ca745978807a8ec752b6b04cc8b5042cf92b01a7406ffeb
+  data.tar.gz: 529c9965e71e9a194204cfdbba3b78be46ba6779a225d4e580ab83621df2f3db
+SHA512:
+  metadata.gz: a26d838f6e2da662c1dd77bea3f021e509a679fb45b14dd86496d238f2dc71417f85a8c96651d90d2402e0be6244b8919b71e9da1171af308f883ea126f6c896
+  data.tar.gz: 4045a6ef4cb8451a56de51a4b6215eb0ed2c7f9418b9cfcc7f665fa1e4c221dc1a91b2b80020e79392166e3a8b75ddfc6a86f2723919bbb29e27485e7c271a8a

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Infrawrench LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,63 @@
+# infrawrench-bulk_instances
+
+AWS EC2 instance attributes and prices, with their full history, from the bundle
+Infrawrench publishes at `instances-bundles.infrawrench.com`.
+
+```sh
+gem install infrawrench-bulk_instances
+```
+
+```ruby
+require "infrawrench/bulk_instances"
+
+data = Infrawrench::BulkInstances.download_aws_ec2_pricing_data(token: ENV["INFRAWRENCH_API_KEY"])
+
+data.keys # ["m5.large", "t2.micro", ...]
+
+m5 = data["m5.large"]
+m5.attributes["vcpu"] # "2"
+m5.attributes_in("US East (N. Virginia)") # with that region's overrides
+m5.pricing.on_demand.find do |p|
+  p.region_code == "us-east-1" && p.operating_system == "Linux" && p.tenancy == "Shared" &&
+    p.pre_installed_sw == "NA" && p.capacitystatus == "Used"
+end&.hourly # 0.096
+
+m5.attributes_history # every attribute change, oldest first
+m5.pricing_history.reserved # every reserved price change, oldest first
+```
+
+Any Infrawrench bearer credential works as `token:` (an `iwk_` API key, an agent
+credential or a WorkOS access token). It defaults to `INFRAWRENCH_API_KEY`.
+
+Requires Ruby 3.2 or later, and liblzma (part of macOS and of practically every
+Linux distribution; `xz-utils`/`xz` otherwise) to decompress downloads.
+
+## Caching
+
+The bundle is downloaded xz-compressed (about 20 MB) and kept decompressed (about
+210 MB) in `~/.infrawrench/aws_pricing_data` (`cache_path:`), so a cached load skips
+decompression. Each load asks the server which bundle is latest and downloads it
+only if it is not cached, deleting the older one. If the server cannot be
+reached, the newest cached bundle is used instead. Pass `cache_pricing_data: false`
+to always download and never touch the disk. The cache is shared with the other
+Infrawrench SDKs.
+
+Loading decompresses and parses the whole bundle. Instances are decoded lazily:
+`get` only wraps the raw rows, and `attributes`, `attributes_history`, `pricing`
+and `pricing_history` each decode on first access and keep the result. The
+returned objects are shared between calls, so they are frozen. Memoization is not
+synchronized: threads racing on a first access may decode the same part twice,
+which is harmless.
+
+Failures raise `Infrawrench::BulkInstances::AWSPricingDataError`, whose `status`
+is the HTTP status if there was a response.
+
+## Prices and regions
+
+- `region` is the AWS location name (`US East (N. Virginia)`), `region_code` its
+  code (`us-east-1`) where the price list gives one, else `nil`.
+- `pricing` holds only what is still offered. In `pricing_history`, a `nil`
+  `hourly` means the price stopped being offered at `at`.
+- Times are UTC `Time`s.
+- Instances AWS no longer lists are still in `keys`, with `deleted_at` set and
+  empty `pricing`.

data/lib/infrawrench/bulk_instances/aws_instance.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Infrawrench
+  module BulkInstances
+    # One instance type. Each reader decodes on first access and caches the result.
+    class AWSInstance
+      # The instance type, e.g. +m5.large+.
+      # @return [String]
+      attr_reader :name
+
+      def initialize(name, raw, data)
+        @name = name
+        @raw = raw
+        @data = data
+      end
+
+      # The publication date of the first price list that listed this instance.
+      # @return [Time]
+      def created_at
+        @created_at ||= Util.time(@raw[5])
+      end
+
+      # The publication date of the first price list that stopped listing it, or +nil+ if AWS still lists it.
+      # @return [Time, nil]
+      def deleted_at
+        return if @raw[6].nil?
+
+        @deleted_at ||= Util.time(@raw[6])
+      end
+
+      # The latest attributes shared by every region. Regional differences are in #attributes_in.
+      # @return [Hash{String => String}]
+      def attributes
+        @attributes ||= attributes_history.each_with_object({}) do |c, out|
+          next unless c.region.nil?
+
+          if c.value.nil?
+            out.delete(c.key)
+          else
+            out[c.key] = c.value
+          end
+        end.freeze
+      end
+
+      # The latest attributes in one AWS location (e.g. <tt>US East (N. Virginia)</tt>): the
+      # global ones with that region's overrides on top. A regional override keeps
+      # applying even after a newer global change to the same key.
+      # @return [Hash{String => String}]
+      def attributes_in(region)
+        out = attributes.dup
+        overrides = {}
+        attributes_history.each { |c| overrides[c.key] = c.value if c.region == region }
+        overrides.each do |k, v|
+          if v.nil?
+            out.delete(k)
+          else
+            out[k] = v
+          end
+        end
+        out.freeze
+      end
+
+      # Every attribute change, oldest first, starting with the attributes the instance was first listed with.
+      # @return [Array<AttributeChange>]
+      def attributes_history
+        @attributes_history ||= begin
+          d = @data
+          initial = @raw[0].map do |k, v|
+            AttributeChange.new(at: created_at, key: d.str(k), value: d.str(v), region: nil)
+          end
+          changes = Util.sort_by_time(@raw[1]).map do |ts, key, value, region|
+            AttributeChange.new(
+              at: Util.time(ts),
+              key: d.str(key),
+              value: value.nil? ? nil : d.str(value),
+              region: region.nil? ? nil : d.str(region),
+            )
+          end
+          (initial + changes).freeze
+        end
+      end
+
+      # The current price of everything still offered.
+      # @return [Pricing]
+      def pricing
+        @pricing ||= begin
+          history = pricing_history
+          od = {}
+          history.on_demand.each { |r| od[od_key(r)] = r }
+          rs = {}
+          history.reserved.each do |r|
+            rs[[*od_key(r), r.lease_contract_length, r.offering_class, r.purchase_option]] = r
+          end
+
+          Pricing.new(
+            on_demand: od.each_value.reject { |r| r.hourly.nil? }.map do |r|
+              OnDemandPrice.new(**r.to_h.except(:at), since: r.at)
+            end.freeze,
+            reserved: rs.each_value.reject { |r| r.hourly.nil? }.map do |r|
+              ReservedPrice.new(**r.to_h.except(:at, :upfront), upfront: r.upfront || 0.0, since: r.at)
+            end.freeze,
+          )
+        end
+      end
+
+      # Every on-demand and reserved price change, oldest first.
+      # @return [PricingHistory]
+      def pricing_history
+        @pricing_history ||= begin
+          d = @data
+          base = @raw[2].to_f
+          PricingHistory.new(
+            # [ts, region, variant, price: omitted = base price, null = no longer offered]
+            on_demand: Util.sort_by_time(@raw[3]).map do |ts, region, variant, *price|
+              OnDemandPriceChange.new(
+                at: Util.time(ts),
+                **d.location(region),
+                **d.variant(variant).to_h,
+                hourly: price.empty? ? base : price[0]&.to_f,
+              )
+            end.freeze,
+            # [ts, region, variant, term, hourly: null = no longer offered, upfront: omitted if 0]
+            reserved: Util.sort_by_time(@raw[4]).map do |ts, region, variant, term, hourly, upfront|
+              removed = hourly.nil?
+              ReservedPriceChange.new(
+                at: Util.time(ts),
+                **d.location(region),
+                **d.variant(variant).to_h,
+                **d.term(term).to_h,
+                hourly: removed ? nil : hourly.to_f,
+                upfront: removed ? nil : (upfront || 0).to_f,
+              )
+            end.freeze,
+          )
+        end
+      end
+
+      private
+
+      def od_key(r)
+        [r.region, r.operating_system, r.tenancy, r.license_model, r.pre_installed_sw, r.capacitystatus]
+      end
+    end
+  end
+end

data/lib/infrawrench/bulk_instances/aws_pricing_data.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Infrawrench
+  module BulkInstances
+    # The AWS EC2 pricing data, keyed by instance type.
+    #
+    # Nothing is decoded until it is asked for: #get wraps the raw rows, and each
+    # reader on the returned AWSInstance decodes its part on first access and keeps
+    # the result. Everything returned is frozen and shared between calls.
+    #
+    # Memoization is not synchronized: threads racing on a first access may decode
+    # the same part twice, which is harmless.
+    class AWSPricingData
+      # +raw+ is the parsed bundle as instances-bundler writes it, where every string is an
+      # index into +compression_table+:
+      #
+      #   {
+      #     "compression_table" => [String, ...],
+      #     "snapshots" => [Integer, ...],
+      #     "regions" => [[location, [[key, value], ...]], ...],
+      #     "variants" => [[os, tenancy, license, sw, capacity], ...],
+      #     "reserved_terms" => [[lease, offering, purchase], ...],
+      #     "instances" => { "m5.large" => [attrs, attr_changes, base_price, on_demand, reserved, created_at, deleted_at] },
+      #   }
+      def initialize(raw)
+        @raw = raw
+        @instances = {}
+      end
+
+      # Every instance type in the bundle, including ones AWS no longer lists (see AWSInstance#deleted_at).
+      # @return [Array<String>]
+      def keys
+        @keys ||= @raw["instances"].keys.freeze
+      end
+
+      # @return [Boolean]
+      def has?(instance_type)
+        @raw["instances"].key?(instance_type)
+      end
+      alias key? has?
+
+      # The instance with this type (e.g. +m5.large+), or +nil+ if the bundle has never listed it.
+      # @return [AWSInstance, nil]
+      def get(instance_type)
+        inst = @instances[instance_type]
+        if inst.nil? && has?(instance_type)
+          inst = @instances[instance_type] = AWSInstance.new(instance_type, @raw["instances"][instance_type], self)
+        end
+        inst
+      end
+      alias [] get
+
+      # The publication date of every AWS price list in the bundle, oldest first.
+      # @return [Array<Time>]
+      def snapshots
+        @snapshots ||= @raw["snapshots"].map { |ts| Util.time(ts) }.freeze
+      end
+
+      # Attributes of each AWS location (+regionCode+, +locationType+), keyed by location name.
+      # @return [Hash{String => Hash{String => String}}]
+      def regions
+        @regions ||= @raw["regions"].to_h do |loc, pairs|
+          [str(loc), pairs.to_h { |k, v| [str(k), str(v)] }.freeze]
+        end.freeze
+      end
+
+      # @api private
+      def str(i)
+        s = @raw["compression_table"][i] if i.is_a?(Integer) && i >= 0
+        raise IndexError, "string index #{i.inspect} out of range" if s.nil?
+
+        s
+      end
+
+      # @api private
+      # @return [Hash] +region:+ and +region_code:+
+      def location(i)
+        region = str(i)
+        { region:, region_code: regions.dig(region, "regionCode") }
+      end
+
+      # @api private
+      # @return [Variant]
+      def variant(i)
+        @variants ||= @raw["variants"].map do |os, tenancy, license, sw, capacity|
+          Variant.new(
+            operating_system: str(os),
+            tenancy: str(tenancy),
+            license_model: str(license),
+            pre_installed_sw: str(sw),
+            capacitystatus: str(capacity),
+          )
+        end.freeze
+        @variants[i]
+      end
+
+      # @api private
+      # @return [ReservedTerm]
+      def term(i)
+        @terms ||= @raw["reserved_terms"].map do |lease, offering, purchase|
+          ReservedTerm.new(lease_contract_length: str(lease), offering_class: str(offering), purchase_option: str(purchase))
+        end.freeze
+        @terms[i]
+      end
+    end
+  end
+end

data/lib/infrawrench/bulk_instances/error.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Infrawrench
+  module BulkInstances
+    # Raised when the pricing data cannot be downloaded. +status+ is the HTTP status, if there was a response.
+    class AWSPricingDataError < StandardError
+      # @return [Integer, nil]
+      attr_reader :status
+
+      def initialize(message = nil, status: nil)
+        super(message)
+        @status = status
+      end
+    end
+  end
+end

data/lib/infrawrench/bulk_instances/types.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Infrawrench
+  module BulkInstances
+    # Every price and price change starts with its location:
+    #
+    # - +region+: the AWS location name, e.g. <tt>US East (N. Virginia)</tt>.
+    # - +region_code+: e.g. +us-east-1+. +nil+ for locations the price list never gave a code.
+
+    # The product fields that pick a price.
+    Variant = Data.define(:operating_system, :tenancy, :license_model, :pre_installed_sw, :capacitystatus)
+
+    ReservedTerm = Data.define(:lease_contract_length, :offering_class, :purchase_option)
+
+    # @!attribute [r] hourly
+    #   @return [Float] USD per hour.
+    # @!attribute [r] since
+    #   @return [Time] When this price took effect.
+    OnDemandPrice = Data.define(
+      :region, :region_code,
+      :operating_system, :tenancy, :license_model, :pre_installed_sw, :capacitystatus,
+      :hourly, :since,
+    )
+
+    # @!attribute [r] hourly
+    #   @return [Float] USD per hour.
+    # @!attribute [r] upfront
+    #   @return [Float] USD paid up front.
+    # @!attribute [r] since
+    #   @return [Time]
+    ReservedPrice = Data.define(
+      :region, :region_code,
+      :operating_system, :tenancy, :license_model, :pre_installed_sw, :capacitystatus,
+      :lease_contract_length, :offering_class, :purchase_option,
+      :hourly, :upfront, :since,
+    )
+
+    # @!attribute [r] on_demand
+    #   @return [Array<OnDemandPrice>]
+    # @!attribute [r] reserved
+    #   @return [Array<ReservedPrice>]
+    Pricing = Data.define(:on_demand, :reserved)
+
+    # @!attribute [r] at
+    #   @return [Time]
+    # @!attribute [r] hourly
+    #   @return [Float, nil] +nil+ means no longer offered from +at+ on.
+    OnDemandPriceChange = Data.define(
+      :at, :region, :region_code,
+      :operating_system, :tenancy, :license_model, :pre_installed_sw, :capacitystatus,
+      :hourly,
+    )
+
+    # @!attribute [r] at
+    #   @return [Time]
+    # @!attribute [r] hourly
+    #   @return [Float, nil] +nil+ means no longer offered from +at+ on.
+    # @!attribute [r] upfront
+    #   @return [Float, nil]
+    ReservedPriceChange = Data.define(
+      :at, :region, :region_code,
+      :operating_system, :tenancy, :license_model, :pre_installed_sw, :capacitystatus,
+      :lease_contract_length, :offering_class, :purchase_option,
+      :hourly, :upfront,
+    )
+
+    # @!attribute [r] on_demand
+    #   @return [Array<OnDemandPriceChange>] Oldest first.
+    # @!attribute [r] reserved
+    #   @return [Array<ReservedPriceChange>] Oldest first.
+    PricingHistory = Data.define(:on_demand, :reserved)
+
+    # @!attribute [r] at
+    #   @return [Time]
+    # @!attribute [r] key
+    #   @return [String]
+    # @!attribute [r] value
+    #   @return [String, nil] +nil+ means the attribute was removed.
+    # @!attribute [r] region
+    #   @return [String, nil] The AWS location this change applies to, or +nil+ for every region.
+    AttributeChange = Data.define(:at, :key, :value, :region)
+  end
+end

data/lib/infrawrench/bulk_instances/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Infrawrench
+  module BulkInstances
+    VERSION = "1.0.0"
+  end
+end

data/lib/infrawrench/bulk_instances/xz.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "fiddle"
+require "fiddle/import"
+
+module Infrawrench
+  module BulkInstances
+    # @api private
+    #
+    # The few liblzma calls needed to decompress an .xz stream, through Fiddle. liblzma
+    # ships with macOS and practically every Linux distribution; this file is only
+    # loaded when a bundle has to be downloaded, so cached loads never need it.
+    module XZ
+      extend Fiddle::Importer
+
+      LIBRARIES = %w[liblzma.so.5 liblzma.5.dylib liblzma.so liblzma.dylib liblzma.dll lzma.dll].freeze
+
+      def self.open_library
+        LIBRARIES.each do |name|
+          return Fiddle::Handle.new(name)
+        rescue Fiddle::DLError
+          next
+        end
+        raise Fiddle::DLError, "liblzma not found (tried #{LIBRARIES.join(", ")}): install xz to download bundles"
+      end
+      private_class_method :open_library
+
+      dlload open_library
+
+      # lzma_stream, unchanged since liblzma 5.0.
+      Stream = struct([
+        "const uint8_t *next_in", "size_t avail_in", "uint64_t total_in",
+        "uint8_t *next_out", "size_t avail_out", "uint64_t total_out",
+        "void *allocator", "void *internal",
+        "void *reserved_ptr1", "void *reserved_ptr2", "void *reserved_ptr3", "void *reserved_ptr4",
+        "uint64_t seek_pos", "uint64_t reserved_int2", "size_t reserved_int3", "size_t reserved_int4",
+        "int reserved_enum1", "int reserved_enum2",
+      ])
+
+      extern "int lzma_stream_decoder(void *, uint64_t, uint32_t)"
+      extern "int lzma_code(void *, int)"
+      extern "void lzma_end(void *)"
+
+      OK = 0
+      STREAM_END = 1
+      RUN = 0
+      FINISH = 3
+      CONCATENATED = 0x08
+      NO_MEMLIMIT = 2**64 - 1
+      BUFFER_SIZE = 1 << 20
+
+      ERRORS = {
+        5 => "out of memory", 7 => "not in the .xz format", 8 => "unsupported options",
+        9 => "corrupt data", 10 => "truncated data",
+      }.freeze
+
+      # Decompresses the .xz data the block writes with +<<+ and returns it as a UTF-8 String.
+      def self.decompress
+        decoder = Decoder.new
+        yield decoder
+        decoder.finish
+      ensure
+        decoder&.close
+      end
+
+      # Feeds the input through malloc'd buffers, so liblzma, which runs without the
+      # GVL, never points into Ruby memory.
+      class Decoder
+        def initialize
+          @in = Fiddle::Pointer.malloc(BUFFER_SIZE, Fiddle::RUBY_FREE)
+          @out = Fiddle::Pointer.malloc(BUFFER_SIZE, Fiddle::RUBY_FREE)
+          ptr = Fiddle::Pointer.malloc(Stream.size, Fiddle::RUBY_FREE)
+          ptr[0, Stream.size] = "\0" * Stream.size # LZMA_STREAM_INIT
+          @stream = Stream.new(ptr)
+          check(XZ.lzma_stream_decoder(@stream, NO_MEMLIMIT, CONCATENATED))
+          @result = String.new(encoding: Encoding::BINARY)
+        end
+
+        def <<(chunk)
+          (0...chunk.bytesize).step(BUFFER_SIZE) do |offset|
+            piece = chunk.byteslice(offset, BUFFER_SIZE)
+            @in[0, piece.bytesize] = piece
+            code(piece.bytesize, RUN)
+          end
+          self
+        end
+
+        def finish
+          code(0, FINISH)
+          @result.force_encoding(Encoding::UTF_8)
+        end
+
+        def close
+          XZ.lzma_end(@stream) if @stream
+          @stream = nil
+        end
+
+        private
+
+        def code(size, action)
+          @stream.next_in = @in
+          @stream.avail_in = size
+          loop do
+            @stream.next_out = @out
+            @stream.avail_out = BUFFER_SIZE
+            ret = XZ.lzma_code(@stream, action)
+            produced = BUFFER_SIZE - @stream.avail_out
+            @result << @out[0, produced] if produced.positive?
+            return if ret == STREAM_END
+            check(ret)
+            # RUN is done once the input is consumed and liblzma had output space to spare.
+            # FINISH runs until STREAM_END; liblzma reports truncated input as an error.
+            return if action == RUN && @stream.avail_in.zero? && @stream.avail_out.positive?
+          end
+        end
+
+        def check(ret)
+          return if ret == OK
+
+          raise AWSPricingDataError, "Could not decompress the bundle: #{ERRORS.fetch(ret, "liblzma error #{ret}")}."
+        end
+      end
+    end
+  end
+end

data/lib/infrawrench/bulk_instances.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "net/http"
+require "openssl"
+require "uri"
+require "zlib"
+
+require_relative "bulk_instances/version"
+require_relative "bulk_instances/error"
+require_relative "bulk_instances/types"
+require_relative "bulk_instances/aws_pricing_data"
+require_relative "bulk_instances/aws_instance"
+
+module Infrawrench
+  # AWS EC2 instance attributes and pricing, with full history, from Infrawrench.
+  module BulkInstances
+    DEFAULT_BASE_URL = "https://instances-bundles.infrawrench.com"
+    USER_AGENT = "infrawrench/bulk-instances, ruby, 1.0"
+
+    # What Net::HTTP raises when the server cannot be reached or the connection breaks.
+    TRANSPORT_ERRORS = [
+      SystemCallError, SocketError, IOError, Timeout::Error, OpenSSL::SSL::SSLError,
+      Net::ProtocolError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Zlib::Error,
+    ].freeze
+    private_constant :TRANSPORT_ERRORS
+
+    # Helpers shared by the decoders.
+    module Util
+      module_function
+
+      def time(ts)
+        Time.at(ts, in: "UTC").freeze
+      end
+
+      # Sorts rows by their first element (a timestamp). +sort_by+ is not stable, so the
+      # index keeps rows with the same timestamp in order (later rows win).
+      def sort_by_time(rows)
+        rows.each_with_index.sort_by { |row, i| [row[0], i] }.map(&:first)
+      end
+    end
+    private_constant :Util
+
+    class << self
+      # Downloads the AWS EC2 pricing data compressed blob and decompresses it.
+      #
+      # The bucket's +latest+ object names the newest bundle. With caching on, a bundle
+      # is only downloaded when +latest+ names one that is not in the cache yet, and if
+      # +latest+ cannot be fetched at all (offline, outage) the newest cached bundle is used.
+      #
+      # @param token [String, nil] Your Infrawrench API token. Defaults to the +INFRAWRENCH_API_KEY+ environment variable.
+      # @param cache_pricing_data [Boolean] Whether to cache the pricing data.
+      # @param cache_path [String, nil] The cache path. Defaults to <tt>~/.infrawrench/aws_pricing_data</tt>.
+      # @param base_url [String, nil] Where the bundles are served from. Defaults to https://instances-bundles.infrawrench.com
+      # @return [AWSPricingData]
+      # @raise [AWSPricingDataError]
+      def download_aws_ec2_pricing_data(token: nil, cache_pricing_data: true, cache_path: nil, base_url: nil)
+        token = ENV.fetch("INFRAWRENCH_API_KEY", nil) if token.nil?
+        if token.nil? || token.empty?
+          raise AWSPricingDataError, "No Infrawrench API token: pass `token:` or set INFRAWRENCH_API_KEY."
+        end
+
+        base_url = (base_url || DEFAULT_BASE_URL).sub(%r{/+\z}, "")
+        cache_path = (cache_path || File.join(Dir.home, ".infrawrench", "aws_pricing_data")).to_s
+
+        begin
+          latest = download("#{base_url}/latest", token, &:read_body).to_s.dup.force_encoding(Encoding::UTF_8).scrub.strip
+          unless latest.match?(/\A[A-Za-z0-9._-]+\z/)
+            raise AWSPricingDataError, %(latest holds "#{latest}", which is not a timestamp.)
+          end
+
+          name = "bundle-#{latest}.json"
+        rescue StandardError => e
+          # A rejected token is a real answer; anything else falls back to what we already have.
+          cached = newest_cached(cache_path) if cache_pricing_data && !(e.is_a?(AWSPricingDataError) && e.status == 401)
+          raise if cached.nil?
+
+          name = cached
+        end
+
+        # The cache holds the decompressed JSON: xz decompression is most of a load, reading the file is not.
+        json = read_cached(File.join(cache_path, name)) if cache_pricing_data
+        if json.nil?
+          require_relative "bulk_instances/xz"
+          json = XZ.decompress do |xz|
+            download("#{base_url}/#{name}.xz", token) { |res| res.read_body { |chunk| xz << chunk } }
+          end
+          if cache_pricing_data
+            FileUtils.mkdir_p(cache_path)
+            # Write then rename, so a crash mid-write never leaves a truncated bundle to be read next time.
+            tmp = File.join(cache_path, "#{name}.#{Process.pid}.tmp")
+            File.binwrite(tmp, json)
+            File.rename(tmp, File.join(cache_path, name))
+            # Older bundles, including compressed ones cached by earlier versions of this SDK.
+            Dir.each_child(cache_path) do |f|
+              FileUtils.rm_f(File.join(cache_path, f)) if f != name && f.match?(/\Abundle-.+\.json(\.xz)?\z/)
+            end
+          end
+        end
+
+        AWSPricingData.new(JSON.parse(json, freeze: true))
+      end
+
+      private
+
+      # GETs +url+ and yields the successful response, retrying the 503 the bundles Worker
+      # answers when it cannot reach the Infrawrench API.
+      def download(url, token)
+        uri = URI(url)
+        (1..).each do |attempt|
+          retry_after = nil
+          Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
+            request = Net::HTTP::Get.new(uri, "Authorization" => "Bearer #{token}", "User-Agent" => USER_AGENT)
+            http.request(request) do |res|
+              return yield res if res.is_a?(Net::HTTPSuccess)
+
+              status = res.code.to_i
+              if status == 401
+                raise AWSPricingDataError.new(
+                  "The Infrawrench API did not accept the token (unknown, revoked or expired).", status:,
+                )
+              end
+              if status != 503 || attempt == 3
+                raise AWSPricingDataError.new("GET #{url} failed with status #{status}.", status:)
+              end
+
+              retry_after = Float(res["Retry-After"], exception: false)
+              retry_after = 5 if retry_after.nil? || retry_after.nan? || retry_after.zero?
+            end
+          end
+          sleep([retry_after, 0].max)
+        end
+      rescue *TRANSPORT_ERRORS => e
+        raise AWSPricingDataError, "GET #{url} failed: #{e.message}"
+      end
+
+      # The newest bundle already in the cache directory, if any.
+      def newest_cached(cache_path)
+        Dir.children(cache_path).grep(/\Abundle-.+\.json\z/).max
+      rescue SystemCallError
+        nil
+      end
+
+      def read_cached(path)
+        File.read(path, encoding: Encoding::UTF_8)
+      rescue SystemCallError
+        nil
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification
+name: infrawrench-bulk_instances
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Infrawrench LLC
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+email:
+- astrid@infrawrench.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/infrawrench/bulk_instances.rb
+- lib/infrawrench/bulk_instances/aws_instance.rb
+- lib/infrawrench/bulk_instances/aws_pricing_data.rb
+- lib/infrawrench/bulk_instances/error.rb
+- lib/infrawrench/bulk_instances/types.rb
+- lib/infrawrench/bulk_instances/version.rb
+- lib/infrawrench/bulk_instances/xz.rb
+homepage: https://infrawrench.com
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://infrawrench.com
+  source_code_uri: https://github.com/Infrawrench/bulk-instances-sdks/tree/main/ruby
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: AWS EC2 instance attributes and pricing, with full history, from Infrawrench.
+test_files: []