iparty 0.1.0

data/lib/iparty/config.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module IParty
+  Config = Struct.new(
+    :account_id,
+    :license_key,
+    :mirror,
+    :directory,
+    :editions,
+    :eager_load,
+    :singletons,
+    :local_ip_alias,
+    :ipv6_significant,
+    :url_to_mmdb,
+    :annotations,
+    keyword_init: true,
+  ) do
+    def singletons=(val)
+      self[:singletons] = val
+      init_singletons! if val == true
+    end
+
+    def init_singletons!
+      self[:singletons] = {} if !self[:singletons] || self[:singletons] == true
+
+      editions.each {|edition| IParty::MaxMind.db(edition) }
+      self[:singletons]
+    end
+
+    def env_value *args, **kw, &block
+      IParty.env_value(*args, **kw, &block)
+    end
+
+    def annotate *args, **kw
+      self[:annotations] ||= {}
+      result = {}
+      IParty.expand_hostnames(args).each do |net|
+        ipp = IParty(net)
+        kw[:tags] = ((result[ipp] || self[:annotations][ipp] || {})[:tags] || []) | kw[:tags] if kw.key?(:tags)
+        result[ipp] = (result[ipp] || self[:annotations][ipp] || {}).merge(kw)
+      end
+      self[:annotations].merge!(result)
+      result
+    end
+
+    def annotate_tag tags, *args
+      self[:annotations] ||= {}
+      IParty.expand_hostnames(args).to_h do |net|
+        ipp = IParty(net)
+        self[:annotations][ipp] ||= {}
+        self[:annotations][ipp][:tags] ||= []
+        self[:annotations][ipp][:tags] |= Array(tags)
+        [net, self[:annotations][ipp][:tags]]
+      end
+    end
+  end
+
+  class << self
+    attr_accessor :config
+
+    def configure
+      yield(config)
+    end
+
+    def env_value key, default = nil, &vproc
+      env_value = ENV.fetch(key, default)
+
+      env_value = case env_value
+      when "1", "true", "on", "yes" then true
+      when "0", "false", "off", "no" then false
+      when "" then nil
+      else env_value
+      end
+
+      vproc ? vproc.call(env_value) : env_value
+    end
+
+    def with_config to_merge = {}, &block
+      config_was = @config
+      new_config = Config.new(config_was.to_h.merge(to_merge))
+
+      if block
+        begin
+          block.call(self.config = new_config)
+        ensure
+          self.config = config_was
+        end
+      else
+        new_config
+      end
+    end
+
+    def default_config
+      Config.new(
+        # If set to false drop last half of v6-addresses as they are insignificant for most applications.
+        # The then 64-bit addresses also fit into unsigned bigints allowing for easy range representations.
+        # Each IParty::Address can overwrite this with
+        #   * #ipv6_significant accessors
+        #   * significant: keyword (affected methods only but including new/initialize)
+        ipv6_significant: env_value("IPARTY_IPV6_SIGNIFICANT", true),
+
+        # Whether to use the low memory file reader or load mmdb into memory as a whole (see docs/BENCHMARK.md)
+        eager_load: env_value("IPARTY_EAGER_LOAD", false),
+
+        # Singleton instances (may be false, true, hash or proc which returns hash-like)
+        singletons: env_value("IPARTY_SINGLETONS", false),
+
+        # An IP that is used instead of local IPs
+        local_ip_alias: env_value("IPARTY_LOCAL_IP_ALIAS", nil),
+
+        # MaxMind account_id and license_key aka mirror basic-auth
+        account_id: env_value("MAXMIND_ACCOUNT_ID", nil),
+        license_key: env_value("MAXMIND_LICENSE_KEY", nil),
+
+        # Mirror to download tar.gz compressed mmdb-files from
+        mirror: env_value("MAXMIND_MIRROR", "https://download.maxmind.com/geoip/databases/:edition/download?suffix=tar.gz"),
+
+        # The mmdb editions to fetch, you don't really need country if you have city
+        editions: env_value("MAXMIND_EDITIONS", "GeoLite2-ASN GeoLite2-Country GeoLite2-City") {|v| v.split(/\s+|,\s*/) },
+
+        # Directory to store mmdb-files in, also creates .updating subdirectory
+        directory: env_value("IPARTY_DIRECTORY", nil) do |dir|
+          if dir && !dir.empty?
+            Pathname.new(dir)
+          else
+            Pathname.new(Dir.tmpdir).join("iparty")
+          end
+        end,
+
+        # Proc to turn URL into mmdb-file(s) inside target directory.
+        # All .mmdb files will be moved and then dir gets removed.
+        url_to_mmdb: proc do |url, dir, config|
+          auth = %{-u "#{config.account_id}:#{config.license_key}"} if config.account_id && config.license_key
+          curl = %{curl -L -s #{"#{auth} " if auth}"#{url}"}
+          tar = %{tar xz --strip-components 1 --exclude "*.txt" --no-same-owner -C #{dir.to_s.shellescape}}
+          system("#{curl} | #{tar}")
+        end,
+      )
+    end
+  end
+end

data/lib/iparty/max_mind/database.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+require_relative "lazy_reader"
+require_relative "eager_reader"
+require_relative "result"
+
+module IParty
+  class MaxMind
+    class Database
+      class Error < IParty::Error; end
+      class InvalidFileFormatError < Error; end
+
+      METADATA_BEGIN_MARKER = "#{[0xAB, 0xCD, 0xEF].pack("C*")}MaxMind.com".encode("ascii-8bit", "ascii-8bit").freeze
+      DATA_SECTION_SEPARATOR_SIZE = 16
+      SIZE_BASE_VALUES = [0, 29, 285, 65_821].freeze
+      POINTER_BASE_VALUES = [0, 0, 2048, 526_336].freeze
+
+      attr_reader :metadata
+
+      def initialize(path, reader: EagerReader)
+        @path = path
+        @data = reader.new(path)
+
+        pos = @data.rindex(METADATA_BEGIN_MARKER) || raise(InvalidFileFormatError, "invalid file format")
+        pos += METADATA_BEGIN_MARKER.size
+        @metadata = decode(0, pos)[1]
+
+        @ip_version = @metadata[:ip_version]
+        @start_idx = @ip_version == 4 ? 96 : 0
+        @node_count = @metadata[:node_count]
+        @node_byte_size = @metadata[:record_size] * 2 / 8
+        @search_tree_size = @node_count * @node_byte_size
+        @data_section_start = @search_tree_size + DATA_SECTION_SEPARATOR_SIZE
+      end
+
+      def close
+        @data.close
+      end
+
+      def closed?
+        @data.closed?
+      end
+
+      def inspect
+        "#<#{self.class}:#{format("0x%x", object_id << 1)}: @path:#{@path} @metadata:#{@metadata}>"
+      end
+
+      def lookup(addr, result_class: Result::Geo)
+        addr = IPAddr.new(addr) unless addr.is_a?(IPAddr)
+        addr = IParty.config.local_ip_alias if IParty.config.local_ip_alias && addr.loopback?
+        addr = IPAddr.new(addr) unless addr.is_a?(IPAddr)
+        addr = addr.ipv4_compat if addr.ipv4?
+        long = addr.is_a?(IParty::Address) ? addr.to_i(significant: true) : addr.to_i
+        node_no = 0
+
+        (@start_idx...128).each do |i|
+          flag = (long >> (127 - i)) & 1
+          next_node_no = read_record(node_no, flag)
+
+          if next_node_no == 0
+            raise(InvalidFileFormatError, "invalid file format")
+          elsif next_node_no >= @node_count
+            pos = (next_node_no - @node_count) - DATA_SECTION_SEPARATOR_SIZE
+            result           = decode(@data_section_start, pos)[1]
+            result[:network] = cidr_from_long(long, i) unless result.empty?
+            return result_class.new(result)
+          else
+            node_no = next_node_no
+          end
+        end
+
+        raise(InvalidFileFormatError, "invalid file format")
+      end
+
+      def cidr_from_long(long, net)
+        addr = IPAddr.new(long, long > (2**32) - 1 ? Socket::AF_INET6 : Socket::AF_INET)
+        subnet_size = addr.ipv4? ? net - 96 + 1 : net + 1
+        subnet      = IPAddr.new("#{addr}/#{subnet_size}")
+
+        "#{subnet}/#{subnet_size}"
+      end
+
+      def read_record(node_no, flag)
+        rec_byte_size = @node_byte_size / 2
+        pos = @node_byte_size * node_no
+        middle = @data[pos + rec_byte_size].ord if @node_byte_size.odd?
+
+        if flag == 0 # left
+          val = read_value(pos, 0, rec_byte_size)
+          val += ((middle & 0xf0) << 20) if middle
+        else # right
+          val = read_value(pos + @node_byte_size - rec_byte_size, 0, rec_byte_size)
+          val += ((middle & 0xf) << 24) if middle
+        end
+
+        val
+      end
+
+      def read_value(base_pos, pos, size)
+        bytes = @data[base_pos + pos, size].unpack("C*")
+        bytes.inject(0){|r, v| (r << 8) + v }
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity -- we could reduce this, sacrificing the neat overview
+      def decode base_pos, pos
+        ctrl = @data[base_pos + pos].ord
+        pos += 1
+        type = ctrl >> 5
+
+        if type == 1 # pointer
+          decode_pointer(base_pos, pos, ctrl)
+        else
+          if type == 0 # extended type
+            type = 7 + @data[base_pos + pos].ord
+            pos += 1
+          end
+
+          size = ctrl & 0x1f
+          if size >= 29
+            byte_size = size - 29 + 1
+            val = read_value(base_pos, pos, byte_size)
+            pos += byte_size
+            size = val + SIZE_BASE_VALUES[byte_size]
+          end
+
+          # rubocop:disable Lint/DuplicateBranch -- readable order is more important
+          case type
+          when 2 # utf8
+            decode_utf8(base_pos, pos, size)
+          when 3 # double
+            decode_double(base_pos, pos, size)
+          when 4 # bytes
+            decode_bytes(base_pos, pos, size)
+          when 5 # unsigned 16-bit int
+            decode_uint(base_pos, pos, size)
+          when 6 # unsigned 32-bit int
+            decode_uint(base_pos, pos, size)
+          when 7 # map
+            decode_map(base_pos, pos, size)
+          when 8 # signed 32-bit int
+            decode_int(base_pos, pos, size)
+          when 9 # unsigned 64-bit int
+            decode_uint(base_pos, pos, size)
+          when 10 # unsigned 128-bit int
+            decode_uint(base_pos, pos, size)
+          when 11 # array
+            decode_array(base_pos, pos, size)
+          when 12 # (deprecated) data cache container
+            raise "TODO: (deprecated) data cache container format"
+          when 13 # (deprecated) end marker
+            [pos, nil]
+          when 14 # boolean
+            [pos, size != 0]
+          when 15 # float
+            decode_float(base_pos, pos, size)
+          end
+          # rubocop:enable Lint/DuplicateBranch
+        end
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity
+
+      def decode_double base_pos, pos, size
+        [pos + size, @data[base_pos + pos, size].unpack1("G")]
+      end
+
+      def decode_bytes base_pos, pos, size
+        [pos + size, @data[base_pos + pos, size]]
+      end
+
+      def decode_int base_pos, pos, size
+        v1 = @data[base_pos + pos, size].unpack1("N")
+        bits = size * 8
+        [pos + size, (v1 & ~(1 << bits)) - (v1 & (1 << bits))]
+      end
+
+      def decode_uint base_pos, pos, size
+        [pos + size, read_value(base_pos, pos, size)]
+      end
+
+      def decode_float base_pos, pos, size
+        [pos + size, @data[base_pos + pos, size].unpack1("g")]
+      end
+
+      def decode_utf8 base_pos, pos, size
+        [pos + size, @data[base_pos + pos, size].encode("utf-8", "utf-8")]
+      end
+
+      def decode_pointer base_pos, pos, ctrl
+        size = ((ctrl >> 3) & 0x3) + 1
+        v1 = ctrl & 0x7
+        v2 = read_value(base_pos, pos, size)
+
+        pointer = (v1 << (8 * size)) + v2 + POINTER_BASE_VALUES[size]
+        [pos + size, decode(base_pos, pointer)[1]]
+      end
+
+      def decode_array base_pos, pos, size
+        ary = Array.new(size) do
+          pos, v = decode(base_pos, pos)
+          v
+        end
+        [pos, ary]
+      end
+
+      def decode_map base_pos, pos, size
+        map = {}
+        size.times do
+          pos, k = decode(base_pos, pos)
+          pos, v = decode(base_pos, pos)
+          map[k.to_sym] = v
+        end
+        [pos, map]
+      end
+    end
+  end
+end

data/lib/iparty/max_mind/eager_reader.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module IParty
+  class MaxMind
+    # A fast read-access reader for MaxMindDB (mmdb) files. Reads the database into memory.
+    # This creates a higher memory overhead and slower init phase but faster lookup times.
+    class EagerReader
+      def initialize path
+        @data = File.binread(path)
+      end
+
+      def [] pos, length = 1
+        raise IOError, "closed stream" unless @data
+
+        @data.slice(pos, length)
+      end
+
+      def rindex search
+        raise IOError, "closed stream" unless @data
+
+        @data.rindex(search)
+      end
+
+      def close
+        @data = nil
+      end
+
+      def closed?
+        @data.nil?
+      end
+    end
+  end
+end

data/lib/iparty/max_mind/lazy_reader.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module IParty
+  class MaxMind
+    # A low memory file reader for MaxMindDB (mmdb) files. Avoids reading the database into memory.
+    # Has a lower memory footprint but slower lookup times.
+    class LazyReader
+      METADATA_MAX_SIZE = 128 * 1024
+
+      def initialize path
+        @mutex = Mutex.new
+        @file = File.open(path, "rb")
+      end
+
+      def [] pos, length = 1
+        atomic_read(length, pos)
+      end
+
+      def close
+        @file.close
+      end
+
+      def closed?
+        @file.closed?
+      end
+
+      def rindex search
+        base = [0, @file.size - METADATA_MAX_SIZE].max
+        tail = atomic_read(METADATA_MAX_SIZE, base)
+        pos = tail.rindex(search)
+        base + pos unless pos.nil?
+      end
+
+      def atomic_read length, pos
+        # Prefer `pread` in environments where it is available. `pread` provides atomic file access across processes.
+        if @file.respond_to?(:pread)
+          @file.pread(length, pos)
+        else
+          @mutex.synchronize do
+            @file.seek(pos)
+            @file.read(length)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/iparty/max_mind/result.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+module IParty
+  class MaxMind
+    class Result < Hash
+      class << self
+        def define_attr(name, attribute = name, type: nil, aliases: nil, memoize: nil, export: nil, &transform)
+          ivar = :"@#{attribute}"
+          transform ||= ->(v) { type.new(v) } if type
+          memoize = true if memoize.nil? && transform
+          method_names = [name] + Array(aliases)
+
+          method_names.each do |meth|
+            define_method(meth) do
+              value = dig(attribute)
+              return transform ? instance_exec(value, &transform) : value unless memoize
+              return instance_variable_get(ivar) if instance_variable_defined?(ivar)
+
+              instance_variable_set(ivar, instance_exec(value, &transform))
+            end
+          end
+
+          Array(export).each{|exp| export_attr(name, exp == true ? name : exp) }
+
+          method_names
+        end
+
+        def export_attr name, export
+          if self == Result::Asn
+            IParty::Address.def_delegator(:asn, name, export)
+          elsif self == Result::Geo || self == Result
+            IParty::Address.def_delegator(:geo, name, export)
+          else
+            raise ArgumentError, "you can only export on Result, Geo(*) and Asn (got #{self})"
+          end
+        end
+      end
+
+      class Location < Result
+        define_attr(:accuracy_radius)
+        define_attr(:latitude)
+        define_attr(:longitude)
+        define_attr(:metro_code)
+        define_attr(:time_zone)
+      end
+
+      class NamedLocation < Result
+        define_attr(:code)
+        define_attr(:geoname_id)
+        define_attr(:is_in_european_union, aliases: :in_european_union?)
+        define_attr(:iso_code)
+        define_attr(:names, type: Result)
+
+        def name(locale = :en, fallback_locale: :en)
+          return unless all = dig(:names)
+
+          all[locale] || (all[fallback_locale] if fallback_locale)
+        end
+
+        # dynamic cast
+        alias_method :to_s, :name
+        alias_method :to_i, :geoname_id
+
+        # dynamic comparison
+        def == other
+          case other
+          when String
+            name && other == name
+          when Numeric
+            geoname_id && other == geoname_id
+          else
+            super
+          end
+        end
+
+        # dynamic inquiry
+        define_attr(:inquire_on_name) { name&.downcase&.tr(" ", "_") }
+        define_attr(:inquire_on_code) { (iso_code || code)&.downcase }
+
+        def respond_to_missing? method_name, include_private = false
+          method_name.end_with?("?") || super
+        end
+
+        def method_missing method_name, *arguments
+          if method_name.end_with?("?")
+            method_name[0..-2] == (method_name.length == 3 ? inquire_on_code : inquire_on_name)
+          else
+            super
+          end
+        end
+      end
+
+      class City < NamedLocation
+      end
+
+      class Continent < NamedLocation
+      end
+
+      class Country < NamedLocation
+      end
+
+      class Subdivision < NamedLocation
+      end
+
+      class Subdivisions < Array
+        def initialize raw
+          super((raw || []).map{|hash| Subdivision.new(hash) })
+        end
+
+        def inspect
+          "#<#{self.class}:#{format("0x%x", object_id << 1)}: #{super}>"
+        end
+
+        alias_method :blank?, :empty?
+
+        def present?
+          !empty?
+        end
+
+        def presence
+          self if present?
+        end
+
+        def least_specific
+          first || Subdivision.new
+        end
+
+        def most_specific
+          last || Subdivision.new
+        end
+      end
+
+      class Postal < Result
+        define_attr(:code)
+      end
+
+      class Traits < Result
+        define_attr(:is_anonymous_proxy, aliases: :anonymous_proxy?)
+        define_attr(:is_satellite_provider, aliases: :satellite_provider?)
+      end
+
+      # ------------------------------------------
+
+      class Asn < Result
+        def initialize(data = {})
+          data ? super({ autonomous_system_network: data[:network] }.compact.merge(data)) : super()
+        end
+
+        define_attr(:autonomous_system_network, aliases: :network, export: true)
+        define_attr(:autonomous_system_number, aliases: :number, export: true)
+        define_attr(:autonomous_system_organization, aliases: :organization, export: true)
+        define_attr(:autonomous_system_detailed, aliases: :detailed, export: true) { "AS#{number} #{organization}" if number }
+      end
+
+      class Geo < Result
+        define_attr(:city, type: City, export: true)
+        define_attr(:connection_type)
+        define_attr(:continent, type: Continent, export: true)
+        define_attr(:country, type: Country, export: true)
+        define_attr(:location, type: Location)
+        define_attr(:network)
+        define_attr(:postal, type: Postal)
+        define_attr(:registered_country, type: Country)
+        define_attr(:represented_country, type: Country)
+        define_attr(:subdivisions, type: Subdivisions)
+        define_attr(:traits, type: Traits)
+
+        define_attr(:accuracy_radius, memoize: false) { location.accuracy_radius }
+        define_attr(:is_in_european_union, aliases: :in_european_union?, export: :in_european_union?, memoize: false) { country.is_in_european_union }
+        define_attr(:latitude, memoize: false, export: true) { location.latitude }
+        define_attr(:longitude, memoize: false, export: true) { location.longitude }
+        define_attr(:metro_code, memoize: false) { location.metro_code }
+        define_attr(:time_zone, memoize: false, export: true) { location.time_zone }
+        define_attr(:postal_code, aliases: :zip, export: true, memoize: false) { postal.code }
+
+        define_attr(:detailed_parts, memoize: false) { [continent.code, country.name, city.name].compact }
+        define_attr(:detailed, memoize: false) { detailed_parts.join(" / ") }
+      end
+
+      class GeoCountry < Geo; end
+      class GeoCity < Geo; end
+
+      # ------------------------------------------
+
+      def initialize(data = {})
+        super()
+        merge!(data) if data
+      end
+
+      def inspect
+        "#<#{self.class}:#{format("0x%x", object_id << 1)}: #{super}>"
+      end
+
+      alias_method :blank?, :empty?
+
+      def present?
+        !empty?
+      end
+
+      def presence
+        self if present?
+      end
+    end
+  end
+end