maxmind-db 1.0.0.beta

data/test/data/tidyall.ini

@@ -0,0 +1,5 @@
+[PerlTidy]
+select = **/*.{pl,pm,t}
+
+[JSON]
+select = **/*.json

data/test/mmdb_util.rb

@@ -0,0 +1,24 @@
+class MMDBUtil # :nodoc:
+  def self.make_metadata_map(record_size)
+    # Map
+    "\xe9".b +
+      # node_count => 0
+      "\x4anode_count\xc0".b +
+      # record_size => 28 would be \xa1\x1c
+      "\x4brecord_size\xa1".b + record_size.chr.b +
+      # ip_version => 4
+      "\x4aip_version\xa1\x04".b +
+      # database_type => 'test'
+      "\x4ddatabase_type\x44test".b +
+      # languages => ['en']
+      "\x49languages\x01\x04\x42en".b +
+      # binary_format_major_version => 2
+      "\x5bbinary_format_major_version\xa1\x02".b +
+      # binary_format_minor_version => 0
+      "\x5bbinary_format_minor_version\xa0".b +
+      # build_epoch => 0
+      "\x4bbuild_epoch\x00\x02".b +
+      # description => 'hi'
+      "\x4bdescription\x42hi".b
+  end
+end

data/test/test_decoder.rb

@@ -0,0 +1,241 @@
+require 'maxmind/db'
+require 'minitest/autorun'
+require 'mmdb_util'
+
+class DecoderTest < Minitest::Test # :nodoc:
+  def test_arrays
+    arrays = {
+      "\x00\x04".b => [],
+      "\x01\x04\x43\x46\x6f\x6f".b => ['Foo'],
+      "\x02\x04\x43\x46\x6f\x6f\x43\xe4\xba\xba".b => %w[Foo 人],
+    }
+    validate_type_decoding('arrays', arrays)
+  end
+
+  def test_boolean
+    booleans = {
+      "\x00\x07".b => false,
+      "\x01\x07".b => true,
+    }
+    validate_type_decoding('booleans', booleans)
+  end
+
+  def test_bytes
+    tests = {
+      "\x83\xE4\xBA\xBA".b => '人'.b,
+    }
+    validate_type_decoding('bytes', tests)
+  end
+
+  def test_double
+    doubles = {
+      "\x68\x00\x00\x00\x00\x00\x00\x00\x00".b => 0.0,
+      "\x68\x3F\xE0\x00\x00\x00\x00\x00\x00".b => 0.5,
+      "\x68\x40\x09\x21\xFB\x54\x44\x2E\xEA".b => 3.14159265359,
+      "\x68\x40\x5E\xC0\x00\x00\x00\x00\x00".b => 123.0,
+      "\x68\x41\xD0\x00\x00\x00\x07\xF8\xF4".b => 1_073_741_824.12457,
+      "\x68\xBF\xE0\x00\x00\x00\x00\x00\x00".b => -0.5,
+      "\x68\xC0\x09\x21\xFB\x54\x44\x2E\xEA".b => -3.14159265359,
+      "\x68\xC1\xD0\x00\x00\x00\x07\xF8\xF4".b => -1_073_741_824.12457,
+    }
+    validate_type_decoding('double', doubles)
+  end
+
+  def test_float
+    floats = {
+      "\x04\x08\x00\x00\x00\x00".b => 0.0,
+      "\x04\x08\x3F\x80\x00\x00".b => 1.0,
+      "\x04\x08\x3F\x8C\xCC\xCD".b => 1.1,
+      "\x04\x08\x40\x48\xF5\xC3".b => 3.14,
+      "\x04\x08\x46\x1C\x3F\xF6".b => 9999.99,
+      "\x04\x08\xBF\x80\x00\x00".b => -1.0,
+      "\x04\x08\xBF\x8C\xCC\xCD".b => -1.1,
+      "\x04\x08\xC0\x48\xF5\xC3".b => -3.14,
+      "\x04\x08\xC6\x1C\x3F\xF6".b => -9999.99
+    }
+    validate_type_decoding('float', floats)
+  end
+
+  def test_int32
+    int32 = {
+      "\x00\x01".b => 0,
+      "\x04\x01\xff\xff\xff\xff".b => -1,
+      "\x01\x01\xff".b => 255,
+      "\x04\x01\xff\xff\xff\x01".b => -255,
+      "\x02\x01\x01\xf4".b => 500,
+      "\x04\x01\xff\xff\xfe\x0c".b => -500,
+      "\x02\x01\xff\xff".b => 65_535,
+      "\x04\x01\xff\xff\x00\x01".b => -65_535,
+      "\x03\x01\xff\xff\xff".b => 16_777_215,
+      "\x04\x01\xff\x00\x00\x01".b => -16_777_215,
+      "\x04\x01\x7f\xff\xff\xff".b => 2_147_483_647,
+      "\x04\x01\x80\x00\x00\x01".b => -2_147_483_647,
+    }
+    validate_type_decoding('int32', int32)
+  end
+
+  def test_map
+    maps = {
+      "\xe0".b => {},
+      "\xe1\x42\x65\x6e\x43\x46\x6f\x6f".b => {
+        'en' => 'Foo'
+      },
+      "\xe2\x42\x65\x6e\x43\x46\x6f\x6f\x42\x7a\x68\x43\xe4\xba\xba".b => {
+        'en' => 'Foo',
+        'zh' => '人'
+      },
+      "\xe1\x44\x6e\x61\x6d\x65\xe2\x42\x65\x6e".b +
+      "\x43\x46\x6f\x6f\x42\x7a\x68\x43\xe4\xba\xba".b => {
+        'name' => {
+          'en' => 'Foo',
+          'zh' => '人'
+        }
+      },
+      "\xe1\x49\x6c\x61\x6e\x67\x75\x61\x67\x65\x73".b +
+      "\x02\x04\x42\x65\x6e\x42\x7a\x68".b => {
+        'languages' => %w[en zh]
+      },
+      MMDBUtil.make_metadata_map(28) => {
+        'node_count'                  => 0,
+        'record_size'                 => 28,
+        'ip_version'                  => 4,
+        'database_type'               => 'test',
+        'languages'                   => ['en'],
+        'binary_format_major_version' => 2,
+        'binary_format_minor_version' => 0,
+        'build_epoch'                 => 0,
+        'description'                 => 'hi',
+      },
+    }
+    validate_type_decoding('maps', maps)
+  end
+
+  def test_pointer
+    pointers = {
+      "\x20\x00".b => 0,
+      "\x20\x05".b => 5,
+      "\x20\x0a".b => 10,
+      "\x23\xff".b => 1023,
+      "\x28\x03\xc9".b => 3017,
+      "\x2f\xf7\xfb".b => 524_283,
+      "\x2f\xff\xff".b => 526_335,
+      "\x37\xf7\xf7\xfe".b => 134_217_726,
+      "\x37\xff\xff\xff".b => 134_744_063,
+      "\x38\x7f\xff\xff\xff".b => 2_147_483_647,
+      "\x38\xff\xff\xff\xff".b => 4_294_967_295,
+    }
+    validate_type_decoding('pointers', pointers)
+  end
+
+  # rubocop:disable Style/ClassVars
+  @@strings = {
+    "\x40".b => '',
+    "\x41\x31".b => '1',
+    "\x43\xE4\xBA\xBA".b => '人',
+    "\x5b\x31\x32\x33\x34".b +
+    "\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35".b +
+    "\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37".b =>
+    '123456789012345678901234567',
+    "\x5c\x31\x32\x33\x34".b +
+    "\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35".b +
+    "\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36".b +
+    "\x37\x38".b => '1234567890123456789012345678',
+    "\x5d\x00\x31\x32\x33".b +
+    "\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34".b +
+    "\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35".b +
+    "\x36\x37\x38\x39".b => '12345678901234567890123456789',
+    "\x5d\x01\x31\x32\x33".b +
+    "\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34".b +
+    "\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35".b +
+    "\x36\x37\x38\x39\x30".b => '123456789012345678901234567890',
+    "\x5e\x00\xd7".b + "\x78".b * 500 => 'x' * 500,
+    "\x5e\x06\xb3".b + "\x78".b * 2000 => 'x' * 2000,
+    "\x5f\x00\x10\x53".b + "\x78".b * 70_000 => 'x' * 70_000,
+  }
+  # rubocop:enable Style/ClassVars
+
+  def test_string
+    values = validate_type_decoding('string', @@strings)
+    values.each do |s|
+      assert_equal(Encoding::UTF_8, s.encoding)
+    end
+  end
+
+  def test_uint16
+    uint16 = {
+      "\xa0".b => 0,
+      "\xa1\xff".b => 255,
+      "\xa2\x01\xf4".b => 500,
+      "\xa2\x2a\x78".b => 10_872,
+      "\xa2\xff\xff".b => 65_535,
+    }
+    validate_type_decoding('uint16', uint16)
+  end
+
+  def test_uint32
+    uint32 = {
+      "\xc0".b => 0,
+      "\xc1\xff".b => 255,
+      "\xc2\x01\xf4".b => 500,
+      "\xc2\x2a\x78".b => 10_872,
+      "\xc2\xff\xff".b => 65_535,
+      "\xc3\xff\xff\xff".b => 16_777_215,
+      "\xc4\xff\xff\xff\xff".b => 4_294_967_295,
+    }
+    validate_type_decoding('uint32', uint32)
+  end
+
+  def generate_large_uint(bits)
+    ctrl_byte = bits == 64 ? "\x02".b : "\x03".b
+    uints = {
+      "\x00".b + ctrl_byte => 0,
+      "\x02".b + ctrl_byte + "\x01\xf4".b => 500,
+      "\x02".b + ctrl_byte + "\x2a\x78".b => 10_872,
+    }
+    (bits / 8 + 1).times do |power|
+      expected = 2**(8 * power) - 1
+      input = [power].pack('C') + ctrl_byte + "\xff".b * power
+      uints[input] = expected
+    end
+    uints
+  end
+
+  def test_uint64
+    validate_type_decoding('uint64', generate_large_uint(64))
+  end
+
+  def test_uint128
+    validate_type_decoding('uint128', generate_large_uint(128))
+  end
+
+  def validate_type_decoding(type, tests)
+    values = []
+    tests.each do |input, expected|
+      values << check_decoding(type, input, expected)
+    end
+    values
+  end
+
+  def check_decoding(type, input, expected, name = nil)
+    name ||= expected
+
+    io = MaxMind::DB::MemoryReader.new(input, is_buffer: true)
+
+    pointer_base = 0
+    pointer_test = true
+    decoder = MaxMind::DB::Decoder.new(io, pointer_base,
+                                       pointer_test)
+
+    offset = 0
+    r = decoder.decode(offset)
+
+    if %w[float double].include?(type)
+      assert_in_delta(expected, r[0], 0.001, name)
+    else
+      assert_equal(expected, r[0], name)
+    end
+
+    io.close
+    r[0]
+  end
+end

data/test/test_reader.rb

@@ -0,0 +1,415 @@
+require 'maxmind/db'
+require 'minitest/autorun'
+require 'mmdb_util'
+
+class ReaderTest < Minitest::Test # :nodoc:
+  def test_reader
+    modes = [
+      MaxMind::DB::MODE_FILE,
+      MaxMind::DB::MODE_MEMORY,
+    ]
+
+    modes.each do |mode|
+      [24, 28, 32].each do |record_size|
+        [4, 6].each do |ip_version|
+          filename = 'test/data/test-data/MaxMind-DB-test-ipv' +
+                     ip_version.to_s + '-' + record_size.to_s + '.mmdb'
+          reader = MaxMind::DB.new(filename, mode: mode)
+          check_metadata(reader, ip_version, record_size)
+          if ip_version == 4
+            check_ipv4(reader, filename)
+          else
+            check_ipv6(reader, filename)
+          end
+          reader.close
+        end
+      end
+    end
+  end
+
+  def test_decoder
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    record = reader.get('::1.1.1.0')
+    assert_equal([1, 2, 3], record['array'])
+    assert_equal(true, record['boolean'])
+    assert_equal("\x00\x00\x00*".b, record['bytes'])
+    assert_equal(42.123456, record['double'])
+    assert_in_delta(1.1, record['float'])
+    assert_equal(-268_435_456, record['int32'])
+    assert_equal(
+      {
+        'mapX' => {
+          'arrayX'       => [7, 8, 9],
+          'utf8_stringX' => 'hello',
+        },
+      },
+      record['map'],
+    )
+    assert_equal(100, record['uint16'])
+    assert_equal(268_435_456, record['uint32'])
+    assert_equal(1_152_921_504_606_846_976, record['uint64'])
+    assert_equal('unicode! ☯ - ♫', record['utf8_string'])
+    assert_equal(1_329_227_995_784_915_872_903_807_060_280_344_576, record['uint128'])
+    reader.close
+  end
+
+  def test_no_ipv4_search_tree
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb'
+    )
+    assert_equal('::0/64', reader.get('1.1.1.1'))
+    assert_equal('::0/64', reader.get('192.1.1.1'))
+    reader.close
+  end
+
+  def test_ipv6_address_in_ipv4_database
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-ipv4-24.mmdb'
+    )
+    e = assert_raises ArgumentError do
+      reader.get('2001::')
+    end
+    assert_equal(
+      'Error looking up 2001::. You attempted to look up an IPv6 address in an IPv4-only database.',
+      e.message,
+    )
+    reader.close
+  end
+
+  def test_bad_ip_parameter
+    reader = MaxMind::DB.new('test/data/test-data/GeoIP2-City-Test.mmdb')
+    e = assert_raises ArgumentError do
+      reader.get(Object.new)
+    end
+    assert_equal(
+      'address family must be specified', # Not great, but type is ok
+      e.message,
+    )
+    reader.close
+  end
+
+  def test_broken_database
+    reader = MaxMind::DB.new(
+      'test/data/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb'
+    )
+    e = assert_raises MaxMind::DB::InvalidDatabaseError do
+      reader.get('2001:220::')
+    end
+    assert_equal(
+      'The MaxMind DB file\'s data section contains bad data (unknown data type or corrupt data)',
+      e.message,
+    )
+    reader.close
+  end
+
+  def test_ip_validation
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    e = assert_raises ArgumentError do
+      reader.get('not_ip')
+    end
+    assert_equal('invalid address', e.message)
+    reader.close
+  end
+
+  def test_missing_database
+    e = assert_raises SystemCallError do
+      MaxMind::DB.new('file-does-not-exist.mmdb')
+    end
+    assert(e.message.match(/No such file or directory/))
+  end
+
+  def test_nondatabase
+    e = assert_raises MaxMind::DB::InvalidDatabaseError do
+      MaxMind::DB.new('README.md')
+    end
+    assert_equal(
+      'Metadata section not found. Is this a valid MaxMind DB file?',
+      e.message,
+    )
+  end
+
+  def test_too_many_constructor_args
+    e = assert_raises ArgumentError do
+      MaxMind::DB.new('README.md', {}, 'blah')
+    end
+    assert(e.message.match(/wrong number of arguments/))
+  end
+
+  def test_no_constructor_args
+    e = assert_raises ArgumentError do
+      MaxMind::DB.new
+    end
+    assert(e.message.match(/wrong number of arguments/))
+  end
+
+  def test_too_many_get_args
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    e = assert_raises ArgumentError do
+      reader.get('1.1.1.1', 'blah')
+    end
+    assert(e.message.match(/wrong number of arguments/))
+    reader.close
+  end
+
+  def test_no_get_args
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    e = assert_raises ArgumentError do
+      reader.get
+    end
+    assert(e.message.match(/wrong number of arguments/))
+    reader.close
+  end
+
+  def test_metadata_args
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    e = assert_raises ArgumentError do
+      reader.metadata('hi')
+    end
+    assert(e.message.match(/wrong number of arguments/))
+    reader.close
+  end
+
+  def test_metadata_unknown_attribute
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    e = assert_raises NoMethodError do
+      reader.metadata.what
+    end
+    assert(e.message.match(/undefined method `what'/))
+    reader.close
+  end
+
+  def test_close
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    reader.close
+  end
+
+  def test_double_close
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    reader.close
+    reader.close
+  end
+
+  def test_closed_get
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    reader.close
+    e = assert_raises IOError do
+      reader.get('1.1.1.1')
+    end
+    assert_equal('closed stream', e.message)
+  end
+
+  def test_closed_metadata
+    reader = MaxMind::DB.new(
+      'test/data/test-data/MaxMind-DB-test-decoder.mmdb'
+    )
+    reader.close
+    assert_equal(
+      { 'en' => 'MaxMind DB Decoder Test database - contains every MaxMind DB data type' },
+      reader.metadata.description,
+    )
+  end
+
+  def test_threads
+    reader = MaxMind::DB.new(
+      'test/data/test-data/GeoIP2-Domain-Test.mmdb'
+    )
+
+    num_threads = 16
+    num_lookups = 32
+    thread_lookups = []
+    num_threads.times do
+      thread_lookups << []
+    end
+
+    threads = []
+    num_threads.times do |i|
+      threads << Thread.new do
+        num_lookups.times do |j|
+          thread_lookups[i] << reader.get("65.115.240.#{j}")
+          thread_lookups[i] << reader.get("2a02:2770:3::#{j}")
+        end
+      end
+    end
+
+    threads.each(&:join)
+
+    thread_lookups.each do |a|
+      assert_equal(num_lookups * 2, a.length)
+      thread_lookups.each do |b|
+        assert_equal(a, b)
+      end
+    end
+
+    reader.close
+  end
+
+  # In these tests I am trying to exercise Reader#read_node directly. It is not
+  # too easy to test its behaviour with real databases, so construct dummy ones
+  # directly.
+  #
+  def test_read_node
+    tests = [
+      {
+        record_size: 24,
+        # Left record + right record
+        node_bytes:  "\xab\xcd\xef".b + "\xbc\xfe\xfa".b,
+        left:        11_259_375,
+        right:       12_386_042,
+        check_left:  "\x00\xab\xcd\xef".b.unpack('N')[0],
+        check_right: "\x00\xbc\xfe\xfa".b.unpack('N')[0],
+      },
+      {
+        record_size: 28,
+        # Left record (part) + middle byte + right record (part)
+        node_bytes:  "\xab\xcd\xef".b + "\x12".b + "\xfd\xdc\xfa".b,
+        left:        28_036_591,
+        right:       50_191_610,
+        check_left:  "\x01\xab\xcd\xef".b.unpack('N')[0],
+        check_right: "\x02\xfd\xdc\xfa".b.unpack('N')[0],
+      },
+      {
+        record_size: 32,
+        # Left record + right record
+        node_bytes:  "\xab\xcd\xef\x12".b + "\xfd\xdc\xfa\x15".b,
+        left:        2_882_400_018,
+        right:       4_259_117_589,
+        check_left:  "\xab\xcd\xef\x12".b.unpack('N')[0],
+        check_right: "\xfd\xdc\xfa\x15".b.unpack('N')[0],
+      },
+    ]
+
+    tests.each do |test|
+      buf = ''.b
+      buf += test[:node_bytes]
+
+      buf += "\x00".b * 16
+
+      buf += "\xab\xcd\xefMaxMind.com".b
+      buf += MMDBUtil.make_metadata_map(test[:record_size])
+
+      reader = MaxMind::DB.new(
+        buf, mode: MaxMind::DB::MODE_PARAM_IS_BUFFER
+      )
+
+      assert_equal(reader.metadata.record_size, test[:record_size])
+
+      assert_equal(test[:left],  reader.send(:read_node, 0, 0))
+      assert_equal(test[:right], reader.send(:read_node, 0, 1))
+      assert_equal(test[:left],  test[:check_left])
+      assert_equal(test[:right], test[:check_right])
+    end
+  end
+
+  def check_metadata(reader, ip_version, record_size)
+    metadata = reader.metadata
+
+    assert_equal(2, metadata.binary_format_major_version, 'major_version')
+    assert_equal(0, metadata.binary_format_minor_version, 'minor_version')
+    assert_operator(metadata.build_epoch, :>, 1_373_571_901, 'build_epoch')
+    assert_equal('Test', metadata.database_type, 'database_type')
+    assert_equal(
+      {
+        'en' => 'Test Database',
+        'zh' => 'Test Database Chinese',
+      },
+      metadata.description,
+      'description',
+    )
+    assert_equal(ip_version, metadata.ip_version, 'ip_version')
+    assert_equal(%w[en zh], metadata.languages, 'languages')
+    assert_operator(metadata.node_count, :>, 36, 'node_count')
+    assert_equal(record_size, metadata.record_size, 'record_size')
+  end
+
+  def check_ipv4(reader, filename)
+    6.times do |i|
+      address = "1.1.1.#{2**i}"
+      assert_equal(
+        { 'ip' => address },
+        reader.get(address),
+        "found expected data record for #{address} in #{filename}",
+      )
+    end
+
+    pairs = {
+      '1.1.1.3'  => '1.1.1.2',
+      '1.1.1.5'  => '1.1.1.4',
+      '1.1.1.7'  => '1.1.1.4',
+      '1.1.1.9'  => '1.1.1.8',
+      '1.1.1.15' => '1.1.1.8',
+      '1.1.1.17' => '1.1.1.16',
+      '1.1.1.31' => '1.1.1.16',
+    }
+    pairs.each do |key_address, value_address|
+      data = { 'ip' => value_address }
+      assert_equal(
+        data,
+        reader.get(key_address),
+        "found expected data record for #{key_address} in #{filename}",
+      )
+    end
+
+    ['1.1.1.33', '255.254.253.123'].each do |ip|
+      assert_nil(
+        reader.get(ip),
+        "#{ip} is not in #{filename}",
+      )
+    end
+  end
+
+  def check_ipv6(reader, filename)
+    subnets = [
+      '::1:ffff:ffff', '::2:0:0', '::2:0:40', '::2:0:50', '::2:0:58',
+    ]
+    subnets.each do |address|
+      assert_equal(
+        { 'ip' => address },
+        reader.get(address),
+        "found expected data record for #{address} in #{filename}",
+      )
+    end
+
+    pairs = {
+      '::2:0:1'  => '::2:0:0',
+      '::2:0:33' => '::2:0:0',
+      '::2:0:39' => '::2:0:0',
+      '::2:0:41' => '::2:0:40',
+      '::2:0:49' => '::2:0:40',
+      '::2:0:52' => '::2:0:50',
+      '::2:0:57' => '::2:0:50',
+      '::2:0:59' => '::2:0:58',
+    }
+    pairs.each do |key_address, value_address|
+      assert_equal(
+        { 'ip' => value_address },
+        reader.get(key_address),
+        "found expected data record for #{key_address} in #{filename}",
+      )
+    end
+
+    ['1.1.1.33', '255.254.253.123', '89fa::'].each do |ip|
+      assert_nil(
+        reader.get(ip),
+        "#{ip} is not in #{filename}",
+      )
+    end
+  end
+end