geoip2_c 0.3.2 → 0.3.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (132) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/ubuntu.yml +33 -0
  3. data/.github/workflows/windows.yml +52 -0
  4. data/README.md +22 -10
  5. data/docker-compose.yml +22 -1
  6. data/dockerfiles/{Dockerfile-ruby2.3 → Dockerfile-ruby2.5} +1 -1
  7. data/dockerfiles/{Dockerfile-ruby2.1 → Dockerfile-ruby2.6} +1 -1
  8. data/dockerfiles/{Dockerfile-ruby2.2 → Dockerfile-ruby2.7} +1 -1
  9. data/ext/geoip2/extconf.rb +11 -4
  10. data/ext/geoip2/geoip2.c +63 -18
  11. data/ext/geoip2/libmaxminddb/.gitignore +3 -0
  12. data/ext/geoip2/libmaxminddb/.travis.yml +24 -2
  13. data/ext/geoip2/libmaxminddb/Changes.md +39 -0
  14. data/ext/geoip2/libmaxminddb/README.dev.md +41 -30
  15. data/ext/geoip2/libmaxminddb/README.md +3 -3
  16. data/ext/geoip2/libmaxminddb/bin/Makefile.am +5 -0
  17. data/ext/geoip2/libmaxminddb/bin/mmdblookup.c +333 -15
  18. data/ext/geoip2/libmaxminddb/configure.ac +5 -5
  19. data/ext/geoip2/libmaxminddb/dev-bin/ppa-release.sh +8 -5
  20. data/ext/geoip2/libmaxminddb/dev-bin/release.sh +7 -0
  21. data/ext/geoip2/libmaxminddb/dev-bin/valgrind-all.pl +10 -3
  22. data/ext/geoip2/libmaxminddb/include/maxminddb.h +11 -2
  23. data/ext/geoip2/libmaxminddb/projects/VS12/libmaxminddb.vcxproj +3 -1
  24. data/ext/geoip2/libmaxminddb/projects/VS12/libmaxminddb.vcxproj.filters +7 -1
  25. data/ext/geoip2/libmaxminddb/src/Makefile.am +18 -2
  26. data/ext/geoip2/libmaxminddb/src/data-pool.c +180 -0
  27. data/ext/geoip2/libmaxminddb/src/data-pool.h +52 -0
  28. data/ext/geoip2/libmaxminddb/src/maxminddb.c +58 -48
  29. data/ext/geoip2/libmaxminddb/t/Makefile.am +6 -2
  30. data/ext/geoip2/libmaxminddb/t/bad_databases_t.c +1 -0
  31. data/ext/geoip2/libmaxminddb/t/basic_lookup_t.c +35 -0
  32. data/ext/geoip2/libmaxminddb/t/data-pool-t.c +374 -0
  33. data/ext/geoip2/libmaxminddb/t/external_symbols_t.pl +106 -0
  34. data/ext/geoip2/libmaxminddb/t/libtap/.gitignore +13 -0
  35. data/ext/geoip2/libmaxminddb/t/libtap/.travis.yml +13 -0
  36. data/ext/geoip2/libmaxminddb/t/libtap/COPYING +165 -0
  37. data/ext/geoip2/libmaxminddb/t/libtap/INSTALL +41 -0
  38. data/ext/geoip2/libmaxminddb/t/libtap/Makefile +72 -0
  39. data/ext/geoip2/libmaxminddb/t/libtap/Makefile.win +37 -0
  40. data/ext/geoip2/libmaxminddb/t/libtap/README.md +268 -0
  41. data/ext/geoip2/libmaxminddb/t/libtap/t/cmp_mem.c +20 -0
  42. data/ext/geoip2/libmaxminddb/t/libtap/t/cmp_mem.expected +28 -0
  43. data/ext/geoip2/libmaxminddb/t/libtap/t/cmpok.c +16 -0
  44. data/ext/geoip2/libmaxminddb/t/libtap/t/cmpok.expected +37 -0
  45. data/ext/geoip2/libmaxminddb/t/libtap/t/diag.c +10 -0
  46. data/ext/geoip2/libmaxminddb/t/libtap/t/diag.expected +2 -0
  47. data/ext/geoip2/libmaxminddb/t/libtap/t/diesok.c +14 -0
  48. data/ext/geoip2/libmaxminddb/t/libtap/t/diesok.expected +6 -0
  49. data/ext/geoip2/libmaxminddb/t/libtap/t/is.c +24 -0
  50. data/ext/geoip2/libmaxminddb/t/libtap/t/is.expected +58 -0
  51. data/ext/geoip2/libmaxminddb/t/libtap/t/like.c +10 -0
  52. data/ext/geoip2/libmaxminddb/t/libtap/t/like.expected +4 -0
  53. data/ext/geoip2/libmaxminddb/t/libtap/t/simple.c +31 -0
  54. data/ext/geoip2/libmaxminddb/t/libtap/t/simple.expected +32 -0
  55. data/ext/geoip2/libmaxminddb/t/libtap/t/skip.c +23 -0
  56. data/ext/geoip2/libmaxminddb/t/libtap/t/skip.expected +9 -0
  57. data/ext/geoip2/libmaxminddb/t/libtap/t/synopsis.c +13 -0
  58. data/ext/geoip2/libmaxminddb/t/libtap/t/synopsis.expected +9 -0
  59. data/ext/geoip2/libmaxminddb/t/libtap/t/test.c +28 -0
  60. data/ext/geoip2/libmaxminddb/t/libtap/t/todo.c +17 -0
  61. data/ext/geoip2/libmaxminddb/t/libtap/t/todo.expected +11 -0
  62. data/ext/geoip2/libmaxminddb/t/libtap/tap.c +354 -0
  63. data/ext/geoip2/libmaxminddb/t/libtap/tap.h +115 -0
  64. data/ext/geoip2/libmaxminddb/t/maxmind-db/.gitattributes +1 -0
  65. data/ext/geoip2/libmaxminddb/t/maxmind-db/.gitconfig +2 -0
  66. data/ext/geoip2/libmaxminddb/t/maxmind-db/.gitignore +2 -0
  67. data/ext/geoip2/libmaxminddb/t/maxmind-db/.perltidyallrc +11 -0
  68. data/ext/geoip2/libmaxminddb/t/maxmind-db/.tidyallrc +7 -0
  69. data/ext/geoip2/libmaxminddb/t/maxmind-db/LICENSE +4 -0
  70. data/ext/geoip2/libmaxminddb/t/maxmind-db/MaxMind-DB-spec.md +558 -0
  71. data/ext/geoip2/libmaxminddb/t/maxmind-db/README.md +4 -0
  72. data/ext/geoip2/libmaxminddb/t/maxmind-db/bad-data/README.md +7 -0
  73. data/ext/geoip2/libmaxminddb/t/maxmind-db/bad-data/libmaxminddb/libmaxminddb-offset-integer-overflow.mmdb +0 -0
  74. data/ext/geoip2/libmaxminddb/t/maxmind-db/bad-data/maxminddb-golang/cyclic-data-structure.mmdb +0 -0
  75. data/ext/geoip2/libmaxminddb/t/maxmind-db/bad-data/maxminddb-golang/invalid-bytes-length.mmdb +1 -0
  76. data/ext/geoip2/libmaxminddb/t/maxmind-db/bad-data/maxminddb-golang/invalid-data-record-offset.mmdb +0 -0
  77. data/ext/geoip2/libmaxminddb/t/maxmind-db/bad-data/maxminddb-golang/invalid-map-key-length.mmdb +0 -0
  78. data/ext/geoip2/libmaxminddb/t/maxmind-db/bad-data/maxminddb-golang/invalid-string-length.mmdb +1 -0
  79. data/ext/geoip2/libmaxminddb/t/maxmind-db/bad-data/maxminddb-golang/metadata-is-an-uint128.mmdb +1 -0
  80. data/ext/geoip2/libmaxminddb/t/maxmind-db/bad-data/maxminddb-golang/unexpected-bytes.mmdb +0 -0
  81. data/ext/geoip2/libmaxminddb/t/maxmind-db/perltidyrc +12 -0
  82. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoIP2-Anonymous-IP-Test.json +32 -0
  83. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoIP2-City-Test.json +12616 -0
  84. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoIP2-Connection-Type-Test.json +102 -0
  85. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoIP2-Country-Test.json +10975 -0
  86. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoIP2-DensityIncome-Test.json +14 -0
  87. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoIP2-Domain-Test.json +452 -0
  88. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoIP2-Enterprise-Test.json +666 -0
  89. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoIP2-ISP-Test.json +12585 -0
  90. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoIP2-Precision-Enterprise-Test.json +1035 -0
  91. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/GeoLite2-ASN-Test.json +37 -0
  92. data/ext/geoip2/libmaxminddb/t/maxmind-db/source-data/README +13 -0
  93. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-Anonymous-IP-Test.mmdb +0 -0
  94. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb +0 -0
  95. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-City-Test-Invalid-Node-Count.mmdb +0 -0
  96. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-City-Test.mmdb +0 -0
  97. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-Connection-Type-Test.mmdb +0 -0
  98. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-Country-Test.mmdb +0 -0
  99. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-DensityIncome-Test.mmdb +0 -0
  100. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-Domain-Test.mmdb +0 -0
  101. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-Enterprise-Test.mmdb +0 -0
  102. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-ISP-Test.mmdb +0 -0
  103. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoIP2-Precision-Enterprise-Test.mmdb +0 -0
  104. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/GeoLite2-ASN-Test.mmdb +0 -0
  105. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb +0 -0
  106. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-string-value-entries.mmdb +0 -0
  107. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-broken-pointers-24.mmdb +0 -0
  108. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-broken-search-tree-24.mmdb +0 -0
  109. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-decoder.mmdb +0 -0
  110. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-ipv4-24.mmdb +0 -0
  111. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-ipv4-28.mmdb +0 -0
  112. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-ipv4-32.mmdb +0 -0
  113. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-ipv6-24.mmdb +0 -0
  114. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-ipv6-28.mmdb +0 -0
  115. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-ipv6-32.mmdb +0 -0
  116. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-metadata-pointers.mmdb +0 -0
  117. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-mixed-24.mmdb +0 -0
  118. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-mixed-28.mmdb +0 -0
  119. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-mixed-32.mmdb +0 -0
  120. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/MaxMind-DB-test-nested.mmdb +0 -0
  121. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/README.md +26 -0
  122. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/maps-with-pointers.raw +0 -0
  123. data/ext/geoip2/libmaxminddb/t/maxmind-db/test-data/write-test-data.pl +614 -0
  124. data/ext/geoip2/libmaxminddb/t/maxmind-db/tidyall.ini +5 -0
  125. data/geoip2_c.gemspec +2 -3
  126. data/lib/geoip2/database.rb +4 -0
  127. data/lib/geoip2/version.rb +1 -1
  128. metadata +114 -22
  129. data/.travis.yml +0 -31
  130. data/Appraisals +0 -7
  131. data/gemfiles/ruby_2.1.gemfile +0 -7
  132. data/gemfiles/ruby_2.2.gemfile +0 -7
@@ -0,0 +1,614 @@
1
+ #!/usr/bin/env perl
2
+
3
+ use strict;
4
+ use warnings;
5
+ use autodie;
6
+ use utf8;
7
+
8
+ use Carp qw( croak );
9
+ use Cwd qw( abs_path );
10
+ use File::Basename qw( dirname );
11
+ use File::Slurp qw( read_file write_file );
12
+ use JSON::XS qw( decode_json );
13
+ use Math::Int128 qw( uint128 );
14
+ use MaxMind::DB::Writer::Serializer 0.100004;
15
+ use MaxMind::DB::Writer::Tree 0.100004;
16
+ use MaxMind::DB::Writer::Util qw( key_for_data );
17
+ use Net::Works::Network;
18
+ use Test::MaxMind::DB::Common::Util qw( standard_test_metadata );
19
+
20
+ my $Dir = dirname( abs_path($0) );
21
+
22
+ sub main {
23
+ my @sizes = ( 24, 28, 32 );
24
+ my @ipv4_range = ( '1.1.1.1', '1.1.1.32' );
25
+
26
+ my @ipv4_subnets = Net::Works::Network->range_as_subnets(@ipv4_range);
27
+ for my $record_size (@sizes) {
28
+ write_test_db(
29
+ $record_size,
30
+ \@ipv4_subnets,
31
+ { ip_version => 4 },
32
+ 'ipv4',
33
+ );
34
+ }
35
+
36
+ write_broken_pointers_test_db(
37
+ 24,
38
+ \@ipv4_subnets,
39
+ { ip_version => 4 },
40
+ 'broken-pointers',
41
+ );
42
+
43
+ write_broken_search_tree_db(
44
+ 24,
45
+ \@ipv4_subnets,
46
+ { ip_version => 4 },
47
+ 'broken-search-tree',
48
+ );
49
+
50
+ my @ipv6_subnets = Net::Works::Network->range_as_subnets(
51
+ '::1:ffff:ffff',
52
+ '::2:0000:0059'
53
+ );
54
+
55
+ for my $record_size (@sizes) {
56
+ write_test_db(
57
+ $record_size,
58
+ \@ipv6_subnets,
59
+ { ip_version => 6 },
60
+ 'ipv6',
61
+ );
62
+
63
+ write_test_db(
64
+ $record_size,
65
+ [
66
+ @ipv6_subnets,
67
+ Net::Works::Network->range_as_subnets( @ipv4_range, 6 ),
68
+ ],
69
+ { ip_version => 6 },
70
+ 'mixed',
71
+ );
72
+ }
73
+
74
+ write_decoder_test_db();
75
+ write_deeply_nested_structures_db();
76
+
77
+ write_geoip2_dbs();
78
+ write_broken_geoip2_city_db();
79
+ write_invalid_node_count();
80
+
81
+ write_no_ipv4_tree_db();
82
+
83
+ write_no_map_db( \@ipv4_subnets );
84
+
85
+ write_test_serialization_data();
86
+
87
+ write_db_with_metadata_pointers();
88
+ }
89
+
90
+ sub write_broken_pointers_test_db {
91
+ no warnings 'redefine';
92
+
93
+ my $orig_store_data = MaxMind::DB::Writer::Serializer->can('store_data');
94
+
95
+ # This breaks the value of the record for the 1.1.1.32 network, causing it
96
+ # to point outside the database.
97
+ local *MaxMind::DB::Writer::Serializer::store_data = sub {
98
+ my $data_pointer = shift->$orig_store_data(@_);
99
+ my $value = $_[1];
100
+ if ( ref($value) eq 'HASH'
101
+ && exists $value->{ip}
102
+ && $value->{ip} eq '1.1.1.32' ) {
103
+
104
+ $data_pointer += 100_000;
105
+ }
106
+ return $data_pointer;
107
+ };
108
+
109
+ # The next hack will poison the data section for the 1.1.16/28 subnet
110
+ # value. It's value will be a pointer that resolves to an offset outside
111
+ # the database.
112
+
113
+ my $key_to_poison = key_for_data( { ip => '1.1.1.16' } );
114
+
115
+ my $orig_position_for_data
116
+ = MaxMind::DB::Writer::Serializer->can('_position_for_data');
117
+ local *MaxMind::DB::Writer::Serializer::_position_for_data = sub {
118
+ my $key = $_[1];
119
+
120
+ if ( $key eq $key_to_poison ) {
121
+ return 1_000_000;
122
+ }
123
+ else {
124
+ return shift->$orig_position_for_data(@_);
125
+ }
126
+ };
127
+
128
+ write_test_db(@_);
129
+
130
+ return;
131
+ }
132
+
133
+ sub write_broken_search_tree_db {
134
+ my $filename = ( write_test_db(@_) )[1];
135
+
136
+ my $content = read_file( $filename, { binmode => ':raw' } );
137
+
138
+ # This causes the right record of the first node to be 0, meaning it
139
+ # points back to the top of the tree. This should never happen in a
140
+ # database that follows the spec.
141
+ substr( $content, 5, 1 ) = "\0";
142
+ write_file( $filename, $content, { binmode => ':raw' } );
143
+
144
+ return;
145
+ }
146
+
147
+ sub write_test_db {
148
+ my $record_size = shift;
149
+ my $subnets = shift;
150
+ my $metadata = shift;
151
+ my $ip_version_name = shift;
152
+
153
+ my $writer = MaxMind::DB::Writer::Tree->new(
154
+ ip_version => $subnets->[0]->version(),
155
+ record_size => $record_size,
156
+ alias_ipv6_to_ipv4 => ( $subnets->[0]->version() == 6 ? 1 : 0 ),
157
+ map_key_type_callback => sub { 'utf8_string' },
158
+ standard_test_metadata(),
159
+ %{$metadata},
160
+ );
161
+
162
+ for my $subnet ( @{$subnets} ) {
163
+ $writer->insert_network(
164
+ $subnet,
165
+ { ip => $subnet->first()->as_string() }
166
+ );
167
+ }
168
+
169
+ my $filename = sprintf(
170
+ "$Dir/MaxMind-DB-test-%s-%i.mmdb",
171
+ $ip_version_name,
172
+ $record_size,
173
+ );
174
+ open my $fh, '>', $filename;
175
+
176
+ $writer->write_tree($fh);
177
+
178
+ close $fh;
179
+
180
+ return ( $writer, $filename );
181
+ }
182
+
183
+ {
184
+ # We will store this once for each subnet so we will also be testing
185
+ # pointers, since the serializer will generate a pointer to this
186
+ # structure.
187
+ my %all_types = (
188
+ utf8_string => 'unicode! ☯ - ♫',
189
+ double => 42.123456,
190
+ bytes => pack( 'N', 42 ),
191
+ uint16 => 100,
192
+ uint32 => 2**28,
193
+ int32 => -1 * ( 2**28 ),
194
+ uint64 => uint128(1) << 60,
195
+ uint128 => uint128(1) << 120,
196
+ array => [ 1, 2, 3, ],
197
+ map => {
198
+ mapX => {
199
+ utf8_stringX => 'hello',
200
+ arrayX => [ 7, 8, 9 ],
201
+ },
202
+ },
203
+ boolean => 1,
204
+ float => 1.1,
205
+ );
206
+
207
+ my %all_types_0 = (
208
+ utf8_string => q{},
209
+ double => 0,
210
+ bytes => q{},
211
+ uint16 => 0,
212
+ uint32 => 0,
213
+ int32 => 0,
214
+ uint64 => uint128(0),
215
+ uint128 => uint128(0),
216
+ array => [],
217
+ map => {},
218
+ boolean => 0,
219
+ float => 0,
220
+ );
221
+
222
+ sub write_decoder_test_db {
223
+ my $writer = MaxMind::DB::Writer::Tree->new(
224
+ ip_version => 6,
225
+ record_size => 24,
226
+ database_type => 'MaxMind DB Decoder Test',
227
+ languages => ['en'],
228
+ description => {
229
+ en =>
230
+ 'MaxMind DB Decoder Test database - contains every MaxMind DB data type',
231
+ },
232
+ alias_ipv6_to_ipv4 => 1,
233
+ remove_reserved_networks => 0,
234
+ map_key_type_callback => sub {
235
+ my $key = $_[0];
236
+ $key =~ s/X$//;
237
+ return $key eq 'array' ? [ 'array', 'uint32' ] : $key;
238
+ },
239
+ );
240
+
241
+ my @subnets
242
+ = map { Net::Works::Network->new_from_string( string => $_ ) }
243
+ qw(
244
+ ::1.1.1.0/120
245
+ ::2.2.0.0/112
246
+ ::3.0.0.0/104
247
+ ::4.5.6.7/128
248
+ abcd::/64
249
+ 1000::1234:0000/112
250
+ );
251
+
252
+ for my $subnet (@subnets) {
253
+ $writer->insert_network(
254
+ $subnet,
255
+ \%all_types,
256
+ );
257
+ }
258
+
259
+ $writer->insert_network(
260
+ Net::Works::Network->new_from_string( string => '::0.0.0.0/128' ),
261
+ \%all_types_0,
262
+ );
263
+
264
+ open my $fh, '>', "$Dir/MaxMind-DB-test-decoder.mmdb";
265
+ $writer->write_tree($fh);
266
+ close $fh;
267
+
268
+ return;
269
+ }
270
+ }
271
+
272
+ {
273
+ my %nested = (
274
+ map1 => {
275
+ map2 => {
276
+ array => [
277
+ {
278
+ map3 => { a => 1, b => 2, c => 3 },
279
+ },
280
+ ],
281
+ },
282
+ },
283
+ );
284
+
285
+ sub write_deeply_nested_structures_db {
286
+ my $writer = MaxMind::DB::Writer::Tree->new(
287
+ ip_version => 6,
288
+ record_size => 24,
289
+ ip_version => 6,
290
+ database_type => 'MaxMind DB Nested Data Structures',
291
+ languages => ['en'],
292
+ description => {
293
+ en =>
294
+ 'MaxMind DB Nested Data Structures Test database - contains deeply nested map/array structures',
295
+ },
296
+ alias_ipv6_to_ipv4 => 1,
297
+ map_key_type_callback => sub {
298
+ my $key = shift;
299
+ return
300
+ $key =~ /^map/ ? 'map'
301
+ : $key eq 'array' ? [ 'array', 'map' ]
302
+ : 'uint32';
303
+ }
304
+ );
305
+
306
+ my @subnets
307
+ = map { Net::Works::Network->new_from_string( string => $_ ) }
308
+ qw(
309
+ ::1.1.1.0/120
310
+ ::2.2.0.0/112
311
+ ::3.0.0.0/104
312
+ ::4.5.6.7/128
313
+ abcd::/64
314
+ 1000::1234:0000/112
315
+ );
316
+
317
+ for my $subnet (@subnets) {
318
+ $writer->insert_network(
319
+ $subnet,
320
+ \%nested,
321
+ );
322
+ }
323
+
324
+ open my $fh, '>', "$Dir/MaxMind-DB-test-nested.mmdb";
325
+ $writer->write_tree($fh);
326
+ close $fh;
327
+
328
+ return;
329
+ }
330
+ }
331
+
332
+ sub write_geoip2_dbs {
333
+ _write_geoip2_db( @{$_}, 'Test' )
334
+ for (
335
+ [ 'GeoIP2-Anonymous-IP', 1 ],
336
+ [ 'GeoIP2-City', 0 ],
337
+ [ 'GeoIP2-Connection-Type', 0 ],
338
+ [ 'GeoIP2-Country', 0 ],
339
+ [ 'GeoIP2-DensityIncome', 0 ],
340
+ [ 'GeoIP2-Domain', 0 ],
341
+ [ 'GeoIP2-Enterprise', 0 ],
342
+ [ 'GeoIP2-ISP', 0 ],
343
+ [ 'GeoIP2-Precision-Enterprise', 0 ],
344
+ [ 'GeoLite2-ASN', 0 ],
345
+ );
346
+ }
347
+
348
+ sub write_broken_geoip2_city_db {
349
+ no warnings 'redefine';
350
+
351
+ # This is how we _used_ to encode doubles. Storing them this way with the
352
+ # current reader tools can lead to weird errors. This broken database is a
353
+ # good way to test the robustness of reader code in the face of broken
354
+ # databases.
355
+ local *MaxMind::DB::Writer::Serializer::_encode_double = sub {
356
+ my $self = shift;
357
+ my $value = shift;
358
+
359
+ $self->_simple_encode( double => $value );
360
+ };
361
+
362
+ _write_geoip2_db( 'GeoIP2-City', 0, 'Test Broken Double Format' );
363
+ }
364
+
365
+ sub write_invalid_node_count {
366
+ no warnings 'redefine';
367
+ local *MaxMind::DB::Writer::Tree::node_count = sub { 100000 };
368
+
369
+ _write_geoip2_db( 'GeoIP2-City', 0, 'Test Invalid Node Count' );
370
+ }
371
+
372
+ sub _universal_map_key_type_callback {
373
+ my $map = {
374
+
375
+ # languages
376
+ de => 'utf8_string',
377
+ en => 'utf8_string',
378
+ es => 'utf8_string',
379
+ fr => 'utf8_string',
380
+ ja => 'utf8_string',
381
+ 'pt-BR' => 'utf8_string',
382
+ ru => 'utf8_string',
383
+ 'zh-CN' => 'utf8_string',
384
+
385
+ # production
386
+ accuracy_radius => 'uint16',
387
+ autonomous_system_number => 'uint32',
388
+ autonomous_system_organization => 'utf8_string',
389
+ average_income => 'uint32',
390
+ city => 'map',
391
+ code => 'utf8_string',
392
+ confidence => 'uint16',
393
+ connection_type => 'utf8_string',
394
+ continent => 'map',
395
+ country => 'map',
396
+ domain => 'utf8_string',
397
+ geoname_id => 'uint32',
398
+ is_anonymous => 'boolean',
399
+ is_anonymous_proxy => 'boolean',
400
+ is_anonymous_vpn => 'boolean',
401
+ is_hosting_provider => 'boolean',
402
+ is_legitimate_proxy => 'boolean',
403
+ is_public_proxy => 'boolean',
404
+ is_satellite_provider => 'boolean',
405
+ is_tor_exit_node => 'boolean',
406
+ iso_code => 'utf8_string',
407
+ isp => 'utf8_string',
408
+ latitude => 'double',
409
+ location => 'map',
410
+ longitude => 'double',
411
+ metro_code => 'uint16',
412
+ names => 'map',
413
+ organization => 'utf8_string',
414
+ population_density => 'uint32',
415
+ postal => 'map',
416
+ registered_country => 'map',
417
+ represented_country => 'map',
418
+ subdivisions => [ 'array', 'map' ],
419
+ time_zone => 'utf8_string',
420
+ traits => 'map',
421
+ traits => 'map',
422
+ type => 'utf8_string',
423
+ user_type => 'utf8_string',
424
+
425
+ # for testing only
426
+ foo => 'utf8_string',
427
+ bar => 'utf8_string',
428
+ buzz => 'utf8_string',
429
+ our_value => 'utf8_string',
430
+ };
431
+
432
+ my $callback = sub {
433
+ my $key = shift;
434
+
435
+ return $map->{$key} || die <<"ERROR";
436
+ Unknown tree key '$key'.
437
+
438
+ The universal_map_key_type_callback doesn't know what type to use for the passed
439
+ key. If you are adding a new key that will be used in a frozen tree / mmdb then
440
+ you should update the mapping in both our internal code and here.
441
+ ERROR
442
+ };
443
+
444
+ return $callback;
445
+ }
446
+
447
+ sub _write_geoip2_db {
448
+ my $type = shift;
449
+ my $populate_all_networks = shift;
450
+ my $description = shift;
451
+
452
+ my $writer = MaxMind::DB::Writer::Tree->new(
453
+ ip_version => 6,
454
+ record_size => 28,
455
+ ip_version => 6,
456
+ database_type => $type,
457
+ languages => [ 'en', $type eq 'GeoIP2-City' ? ('zh') : () ],
458
+ description => {
459
+ en => ( $type =~ s/-/ /gr )
460
+ . " $description Database (fake GeoIP2 data, for example purposes only)",
461
+ $type eq 'GeoIP2-City' ? ( zh => '小型数据库' ) : (),
462
+ },
463
+ alias_ipv6_to_ipv4 => 1,
464
+ map_key_type_callback => _universal_map_key_type_callback(),
465
+ );
466
+
467
+ _populate_all_networks($writer) if $populate_all_networks;
468
+
469
+ my $nodes = decode_json(
470
+ read_file(
471
+ "$Dir/../source-data/$type-Test.json",
472
+ binmode => ':raw'
473
+ )
474
+ );
475
+
476
+ for my $node (@$nodes) {
477
+ for my $network ( keys %$node ) {
478
+ $writer->insert_network(
479
+ Net::Works::Network->new_from_string( string => $network ),
480
+ $node->{$network}
481
+ );
482
+ }
483
+ }
484
+
485
+ my $suffix = $description =~ s/ /-/gr;
486
+ open my $output_fh, '>', "$Dir/$type-$suffix.mmdb";
487
+ $writer->write_tree($output_fh);
488
+ close $output_fh;
489
+
490
+ return;
491
+ }
492
+
493
+ sub _populate_all_networks {
494
+ my $writer = shift;
495
+
496
+ my $max_uint128 = uint128(0) - 1;
497
+ my @networks = Net::Works::Network->range_as_subnets(
498
+ Net::Works::Address->new_from_integer(
499
+ integer => 0,
500
+ version => 6,
501
+ ),
502
+ Net::Works::Address->new_from_integer(
503
+ integer => $max_uint128,
504
+ version => 6,
505
+ ),
506
+ );
507
+
508
+ for my $network (@networks) {
509
+ $writer->insert_network( $network => {} );
510
+ }
511
+ }
512
+
513
+ sub write_no_ipv4_tree_db {
514
+ my $subnets = shift;
515
+
516
+ my $writer = MaxMind::DB::Writer::Tree->new(
517
+ ip_version => 6,
518
+ record_size => 24,
519
+ ip_version => 6,
520
+ database_type => 'MaxMind DB No IPv4 Search Tree',
521
+ languages => ['en'],
522
+ description => {
523
+ en => 'MaxMind DB No IPv4 Search Tree',
524
+ },
525
+ remove_reserved_networks => 0,
526
+ root_data_type => 'utf8_string',
527
+ map_key_type_callback => sub { {} },
528
+ );
529
+
530
+ my $subnet = Net::Works::Network->new_from_string( string => '::/64' );
531
+ $writer->insert_network( $subnet, $subnet->as_string() );
532
+
533
+ open my $output_fh, '>', "$Dir/MaxMind-DB-no-ipv4-search-tree.mmdb";
534
+ $writer->write_tree($output_fh);
535
+ close $output_fh;
536
+
537
+ return;
538
+ }
539
+
540
+ # The point of this database is to provide something where we can test looking
541
+ # up a single value. In other words, each IP address points to a non-compound
542
+ # value, a string rather than a map or array.
543
+ sub write_no_map_db {
544
+ my $subnets = shift;
545
+
546
+ my $writer = MaxMind::DB::Writer::Tree->new(
547
+ ip_version => 4,
548
+ record_size => 24,
549
+ database_type => 'MaxMind DB String Value Entries',
550
+ languages => ['en'],
551
+ description => {
552
+ en =>
553
+ 'MaxMind DB String Value Entries (no maps or arrays as values)',
554
+ },
555
+ root_data_type => 'utf8_string',
556
+ map_key_type_callback => sub { {} },
557
+ );
558
+
559
+ for my $subnet ( @{$subnets} ) {
560
+ $writer->insert_network( $subnet, $subnet->as_string() );
561
+ }
562
+
563
+ open my $output_fh, '>', "$Dir/MaxMind-DB-string-value-entries.mmdb";
564
+ $writer->write_tree($output_fh);
565
+ close $output_fh;
566
+
567
+ return;
568
+ }
569
+
570
+ sub write_test_serialization_data {
571
+ my $serializer = MaxMind::DB::Writer::Serializer->new(
572
+ map_key_type_callback => sub { 'utf8_string' } );
573
+
574
+ $serializer->store_data( map => { long_key => 'long_value1' } );
575
+ $serializer->store_data( map => { long_key => 'long_value2' } );
576
+ $serializer->store_data( map => { long_key2 => 'long_value1' } );
577
+ $serializer->store_data( map => { long_key2 => 'long_value2' } );
578
+ $serializer->store_data( map => { long_key => 'long_value1' } );
579
+ $serializer->store_data( map => { long_key2 => 'long_value2' } );
580
+
581
+ open my $fh, '>', 'maps-with-pointers.raw';
582
+ print {$fh} ${ $serializer->buffer() }
583
+ or die "Cannot write to maps-with-pointers.raw: $!";
584
+ close $fh;
585
+
586
+ return;
587
+ }
588
+
589
+ sub write_db_with_metadata_pointers {
590
+ my $repeated_string = 'Lots of pointers in metadata';
591
+ my $writer = MaxMind::DB::Writer::Tree->new(
592
+ ip_version => 6,
593
+ record_size => 24,
594
+ map_key_type_callback => sub { 'utf8_string' },
595
+ database_type => $repeated_string,
596
+ languages => [ 'en', 'es', 'zh' ],
597
+ description => {
598
+ en => $repeated_string,
599
+ es => $repeated_string,
600
+ zh => $repeated_string,
601
+ },
602
+
603
+ );
604
+
605
+ _populate_all_networks($writer);
606
+
607
+ open my $fh, '>', 'MaxMind-DB-test-metadata-pointers.mmdb';
608
+
609
+ $writer->write_tree($fh);
610
+
611
+ close $fh;
612
+ }
613
+
614
+ main();
@@ -0,0 +1,5 @@
1
+ [PerlTidy]
2
+ select = **/*.{pl,pm,t}
3
+
4
+ [JSON]
5
+ select = **/*.json
data/geoip2_c.gemspec CHANGED
@@ -1,4 +1,3 @@
1
- # coding: utf-8
2
1
  lib = File.expand_path('../lib', __FILE__)
3
2
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
3
  require 'geoip2/version'
@@ -23,8 +22,8 @@ Gem::Specification.new do |spec|
23
22
  spec.extensions = ["ext/geoip2/extconf.rb"]
24
23
 
25
24
  spec.add_development_dependency "appraisal"
26
- spec.add_development_dependency "bundler", "~> 1.13"
27
- spec.add_development_dependency "rake", "~> 11.0"
25
+ spec.add_development_dependency "bundler"
26
+ spec.add_development_dependency "rake"
28
27
  spec.add_development_dependency "rake-compiler"
29
28
  spec.add_development_dependency "test-unit", "~> 3.2"
30
29
  end
@@ -1,4 +1,8 @@
1
1
  module GeoIP2
2
2
  class Database
3
+ def initialize(path, symbolize_keys: false)
4
+ @symbolize_keys = !!symbolize_keys
5
+ open_mmdb(path)
6
+ end
3
7
  end
4
8
  end
@@ -1,3 +1,3 @@
1
1
  module GeoIP2
2
- VERSION = "0.3.2"
2
+ VERSION = "0.3.4"
3
3
  end