maxmind-geoip2 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/README.md +10 -5
- data/lib/maxmind/geoip2/client.rb +18 -9
- data/lib/maxmind/geoip2/model/city.rb +4 -5
- data/lib/maxmind/geoip2/model/connection_type.rb +3 -2
- data/lib/maxmind/geoip2/model/enterprise.rb +2 -3
- data/lib/maxmind/geoip2/model/insights.rb +3 -5
- data/lib/maxmind/geoip2/record/traits.rb +39 -27
- data/lib/maxmind/geoip2/version.rb +8 -0
- data/maxmind-geoip2.gemspec +7 -2
- data/test/data/cmd/write-test-data/main.go +68 -0
- data/test/data/go.mod +13 -0
- data/test/data/go.sum +16 -0
- data/test/data/perltidyrc +6 -0
- data/test/data/pkg/writer/decoder.go +178 -0
- data/test/data/pkg/writer/geoip2.go +182 -0
- data/test/data/pkg/writer/ip.go +39 -0
- data/test/data/pkg/writer/maxmind.go +245 -0
- data/test/data/pkg/writer/nestedstructures.go +73 -0
- data/test/data/pkg/writer/writer.go +58 -0
- data/test/data/source-data/GeoIP2-City-Test.json +322 -1
- data/test/data/source-data/GeoIP2-Connection-Type-Test.json +15 -10
- data/test/data/source-data/GeoIP2-Country-Test.json +99 -0
- data/test/data/source-data/GeoIP2-Domain-Test.json +5 -0
- data/test/data/source-data/GeoIP2-Enterprise-Test.json +347 -1
- data/test/data/source-data/GeoIP2-Precision-Enterprise-Sandbox-Test.json +296 -0
- data/test/data/source-data/GeoIP2-Precision-Enterprise-Test.json +412 -2
- data/test/data/source-data/GeoIP2-Static-IP-Score-Test.json +15 -0
- data/test/data/source-data/GeoIP2-User-Count-Test.json +18 -0
- data/test/data/source-data/GeoLite2-City-Test.json +168 -0
- data/test/data/source-data/GeoLite2-Country-Test.json +92 -0
- data/test/data/test-data/GeoIP2-Anonymous-IP-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-City-Test-Broken-Double-Format.mmdb +0 -0
- data/test/data/test-data/GeoIP2-City-Test-Invalid-Node-Count.mmdb +0 -0
- data/test/data/test-data/GeoIP2-City-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-Connection-Type-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-Country-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-DensityIncome-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-Domain-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-Enterprise-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-ISP-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-Precision-Enterprise-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-Static-IP-Score-Test.mmdb +0 -0
- data/test/data/test-data/GeoIP2-User-Count-Test.mmdb +0 -0
- data/test/data/test-data/GeoLite2-ASN-Test.mmdb +0 -0
- data/test/data/test-data/GeoLite2-City-Test.mmdb +0 -0
- data/test/data/test-data/GeoLite2-Country-Test.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-no-ipv4-search-tree.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-string-value-entries.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-broken-pointers-24.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-broken-search-tree-24.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-decoder.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-ipv4-24.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-ipv4-28.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-ipv4-32.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-ipv6-24.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-ipv6-28.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-ipv6-32.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-metadata-pointers.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-mixed-24.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-mixed-28.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-mixed-32.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-nested.mmdb +0 -0
- data/test/data/test-data/MaxMind-DB-test-pointer-decoder.mmdb +0 -0
- data/test/data/test-data/README.md +28 -12
- data/test/test_client.rb +18 -2
- data/test/test_reader.rb +19 -1
- metadata +16 -7
- data/test/data/source-data/README +0 -15
- data/test/data/test-data/write-test-data.pl +0 -695
@@ -0,0 +1,178 @@
|
|
1
|
+
package writer
|
2
|
+
|
3
|
+
import (
|
4
|
+
"bytes"
|
5
|
+
"encoding/binary"
|
6
|
+
"fmt"
|
7
|
+
"math"
|
8
|
+
"math/big"
|
9
|
+
"net/netip"
|
10
|
+
|
11
|
+
"github.com/maxmind/mmdbwriter"
|
12
|
+
"github.com/maxmind/mmdbwriter/mmdbtype"
|
13
|
+
"go4.org/netipx"
|
14
|
+
)
|
15
|
+
|
16
|
+
// WriteDecoderTestDB writes an mmdb file with all possible record value types.
|
17
|
+
func (w *Writer) WriteDecoderTestDB() error {
|
18
|
+
dbWriter, err := mmdbwriter.New(
|
19
|
+
mmdbwriter.Options{
|
20
|
+
DatabaseType: "MaxMind DB Decoder Test",
|
21
|
+
Description: map[string]string{
|
22
|
+
"en": "MaxMind DB Decoder Test database - contains every MaxMind DB data type",
|
23
|
+
},
|
24
|
+
DisableIPv4Aliasing: false,
|
25
|
+
IncludeReservedNetworks: true,
|
26
|
+
IPVersion: 6,
|
27
|
+
Languages: []string{"en"},
|
28
|
+
RecordSize: 24,
|
29
|
+
},
|
30
|
+
)
|
31
|
+
if err != nil {
|
32
|
+
return fmt.Errorf("creating mmdbwriter: %w", err)
|
33
|
+
}
|
34
|
+
|
35
|
+
addrs, err := parseIPSlice(ipSample)
|
36
|
+
if err != nil {
|
37
|
+
return fmt.Errorf("parsing ip addresses: %w", err)
|
38
|
+
}
|
39
|
+
if err := insertAllTypes(dbWriter, addrs); err != nil {
|
40
|
+
return fmt.Errorf("inserting all types records: %w", err)
|
41
|
+
}
|
42
|
+
|
43
|
+
zeroAddr, err := netip.ParsePrefix("::0.0.0.0/128")
|
44
|
+
if err != nil {
|
45
|
+
return fmt.Errorf("parsing ip: %w", err)
|
46
|
+
}
|
47
|
+
if err := insertAllTypesZero(dbWriter, []netip.Prefix{zeroAddr}); err != nil {
|
48
|
+
return fmt.Errorf("inserting all types records: %w", err)
|
49
|
+
}
|
50
|
+
|
51
|
+
maxAddr, err := netip.ParsePrefix("::255.255.255.255/128")
|
52
|
+
if err != nil {
|
53
|
+
return fmt.Errorf("parsing ip: %w", err)
|
54
|
+
}
|
55
|
+
if err := insertNumericMax(dbWriter, []netip.Prefix{maxAddr}); err != nil {
|
56
|
+
return fmt.Errorf("inserting all types records: %w", err)
|
57
|
+
}
|
58
|
+
|
59
|
+
if err := w.write(dbWriter, "MaxMind-DB-test-decoder.mmdb"); err != nil {
|
60
|
+
return fmt.Errorf("writing database: %w", err)
|
61
|
+
}
|
62
|
+
return nil
|
63
|
+
}
|
64
|
+
|
65
|
+
// insertAllTypes inserts records with all possible value types.
|
66
|
+
func insertAllTypes(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
|
67
|
+
buf := new(bytes.Buffer)
|
68
|
+
if err := binary.Write(buf, binary.BigEndian, uint32(42)); err != nil {
|
69
|
+
return fmt.Errorf("creating buffer for all types record: %w", err)
|
70
|
+
}
|
71
|
+
|
72
|
+
ui64 := big.Int{}
|
73
|
+
ui64.Lsh(big.NewInt(1), 60)
|
74
|
+
|
75
|
+
ui128 := big.Int{}
|
76
|
+
ui128.Lsh(big.NewInt(1), 120)
|
77
|
+
mmdbUint128 := mmdbtype.Uint128(ui128)
|
78
|
+
|
79
|
+
allTypes := mmdbtype.Map{
|
80
|
+
"array": mmdbtype.Slice{
|
81
|
+
mmdbtype.Uint32(1),
|
82
|
+
mmdbtype.Uint32(2),
|
83
|
+
mmdbtype.Uint32(3),
|
84
|
+
},
|
85
|
+
"bytes": mmdbtype.Bytes(buf.Bytes()),
|
86
|
+
"boolean": mmdbtype.Bool(true),
|
87
|
+
"double": mmdbtype.Float64(42.123456),
|
88
|
+
"float": mmdbtype.Float32(1.1),
|
89
|
+
"int32": mmdbtype.Int32(-1 * math.Pow(2, 28)),
|
90
|
+
"map": mmdbtype.Map{
|
91
|
+
"mapX": mmdbtype.Map{
|
92
|
+
"utf8_stringX": mmdbtype.String("hello"),
|
93
|
+
"arrayX": mmdbtype.Slice{
|
94
|
+
mmdbtype.Uint32(7),
|
95
|
+
mmdbtype.Uint32(8),
|
96
|
+
mmdbtype.Uint32(9),
|
97
|
+
},
|
98
|
+
},
|
99
|
+
},
|
100
|
+
"uint16": mmdbtype.Uint16(100),
|
101
|
+
"uint32": mmdbtype.Uint32(math.Pow(2, 28)),
|
102
|
+
"uint64": mmdbtype.Uint64(ui64.Uint64()),
|
103
|
+
"uint128": mmdbUint128.Copy(),
|
104
|
+
"utf8_string": mmdbtype.String("unicode! ☯ - ♫"),
|
105
|
+
}
|
106
|
+
|
107
|
+
for _, addr := range ipAddresses {
|
108
|
+
err := w.Insert(
|
109
|
+
netipx.PrefixIPNet(addr),
|
110
|
+
allTypes,
|
111
|
+
)
|
112
|
+
if err != nil {
|
113
|
+
return fmt.Errorf("inserting ip: %w", err)
|
114
|
+
}
|
115
|
+
}
|
116
|
+
return nil
|
117
|
+
}
|
118
|
+
|
119
|
+
// insertAllTypesZero inserts records with all possible value types with zero values.
|
120
|
+
func insertAllTypesZero(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
|
121
|
+
var uint128 big.Int
|
122
|
+
mmdbUint128 := mmdbtype.Uint128(uint128)
|
123
|
+
|
124
|
+
zeroValues := mmdbtype.Map{
|
125
|
+
"array": mmdbtype.Slice{},
|
126
|
+
"bytes": mmdbtype.Bytes([]byte{}),
|
127
|
+
"boolean": mmdbtype.Bool(false),
|
128
|
+
"double": mmdbtype.Float64(0),
|
129
|
+
"float": mmdbtype.Float32(0),
|
130
|
+
"int32": mmdbtype.Int32(0),
|
131
|
+
"map": mmdbtype.Map{},
|
132
|
+
"uint16": mmdbtype.Uint16(0),
|
133
|
+
"uint32": mmdbtype.Uint32(0),
|
134
|
+
"uint64": mmdbtype.Uint64(0),
|
135
|
+
"uint128": mmdbUint128.Copy(),
|
136
|
+
"utf8_string": mmdbtype.String(""),
|
137
|
+
}
|
138
|
+
|
139
|
+
for _, addr := range ipAddresses {
|
140
|
+
err := w.Insert(
|
141
|
+
netipx.PrefixIPNet(addr),
|
142
|
+
zeroValues,
|
143
|
+
)
|
144
|
+
if err != nil {
|
145
|
+
return fmt.Errorf("inserting ip: %w", err)
|
146
|
+
}
|
147
|
+
}
|
148
|
+
return nil
|
149
|
+
}
|
150
|
+
|
151
|
+
// insertNumericMax inserts records with numeric types maxed out.
|
152
|
+
func insertNumericMax(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
|
153
|
+
var uint128Max big.Int
|
154
|
+
uint128Max.Exp(big.NewInt(2), big.NewInt(128), nil)
|
155
|
+
uint128Max.Sub(&uint128Max, big.NewInt(1))
|
156
|
+
mmdbUint128 := mmdbtype.Uint128(uint128Max)
|
157
|
+
|
158
|
+
numMax := mmdbtype.Map{
|
159
|
+
"double": mmdbtype.Float64(math.Inf(1)),
|
160
|
+
"float": mmdbtype.Float32(float32(math.Inf(1))),
|
161
|
+
"int32": mmdbtype.Int32(1<<31 - 1),
|
162
|
+
"uint16": mmdbtype.Uint16(0xffff),
|
163
|
+
"uint32": mmdbtype.Uint32(0xffffffff),
|
164
|
+
"uint64": mmdbtype.Uint64(0xffffffffffffffff),
|
165
|
+
"uint128": mmdbUint128.Copy(),
|
166
|
+
}
|
167
|
+
|
168
|
+
for _, addr := range ipAddresses {
|
169
|
+
err := w.Insert(
|
170
|
+
netipx.PrefixIPNet(addr),
|
171
|
+
numMax,
|
172
|
+
)
|
173
|
+
if err != nil {
|
174
|
+
return fmt.Errorf("inserting ip: %w", err)
|
175
|
+
}
|
176
|
+
}
|
177
|
+
return nil
|
178
|
+
}
|
@@ -0,0 +1,182 @@
|
|
1
|
+
package writer
|
2
|
+
|
3
|
+
import (
|
4
|
+
"encoding/json"
|
5
|
+
"fmt"
|
6
|
+
"net/netip"
|
7
|
+
"os"
|
8
|
+
"path/filepath"
|
9
|
+
"strings"
|
10
|
+
|
11
|
+
"github.com/maxmind/mmdbwriter"
|
12
|
+
"github.com/maxmind/mmdbwriter/mmdbtype"
|
13
|
+
"go4.org/netipx"
|
14
|
+
)
|
15
|
+
|
16
|
+
// WriteGeoIP2TestDB writes GeoIP2 test mmdb files.
|
17
|
+
func (w *Writer) WriteGeoIP2TestDB() error {
|
18
|
+
dbTypes := []string{
|
19
|
+
"GeoIP2-Anonymous-IP",
|
20
|
+
"GeoIP2-City",
|
21
|
+
"GeoIP2-Connection-Type",
|
22
|
+
"GeoIP2-Country",
|
23
|
+
"GeoIP2-DensityIncome",
|
24
|
+
"GeoIP2-Domain",
|
25
|
+
"GeoIP2-Enterprise",
|
26
|
+
"GeoIP2-ISP",
|
27
|
+
"GeoIP2-Precision-Enterprise",
|
28
|
+
"GeoIP2-Static-IP-Score",
|
29
|
+
"GeoIP2-User-Count",
|
30
|
+
"GeoLite2-ASN",
|
31
|
+
"GeoLite2-City",
|
32
|
+
"GeoLite2-Country",
|
33
|
+
}
|
34
|
+
|
35
|
+
for _, dbType := range dbTypes {
|
36
|
+
languages := []string{"en"}
|
37
|
+
description := map[string]string{
|
38
|
+
"en": strings.ReplaceAll(dbType, "-", " ") +
|
39
|
+
" Test Database (fake GeoIP2 data, for example purposes only)",
|
40
|
+
}
|
41
|
+
|
42
|
+
if dbType == "GeoIP2-City" {
|
43
|
+
languages = append(languages, "zh")
|
44
|
+
description["zh"] = "小型数据库"
|
45
|
+
}
|
46
|
+
|
47
|
+
dbWriter, err := mmdbwriter.New(
|
48
|
+
mmdbwriter.Options{
|
49
|
+
DatabaseType: dbType,
|
50
|
+
Description: description,
|
51
|
+
DisableIPv4Aliasing: false,
|
52
|
+
IPVersion: 6,
|
53
|
+
Languages: languages,
|
54
|
+
RecordSize: 28,
|
55
|
+
},
|
56
|
+
)
|
57
|
+
if err != nil {
|
58
|
+
return fmt.Errorf("creating mmdbwriter: %w", err)
|
59
|
+
}
|
60
|
+
|
61
|
+
if dbType == "GeoIP2-Anonymous-IP" {
|
62
|
+
if err := populateAllNetworks(dbWriter); err != nil {
|
63
|
+
return fmt.Errorf("inserting all networks: %w", err)
|
64
|
+
}
|
65
|
+
}
|
66
|
+
|
67
|
+
jsonFileName := fmt.Sprintf("%s-Test.json", dbType)
|
68
|
+
if err := w.insertJSON(dbWriter, jsonFileName); err != nil {
|
69
|
+
return fmt.Errorf("inserting json: %w", err)
|
70
|
+
}
|
71
|
+
|
72
|
+
dbFileName := fmt.Sprintf("%s-Test.mmdb", dbType)
|
73
|
+
if err := w.write(dbWriter, dbFileName); err != nil {
|
74
|
+
return fmt.Errorf("writing database: %w", err)
|
75
|
+
}
|
76
|
+
}
|
77
|
+
|
78
|
+
return nil
|
79
|
+
}
|
80
|
+
|
81
|
+
// insertJSON reads and parses a json file into mmdbtypes values and inserts
|
82
|
+
// them into the mmdbwriter tree.
|
83
|
+
func (w *Writer) insertJSON(dbWriter *mmdbwriter.Tree, fileName string) error {
|
84
|
+
file, err := os.Open(filepath.Clean(filepath.Join(w.source, fileName)))
|
85
|
+
if err != nil {
|
86
|
+
return fmt.Errorf("opening json file: %w", err)
|
87
|
+
}
|
88
|
+
defer file.Close()
|
89
|
+
|
90
|
+
var data []map[string]any
|
91
|
+
if err := json.NewDecoder(file).Decode(&data); err != nil {
|
92
|
+
return fmt.Errorf("decoding json file: %w", err)
|
93
|
+
}
|
94
|
+
|
95
|
+
for _, record := range data {
|
96
|
+
for k, v := range record {
|
97
|
+
prefix, err := netip.ParsePrefix(k)
|
98
|
+
if err != nil {
|
99
|
+
return fmt.Errorf("parsing ip: %w", err)
|
100
|
+
}
|
101
|
+
|
102
|
+
mmdbValue, err := toMMDBType(prefix.String(), v)
|
103
|
+
if err != nil {
|
104
|
+
return fmt.Errorf("converting value to mmdbtype: %w", err)
|
105
|
+
}
|
106
|
+
|
107
|
+
err = dbWriter.Insert(
|
108
|
+
netipx.PrefixIPNet(prefix),
|
109
|
+
mmdbValue,
|
110
|
+
)
|
111
|
+
if err != nil {
|
112
|
+
return fmt.Errorf("inserting ip: %w", err)
|
113
|
+
}
|
114
|
+
}
|
115
|
+
}
|
116
|
+
return nil
|
117
|
+
}
|
118
|
+
|
119
|
+
// toMMDBType key converts field values read from json into their corresponding mmdbtype.DataType.
|
120
|
+
// It makes some assumptions for numeric types based on previous knowledge about field types.
|
121
|
+
func toMMDBType(key string, value any) (mmdbtype.DataType, error) {
|
122
|
+
switch v := value.(type) {
|
123
|
+
case bool:
|
124
|
+
return mmdbtype.Bool(v), nil
|
125
|
+
case string:
|
126
|
+
return mmdbtype.String(v), nil
|
127
|
+
case map[string]any:
|
128
|
+
m := mmdbtype.Map{}
|
129
|
+
for innerKey, val := range v {
|
130
|
+
innerVal, err := toMMDBType(innerKey, val)
|
131
|
+
if err != nil {
|
132
|
+
return nil, fmt.Errorf("parsing mmdbtype.Map for key %q: %w", key, err)
|
133
|
+
}
|
134
|
+
m[mmdbtype.String(innerKey)] = innerVal
|
135
|
+
}
|
136
|
+
return m, nil
|
137
|
+
case []any:
|
138
|
+
s := mmdbtype.Slice{}
|
139
|
+
for _, val := range v {
|
140
|
+
innerVal, err := toMMDBType(key, val)
|
141
|
+
if err != nil {
|
142
|
+
return nil, fmt.Errorf("parsing mmdbtype.Slice for key %q: %w", key, err)
|
143
|
+
}
|
144
|
+
s = append(s, innerVal)
|
145
|
+
}
|
146
|
+
return s, nil
|
147
|
+
case float64:
|
148
|
+
switch key {
|
149
|
+
case "accuracy_radius", "confidence", "metro_code":
|
150
|
+
return mmdbtype.Uint16(v), nil
|
151
|
+
case "autonomous_system_number", "average_income",
|
152
|
+
"geoname_id", "ipv4_24", "ipv4_32", "ipv6_32",
|
153
|
+
"ipv6_48", "ipv6_64", "population_density":
|
154
|
+
return mmdbtype.Uint32(v), nil
|
155
|
+
case "ip_risk", "latitude", "longitude", "score",
|
156
|
+
"static_ip_score":
|
157
|
+
return mmdbtype.Float64(v), nil
|
158
|
+
default:
|
159
|
+
return nil, fmt.Errorf("unsupported numberic type for key %q: %T", key, value)
|
160
|
+
}
|
161
|
+
default:
|
162
|
+
return nil, fmt.Errorf("unsupported type for key %q: %T", key, value)
|
163
|
+
}
|
164
|
+
}
|
165
|
+
|
166
|
+
// populate all networks inserts all networks into the writer with an empty map value.
|
167
|
+
func populateAllNetworks(w *mmdbwriter.Tree) error {
|
168
|
+
defaultNet, err := netip.ParsePrefix("::/0")
|
169
|
+
if err != nil {
|
170
|
+
return fmt.Errorf("parsing ip: %w", err)
|
171
|
+
}
|
172
|
+
|
173
|
+
err = w.Insert(
|
174
|
+
netipx.PrefixIPNet(defaultNet),
|
175
|
+
mmdbtype.Map{},
|
176
|
+
)
|
177
|
+
if err != nil {
|
178
|
+
return fmt.Errorf("inserting ip: %w", err)
|
179
|
+
}
|
180
|
+
|
181
|
+
return nil
|
182
|
+
}
|
@@ -0,0 +1,39 @@
|
|
1
|
+
package writer
|
2
|
+
|
3
|
+
import (
|
4
|
+
"fmt"
|
5
|
+
"net/netip"
|
6
|
+
|
7
|
+
"go4.org/netipx"
|
8
|
+
)
|
9
|
+
|
10
|
+
// parseIPRange takes IP addresses in string presentation form that represent a
|
11
|
+
// range and returns an IP range.
|
12
|
+
func parseIPRange(from, to string) (netipx.IPRange, error) {
|
13
|
+
startIP, err := netip.ParseAddr(from)
|
14
|
+
if err != nil {
|
15
|
+
return netipx.IPRange{}, fmt.Errorf("parsing %s as an IP: %w", from, err)
|
16
|
+
}
|
17
|
+
endIP, err := netip.ParseAddr(to)
|
18
|
+
if err != nil {
|
19
|
+
return netipx.IPRange{}, fmt.Errorf("parsing %s as an IP: %w", to, err)
|
20
|
+
}
|
21
|
+
ipRange := netipx.IPRangeFrom(startIP, endIP)
|
22
|
+
if !ipRange.IsValid() {
|
23
|
+
return netipx.IPRange{}, fmt.Errorf("%s-%s is an invalid IP range", startIP, endIP)
|
24
|
+
}
|
25
|
+
return ipRange, nil
|
26
|
+
}
|
27
|
+
|
28
|
+
// parseIPSlice parses a slice of IP address strings and returns a slice of netip.Prefix.
|
29
|
+
func parseIPSlice(ipAddresses []string) ([]netip.Prefix, error) {
|
30
|
+
var addrs []netip.Prefix
|
31
|
+
for _, ip := range ipAddresses {
|
32
|
+
addr, err := netip.ParsePrefix(ip)
|
33
|
+
if err != nil {
|
34
|
+
return nil, fmt.Errorf("parsing %s as an IP: %w", ip, err)
|
35
|
+
}
|
36
|
+
addrs = append(addrs, addr)
|
37
|
+
}
|
38
|
+
return addrs, nil
|
39
|
+
}
|
@@ -0,0 +1,245 @@
|
|
1
|
+
package writer
|
2
|
+
|
3
|
+
import (
|
4
|
+
"fmt"
|
5
|
+
"net/netip"
|
6
|
+
|
7
|
+
"github.com/maxmind/mmdbwriter"
|
8
|
+
"github.com/maxmind/mmdbwriter/mmdbtype"
|
9
|
+
"go4.org/netipx"
|
10
|
+
)
|
11
|
+
|
12
|
+
// WriteIPv4TestDB writes mmdb files for an ip range between 1.1.1.1 and 1.1.1.32
|
13
|
+
// with various record sizes.
|
14
|
+
func (w *Writer) WriteIPv4TestDB() error {
|
15
|
+
ipRange, err := parseIPRange("1.1.1.1", "1.1.1.32")
|
16
|
+
if err != nil {
|
17
|
+
return fmt.Errorf("parsing ip range: %w", err)
|
18
|
+
}
|
19
|
+
|
20
|
+
for _, recordSize := range []int{24, 28, 32} {
|
21
|
+
err := w.writeMaxMindTestDB(
|
22
|
+
recordSize,
|
23
|
+
[]netipx.IPRange{ipRange},
|
24
|
+
"ipv4",
|
25
|
+
)
|
26
|
+
if err != nil {
|
27
|
+
return fmt.Errorf("writing test database: %w", err)
|
28
|
+
}
|
29
|
+
}
|
30
|
+
|
31
|
+
return nil
|
32
|
+
}
|
33
|
+
|
34
|
+
// WriteIPv6TestDB writes mmdb files for an ip range between ::1:ffff:ffff and ::2:0000:0059
|
35
|
+
// with various record sizes.
|
36
|
+
func (w *Writer) WriteIPv6TestDB() error {
|
37
|
+
ipRange, err := parseIPRange("::1:ffff:ffff", "::2:0000:0059")
|
38
|
+
if err != nil {
|
39
|
+
return fmt.Errorf("parsing ip range: %w", err)
|
40
|
+
}
|
41
|
+
|
42
|
+
for _, recordSize := range []int{24, 28, 32} {
|
43
|
+
err := w.writeMaxMindTestDB(
|
44
|
+
recordSize,
|
45
|
+
[]netipx.IPRange{ipRange},
|
46
|
+
"ipv6",
|
47
|
+
)
|
48
|
+
if err != nil {
|
49
|
+
return fmt.Errorf("writing test database: %w", err)
|
50
|
+
}
|
51
|
+
}
|
52
|
+
|
53
|
+
return nil
|
54
|
+
}
|
55
|
+
|
56
|
+
// WriteMixedIPTestDB writes mmdb files for a mixed ip version range between ::1:ffff:ffff and ::2:0000:0059
|
57
|
+
// with various record sizes.
|
58
|
+
func (w *Writer) WriteMixedIPTestDB() error {
|
59
|
+
ipv6Range, err := parseIPRange("::1:ffff:ffff", "::2:0000:0059")
|
60
|
+
if err != nil {
|
61
|
+
return fmt.Errorf("parsing ip range: %w", err)
|
62
|
+
}
|
63
|
+
|
64
|
+
ipv4Range, err := parseIPRange("1.1.1.1", "1.1.1.32")
|
65
|
+
if err != nil {
|
66
|
+
return fmt.Errorf("parsing ip range: %w", err)
|
67
|
+
}
|
68
|
+
|
69
|
+
for _, recordSize := range []int{24, 28, 32} {
|
70
|
+
err := w.writeMaxMindTestDB(
|
71
|
+
recordSize,
|
72
|
+
[]netipx.IPRange{ipv6Range, ipv4Range},
|
73
|
+
"mixed",
|
74
|
+
)
|
75
|
+
if err != nil {
|
76
|
+
return fmt.Errorf("writing test database: %w", err)
|
77
|
+
}
|
78
|
+
}
|
79
|
+
|
80
|
+
return nil
|
81
|
+
}
|
82
|
+
|
83
|
+
// writeMaxMindTestDB writes test mmdb files.
|
84
|
+
func (w *Writer) writeMaxMindTestDB(
|
85
|
+
recordSize int,
|
86
|
+
ipRange []netipx.IPRange,
|
87
|
+
ipVersionName string,
|
88
|
+
) error {
|
89
|
+
ipVersion := 6
|
90
|
+
if ipRange[0].From().Is4() {
|
91
|
+
ipVersion = 4
|
92
|
+
}
|
93
|
+
|
94
|
+
metadata := map[string]string{}
|
95
|
+
metadata["en"] = "Test Database"
|
96
|
+
metadata["zh"] = "Test Database Chinese"
|
97
|
+
|
98
|
+
dbWriter, err := mmdbwriter.New(
|
99
|
+
mmdbwriter.Options{
|
100
|
+
DatabaseType: "Test",
|
101
|
+
Description: metadata,
|
102
|
+
DisableIPv4Aliasing: ipVersion == 4,
|
103
|
+
IPVersion: ipVersion,
|
104
|
+
Languages: []string{"en", "zh"},
|
105
|
+
RecordSize: recordSize,
|
106
|
+
},
|
107
|
+
)
|
108
|
+
if err != nil {
|
109
|
+
return fmt.Errorf("creating mmdbwriter: %w", err)
|
110
|
+
}
|
111
|
+
|
112
|
+
for _, ir := range ipRange {
|
113
|
+
for _, prefix := range ir.Prefixes() {
|
114
|
+
ipString := prefix.Addr().String()
|
115
|
+
if ipVersion == 6 && prefix.Addr().Is4() {
|
116
|
+
ipString = "::" + ipString
|
117
|
+
}
|
118
|
+
|
119
|
+
err := dbWriter.Insert(
|
120
|
+
netipx.PrefixIPNet(prefix),
|
121
|
+
mmdbtype.Map{
|
122
|
+
"ip": mmdbtype.String(ipString),
|
123
|
+
},
|
124
|
+
)
|
125
|
+
if err != nil {
|
126
|
+
return fmt.Errorf("inserting ip: %w", err)
|
127
|
+
}
|
128
|
+
}
|
129
|
+
}
|
130
|
+
|
131
|
+
fileName := fmt.Sprintf("MaxMind-DB-test-%s-%d.mmdb", ipVersionName, recordSize)
|
132
|
+
if err := w.write(dbWriter, fileName); err != nil {
|
133
|
+
return fmt.Errorf("writing database: %w", err)
|
134
|
+
}
|
135
|
+
|
136
|
+
return nil
|
137
|
+
}
|
138
|
+
|
139
|
+
// WriteNoIPv4TestDB writes an mmdb file with no ipv4 records.
|
140
|
+
func (w *Writer) WriteNoIPv4TestDB() error {
|
141
|
+
dbWriter, err := mmdbwriter.New(
|
142
|
+
mmdbwriter.Options{
|
143
|
+
DatabaseType: "MaxMind DB No IPv4 Search Tree",
|
144
|
+
Description: map[string]string{
|
145
|
+
"en": "MaxMind DB No IPv4 Search Tree",
|
146
|
+
},
|
147
|
+
DisableIPv4Aliasing: true,
|
148
|
+
IncludeReservedNetworks: true,
|
149
|
+
IPVersion: 6,
|
150
|
+
Languages: []string{"en"},
|
151
|
+
RecordSize: 24,
|
152
|
+
},
|
153
|
+
)
|
154
|
+
if err != nil {
|
155
|
+
return fmt.Errorf("creating mmdbwriter: %w", err)
|
156
|
+
}
|
157
|
+
|
158
|
+
addr, err := netip.ParsePrefix("::/64")
|
159
|
+
if err != nil {
|
160
|
+
return fmt.Errorf("parsing ip: %w", err)
|
161
|
+
}
|
162
|
+
|
163
|
+
err = dbWriter.Insert(
|
164
|
+
netipx.PrefixIPNet(addr),
|
165
|
+
mmdbtype.String(addr.String()),
|
166
|
+
)
|
167
|
+
if err != nil {
|
168
|
+
return fmt.Errorf("inserting ip: %w", err)
|
169
|
+
}
|
170
|
+
|
171
|
+
if err := w.write(dbWriter, "MaxMind-DB-no-ipv4-search-tree.mmdb"); err != nil {
|
172
|
+
return fmt.Errorf("writing database: %w", err)
|
173
|
+
}
|
174
|
+
return nil
|
175
|
+
}
|
176
|
+
|
177
|
+
// WriteNoMapTestDB writes an mmdb file where each record points to
|
178
|
+
// a string value.
|
179
|
+
func (w *Writer) WriteNoMapTestDB() error {
|
180
|
+
dbWriter, err := mmdbwriter.New(
|
181
|
+
mmdbwriter.Options{
|
182
|
+
DatabaseType: "MaxMind DB String Value Entries",
|
183
|
+
Description: map[string]string{
|
184
|
+
"en": "MaxMind DB String Value Entries (no maps or arrays as values)",
|
185
|
+
},
|
186
|
+
IPVersion: 4,
|
187
|
+
Languages: []string{"en"},
|
188
|
+
RecordSize: 24,
|
189
|
+
},
|
190
|
+
)
|
191
|
+
if err != nil {
|
192
|
+
return fmt.Errorf("creating mmdbwriter: %w", err)
|
193
|
+
}
|
194
|
+
|
195
|
+
ipRange, err := parseIPRange("1.1.1.1", "1.1.1.32")
|
196
|
+
if err != nil {
|
197
|
+
return fmt.Errorf("parsing ip range: %w", err)
|
198
|
+
}
|
199
|
+
|
200
|
+
for _, prefix := range ipRange.Prefixes() {
|
201
|
+
err := dbWriter.Insert(
|
202
|
+
netipx.PrefixIPNet(prefix),
|
203
|
+
mmdbtype.String(prefix.String()),
|
204
|
+
)
|
205
|
+
if err != nil {
|
206
|
+
return fmt.Errorf("inserting ip: %w", err)
|
207
|
+
}
|
208
|
+
}
|
209
|
+
|
210
|
+
if err := w.write(dbWriter, "MaxMind-DB-string-value-entries.mmdb"); err != nil {
|
211
|
+
return fmt.Errorf("writing database: %w", err)
|
212
|
+
}
|
213
|
+
return nil
|
214
|
+
}
|
215
|
+
|
216
|
+
// WriteMetadataPointersTestDB writes an mmdb file with metadata pointers allowed.
|
217
|
+
func (w *Writer) WriteMetadataPointersTestDB() error {
|
218
|
+
repeatedString := "Lots of pointers in metadata"
|
219
|
+
dbWriter, err := mmdbwriter.New(
|
220
|
+
mmdbwriter.Options{
|
221
|
+
DatabaseType: repeatedString,
|
222
|
+
Description: map[string]string{
|
223
|
+
"en": repeatedString,
|
224
|
+
"es": repeatedString,
|
225
|
+
"zh": repeatedString,
|
226
|
+
},
|
227
|
+
DisableIPv4Aliasing: true,
|
228
|
+
IPVersion: 6,
|
229
|
+
Languages: []string{"en", "es", "zh"},
|
230
|
+
RecordSize: 24,
|
231
|
+
},
|
232
|
+
)
|
233
|
+
if err != nil {
|
234
|
+
return fmt.Errorf("creating mmdbwriter: %w", err)
|
235
|
+
}
|
236
|
+
|
237
|
+
if err := populateAllNetworks(dbWriter); err != nil {
|
238
|
+
return fmt.Errorf("inserting all networks: %w", err)
|
239
|
+
}
|
240
|
+
|
241
|
+
if err := w.write(dbWriter, "MaxMind-DB-test-metadata-pointers.mmdb"); err != nil {
|
242
|
+
return fmt.Errorf("writing database: %w", err)
|
243
|
+
}
|
244
|
+
return nil
|
245
|
+
}
|
@@ -0,0 +1,73 @@
|
|
1
|
+
package writer
|
2
|
+
|
3
|
+
import (
|
4
|
+
"fmt"
|
5
|
+
"net/netip"
|
6
|
+
|
7
|
+
"github.com/maxmind/mmdbwriter"
|
8
|
+
"github.com/maxmind/mmdbwriter/mmdbtype"
|
9
|
+
"go4.org/netipx"
|
10
|
+
)
|
11
|
+
|
12
|
+
// WriteDeeplyNestedStructuresTestDB writes an mmdb file with deeply nested record value types.
|
13
|
+
func (w *Writer) WriteDeeplyNestedStructuresTestDB() error {
|
14
|
+
dbWriter, err := mmdbwriter.New(
|
15
|
+
mmdbwriter.Options{
|
16
|
+
DatabaseType: "MaxMind DB Nested Data Structures",
|
17
|
+
Description: map[string]string{
|
18
|
+
"en": "MaxMind DB Nested Data Structures Test database - contains deeply nested map/array structures",
|
19
|
+
},
|
20
|
+
DisableIPv4Aliasing: false,
|
21
|
+
IncludeReservedNetworks: true,
|
22
|
+
IPVersion: 6,
|
23
|
+
Languages: []string{"en"},
|
24
|
+
RecordSize: 24,
|
25
|
+
},
|
26
|
+
)
|
27
|
+
if err != nil {
|
28
|
+
return fmt.Errorf("creating mmdbwriter: %w", err)
|
29
|
+
}
|
30
|
+
|
31
|
+
addrs, err := parseIPSlice(ipSample)
|
32
|
+
if err != nil {
|
33
|
+
return fmt.Errorf("parsing ip addresses: %w", err)
|
34
|
+
}
|
35
|
+
if err := insertNestedStructure(dbWriter, addrs); err != nil {
|
36
|
+
return fmt.Errorf("inserting all types records: %w", err)
|
37
|
+
}
|
38
|
+
|
39
|
+
if err := w.write(dbWriter, "MaxMind-DB-test-nested.mmdb"); err != nil {
|
40
|
+
return fmt.Errorf("writing database: %w", err)
|
41
|
+
}
|
42
|
+
return nil
|
43
|
+
}
|
44
|
+
|
45
|
+
// insertNestedStructure inserts records with deeply nested structures.
|
46
|
+
func insertNestedStructure(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
|
47
|
+
nestedStruct := mmdbtype.Map{
|
48
|
+
"map1": mmdbtype.Map{
|
49
|
+
"map2": mmdbtype.Map{
|
50
|
+
"array": mmdbtype.Slice{
|
51
|
+
mmdbtype.Map{
|
52
|
+
"map3": mmdbtype.Map{
|
53
|
+
"a": mmdbtype.Uint32(1),
|
54
|
+
"b": mmdbtype.Uint32(2),
|
55
|
+
"c": mmdbtype.Uint32(3),
|
56
|
+
},
|
57
|
+
},
|
58
|
+
},
|
59
|
+
},
|
60
|
+
},
|
61
|
+
}
|
62
|
+
|
63
|
+
for _, addr := range ipAddresses {
|
64
|
+
err := w.Insert(
|
65
|
+
netipx.PrefixIPNet(addr),
|
66
|
+
nestedStruct,
|
67
|
+
)
|
68
|
+
if err != nil {
|
69
|
+
return fmt.Errorf("inserting ip: %w", err)
|
70
|
+
}
|
71
|
+
}
|
72
|
+
return nil
|
73
|
+
}
|