maxmind-geoip2 1.0.0 → 1.2.0

data/test/data/cmd/write-test-data/main.go

@@ -0,0 +1,68 @@
+// write-test-data generates test mmdb files.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/maxmind/MaxMind-DB/pkg/writer"
+)
+
+func main() {
+	source := flag.String("source", "", "Source data directory")
+	target := flag.String("target", "", "Destination directory for the generated mmdb files")
+
+	flag.Parse()
+
+	w, err := writer.New(*source, *target)
+	if err != nil {
+		fmt.Printf("creating writer: %+v\n", err)
+		os.Exit(1)
+	}
+
+	if err := w.WriteIPv4TestDB(); err != nil {
+		fmt.Printf("writing IPv4 test databases: %+v\n", err)
+		os.Exit(1)
+	}
+
+	if err := w.WriteIPv6TestDB(); err != nil {
+		fmt.Printf("writing IPv6 test databases: %+v\n", err)
+		os.Exit(1)
+	}
+
+	if err := w.WriteMixedIPTestDB(); err != nil {
+		fmt.Printf("writing IPv6 test databases: %+v\n", err)
+		os.Exit(1)
+	}
+
+	if err := w.WriteNoIPv4TestDB(); err != nil {
+		fmt.Printf("writing no IPv4 test databases: %+v\n", err)
+		os.Exit(1)
+	}
+
+	if err := w.WriteNoMapTestDB(); err != nil {
+		fmt.Printf("writing no map test databases: %+v\n", err)
+		os.Exit(1)
+	}
+
+	if err := w.WriteMetadataPointersTestDB(); err != nil {
+		fmt.Printf("writing metadata pointers test databases: %+v\n", err)
+		os.Exit(1)
+	}
+
+	if err := w.WriteDecoderTestDB(); err != nil {
+		fmt.Printf("writing decoder test databases: %+v\n", err)
+		os.Exit(1)
+	}
+
+	if err := w.WriteDeeplyNestedStructuresTestDB(); err != nil {
+		fmt.Printf("writing decoder test databases: %+v\n", err)
+		os.Exit(1)
+	}
+
+	if err := w.WriteGeoIP2TestDB(); err != nil {
+		fmt.Printf("writing GeoIP2 test databases: %+v\n", err)
+		os.Exit(1)
+	}
+}

data/test/data/go.mod

@@ -0,0 +1,13 @@
+module github.com/maxmind/MaxMind-DB
+
+go 1.21
+
+require (
+	github.com/maxmind/mmdbwriter v1.0.0
+	go4.org/netipx v0.0.0-20230824141953-6213f710f925
+)
+
+require (
+	github.com/oschwald/maxminddb-golang v1.12.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+)

data/test/data/go.sum

@@ -0,0 +1,16 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/maxmind/mmdbwriter v1.0.0 h1:bieL4P6yaYaHvbtLSwnKtEvScUKKD6jcKaLiTM3WSMw=
+github.com/maxmind/mmdbwriter v1.0.0/go.mod h1:noBMCUtyN5PUQ4H8ikkOvGSHhzhLok51fON2hcrpKj8=
+github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs=
+github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+go4.org/netipx v0.0.0-20230824141953-6213f710f925 h1:eeQDDVKFkx0g4Hyy8pHgmZaK0EqB4SD6rvKbUdN3ziQ=
+go4.org/netipx v0.0.0-20230824141953-6213f710f925/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

data/test/data/perltidyrc

@@ -1,6 +1,7 @@
 --blank-lines-before-packages=0
 --iterations=2
 --no-outdent-long-comments
+--weld-nested-containers
 -b
 -bar
 -boc
@@ -10,3 +11,8 @@
 -nolq
 -se
 -wbb="% + - * / x != == >= <= =~ !~ < > | & >= < = **= += *= &= <<= &&= -= /= |= >>= ||= .= %= ^= x="
+--character-encoding=utf8
+--valign-exclusion-list="q"
+--want-trailing-commas=m
+--add-trailing-commas
+--delete-repeated-commas

data/test/data/pkg/writer/decoder.go

@@ -0,0 +1,178 @@
+package writer
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"math"
+	"math/big"
+	"net/netip"
+
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+	"go4.org/netipx"
+)
+
+// WriteDecoderTestDB writes an mmdb file with all possible record value types.
+func (w *Writer) WriteDecoderTestDB() error {
+	dbWriter, err := mmdbwriter.New(
+		mmdbwriter.Options{
+			DatabaseType: "MaxMind DB Decoder Test",
+			Description: map[string]string{
+				"en": "MaxMind DB Decoder Test database - contains every MaxMind DB data type",
+			},
+			DisableIPv4Aliasing:     false,
+			IncludeReservedNetworks: true,
+			IPVersion:               6,
+			Languages:               []string{"en"},
+			RecordSize:              24,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("creating mmdbwriter: %w", err)
+	}
+
+	addrs, err := parseIPSlice(ipSample)
+	if err != nil {
+		return fmt.Errorf("parsing ip addresses: %w", err)
+	}
+	if err := insertAllTypes(dbWriter, addrs); err != nil {
+		return fmt.Errorf("inserting all types records: %w", err)
+	}
+
+	zeroAddr, err := netip.ParsePrefix("::0.0.0.0/128")
+	if err != nil {
+		return fmt.Errorf("parsing ip: %w", err)
+	}
+	if err := insertAllTypesZero(dbWriter, []netip.Prefix{zeroAddr}); err != nil {
+		return fmt.Errorf("inserting all types records: %w", err)
+	}
+
+	maxAddr, err := netip.ParsePrefix("::255.255.255.255/128")
+	if err != nil {
+		return fmt.Errorf("parsing ip: %w", err)
+	}
+	if err := insertNumericMax(dbWriter, []netip.Prefix{maxAddr}); err != nil {
+		return fmt.Errorf("inserting all types records: %w", err)
+	}
+
+	if err := w.write(dbWriter, "MaxMind-DB-test-decoder.mmdb"); err != nil {
+		return fmt.Errorf("writing database: %w", err)
+	}
+	return nil
+}
+
+// insertAllTypes inserts records with all possible value types.
+func insertAllTypes(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
+	buf := new(bytes.Buffer)
+	if err := binary.Write(buf, binary.BigEndian, uint32(42)); err != nil {
+		return fmt.Errorf("creating buffer for all types record: %w", err)
+	}
+
+	ui64 := big.Int{}
+	ui64.Lsh(big.NewInt(1), 60)
+
+	ui128 := big.Int{}
+	ui128.Lsh(big.NewInt(1), 120)
+	mmdbUint128 := mmdbtype.Uint128(ui128)
+
+	allTypes := mmdbtype.Map{
+		"array": mmdbtype.Slice{
+			mmdbtype.Uint32(1),
+			mmdbtype.Uint32(2),
+			mmdbtype.Uint32(3),
+		},
+		"bytes":   mmdbtype.Bytes(buf.Bytes()),
+		"boolean": mmdbtype.Bool(true),
+		"double":  mmdbtype.Float64(42.123456),
+		"float":   mmdbtype.Float32(1.1),
+		"int32":   mmdbtype.Int32(-1 * math.Pow(2, 28)),
+		"map": mmdbtype.Map{
+			"mapX": mmdbtype.Map{
+				"utf8_stringX": mmdbtype.String("hello"),
+				"arrayX": mmdbtype.Slice{
+					mmdbtype.Uint32(7),
+					mmdbtype.Uint32(8),
+					mmdbtype.Uint32(9),
+				},
+			},
+		},
+		"uint16":      mmdbtype.Uint16(100),
+		"uint32":      mmdbtype.Uint32(math.Pow(2, 28)),
+		"uint64":      mmdbtype.Uint64(ui64.Uint64()),
+		"uint128":     mmdbUint128.Copy(),
+		"utf8_string": mmdbtype.String("unicode! ☯ - ♫"),
+	}
+
+	for _, addr := range ipAddresses {
+		err := w.Insert(
+			netipx.PrefixIPNet(addr),
+			allTypes,
+		)
+		if err != nil {
+			return fmt.Errorf("inserting ip: %w", err)
+		}
+	}
+	return nil
+}
+
+// insertAllTypesZero inserts records with all possible value types with zero values.
+func insertAllTypesZero(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
+	var uint128 big.Int
+	mmdbUint128 := mmdbtype.Uint128(uint128)
+
+	zeroValues := mmdbtype.Map{
+		"array":       mmdbtype.Slice{},
+		"bytes":       mmdbtype.Bytes([]byte{}),
+		"boolean":     mmdbtype.Bool(false),
+		"double":      mmdbtype.Float64(0),
+		"float":       mmdbtype.Float32(0),
+		"int32":       mmdbtype.Int32(0),
+		"map":         mmdbtype.Map{},
+		"uint16":      mmdbtype.Uint16(0),
+		"uint32":      mmdbtype.Uint32(0),
+		"uint64":      mmdbtype.Uint64(0),
+		"uint128":     mmdbUint128.Copy(),
+		"utf8_string": mmdbtype.String(""),
+	}
+
+	for _, addr := range ipAddresses {
+		err := w.Insert(
+			netipx.PrefixIPNet(addr),
+			zeroValues,
+		)
+		if err != nil {
+			return fmt.Errorf("inserting ip: %w", err)
+		}
+	}
+	return nil
+}
+
+// insertNumericMax inserts records with numeric types maxed out.
+func insertNumericMax(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
+	var uint128Max big.Int
+	uint128Max.Exp(big.NewInt(2), big.NewInt(128), nil)
+	uint128Max.Sub(&uint128Max, big.NewInt(1))
+	mmdbUint128 := mmdbtype.Uint128(uint128Max)
+
+	numMax := mmdbtype.Map{
+		"double":  mmdbtype.Float64(math.Inf(1)),
+		"float":   mmdbtype.Float32(float32(math.Inf(1))),
+		"int32":   mmdbtype.Int32(1<<31 - 1),
+		"uint16":  mmdbtype.Uint16(0xffff),
+		"uint32":  mmdbtype.Uint32(0xffffffff),
+		"uint64":  mmdbtype.Uint64(0xffffffffffffffff),
+		"uint128": mmdbUint128.Copy(),
+	}
+
+	for _, addr := range ipAddresses {
+		err := w.Insert(
+			netipx.PrefixIPNet(addr),
+			numMax,
+		)
+		if err != nil {
+			return fmt.Errorf("inserting ip: %w", err)
+		}
+	}
+	return nil
+}

data/test/data/pkg/writer/geoip2.go

@@ -0,0 +1,182 @@
+package writer
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+	"go4.org/netipx"
+)
+
+// WriteGeoIP2TestDB writes GeoIP2 test mmdb files.
+func (w *Writer) WriteGeoIP2TestDB() error {
+	dbTypes := []string{
+		"GeoIP2-Anonymous-IP",
+		"GeoIP2-City",
+		"GeoIP2-Connection-Type",
+		"GeoIP2-Country",
+		"GeoIP2-DensityIncome",
+		"GeoIP2-Domain",
+		"GeoIP2-Enterprise",
+		"GeoIP2-ISP",
+		"GeoIP2-Precision-Enterprise",
+		"GeoIP2-Static-IP-Score",
+		"GeoIP2-User-Count",
+		"GeoLite2-ASN",
+		"GeoLite2-City",
+		"GeoLite2-Country",
+	}
+
+	for _, dbType := range dbTypes {
+		languages := []string{"en"}
+		description := map[string]string{
+			"en": strings.ReplaceAll(dbType, "-", " ") +
+				" Test Database (fake GeoIP2 data, for example purposes only)",
+		}
+
+		if dbType == "GeoIP2-City" {
+			languages = append(languages, "zh")
+			description["zh"] = "小型数据库"
+		}
+
+		dbWriter, err := mmdbwriter.New(
+			mmdbwriter.Options{
+				DatabaseType:        dbType,
+				Description:         description,
+				DisableIPv4Aliasing: false,
+				IPVersion:           6,
+				Languages:           languages,
+				RecordSize:          28,
+			},
+		)
+		if err != nil {
+			return fmt.Errorf("creating mmdbwriter: %w", err)
+		}
+
+		if dbType == "GeoIP2-Anonymous-IP" {
+			if err := populateAllNetworks(dbWriter); err != nil {
+				return fmt.Errorf("inserting all networks: %w", err)
+			}
+		}
+
+		jsonFileName := fmt.Sprintf("%s-Test.json", dbType)
+		if err := w.insertJSON(dbWriter, jsonFileName); err != nil {
+			return fmt.Errorf("inserting json: %w", err)
+		}
+
+		dbFileName := fmt.Sprintf("%s-Test.mmdb", dbType)
+		if err := w.write(dbWriter, dbFileName); err != nil {
+			return fmt.Errorf("writing database: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// insertJSON reads and parses a json file into mmdbtypes values and inserts
+// them into the mmdbwriter tree.
+func (w *Writer) insertJSON(dbWriter *mmdbwriter.Tree, fileName string) error {
+	file, err := os.Open(filepath.Clean(filepath.Join(w.source, fileName)))
+	if err != nil {
+		return fmt.Errorf("opening json file: %w", err)
+	}
+	defer file.Close()
+
+	var data []map[string]any
+	if err := json.NewDecoder(file).Decode(&data); err != nil {
+		return fmt.Errorf("decoding json file: %w", err)
+	}
+
+	for _, record := range data {
+		for k, v := range record {
+			prefix, err := netip.ParsePrefix(k)
+			if err != nil {
+				return fmt.Errorf("parsing ip: %w", err)
+			}
+
+			mmdbValue, err := toMMDBType(prefix.String(), v)
+			if err != nil {
+				return fmt.Errorf("converting value to mmdbtype: %w", err)
+			}
+
+			err = dbWriter.Insert(
+				netipx.PrefixIPNet(prefix),
+				mmdbValue,
+			)
+			if err != nil {
+				return fmt.Errorf("inserting ip: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+// toMMDBType key converts field values read from json into their corresponding mmdbtype.DataType.
+// It makes some assumptions for numeric types based on previous knowledge about field types.
+func toMMDBType(key string, value any) (mmdbtype.DataType, error) {
+	switch v := value.(type) {
+	case bool:
+		return mmdbtype.Bool(v), nil
+	case string:
+		return mmdbtype.String(v), nil
+	case map[string]any:
+		m := mmdbtype.Map{}
+		for innerKey, val := range v {
+			innerVal, err := toMMDBType(innerKey, val)
+			if err != nil {
+				return nil, fmt.Errorf("parsing mmdbtype.Map for key %q: %w", key, err)
+			}
+			m[mmdbtype.String(innerKey)] = innerVal
+		}
+		return m, nil
+	case []any:
+		s := mmdbtype.Slice{}
+		for _, val := range v {
+			innerVal, err := toMMDBType(key, val)
+			if err != nil {
+				return nil, fmt.Errorf("parsing mmdbtype.Slice for key %q: %w", key, err)
+			}
+			s = append(s, innerVal)
+		}
+		return s, nil
+	case float64:
+		switch key {
+		case "accuracy_radius", "confidence", "metro_code":
+			return mmdbtype.Uint16(v), nil
+		case "autonomous_system_number", "average_income",
+			"geoname_id", "ipv4_24", "ipv4_32", "ipv6_32",
+			"ipv6_48", "ipv6_64", "population_density":
+			return mmdbtype.Uint32(v), nil
+		case "ip_risk", "latitude", "longitude", "score",
+			"static_ip_score":
+			return mmdbtype.Float64(v), nil
+		default:
+			return nil, fmt.Errorf("unsupported numberic type for key %q: %T", key, value)
+		}
+	default:
+		return nil, fmt.Errorf("unsupported type for key %q: %T", key, value)
+	}
+}
+
+// populate all networks inserts all networks into the writer with an empty map value.
+func populateAllNetworks(w *mmdbwriter.Tree) error {
+	defaultNet, err := netip.ParsePrefix("::/0")
+	if err != nil {
+		return fmt.Errorf("parsing ip: %w", err)
+	}
+
+	err = w.Insert(
+		netipx.PrefixIPNet(defaultNet),
+		mmdbtype.Map{},
+	)
+	if err != nil {
+		return fmt.Errorf("inserting ip: %w", err)
+	}
+
+	return nil
+}

data/test/data/pkg/writer/ip.go

@@ -0,0 +1,39 @@
+package writer
+
+import (
+	"fmt"
+	"net/netip"
+
+	"go4.org/netipx"
+)
+
+// parseIPRange takes IP addresses in string presentation form that represent a
+// range and returns an IP range.
+func parseIPRange(from, to string) (netipx.IPRange, error) {
+	startIP, err := netip.ParseAddr(from)
+	if err != nil {
+		return netipx.IPRange{}, fmt.Errorf("parsing %s as an IP: %w", from, err)
+	}
+	endIP, err := netip.ParseAddr(to)
+	if err != nil {
+		return netipx.IPRange{}, fmt.Errorf("parsing %s as an IP: %w", to, err)
+	}
+	ipRange := netipx.IPRangeFrom(startIP, endIP)
+	if !ipRange.IsValid() {
+		return netipx.IPRange{}, fmt.Errorf("%s-%s is an invalid IP range", startIP, endIP)
+	}
+	return ipRange, nil
+}
+
+// parseIPSlice parses a slice of IP address strings and returns a slice of netip.Prefix.
+func parseIPSlice(ipAddresses []string) ([]netip.Prefix, error) {
+	var addrs []netip.Prefix
+	for _, ip := range ipAddresses {
+		addr, err := netip.ParsePrefix(ip)
+		if err != nil {
+			return nil, fmt.Errorf("parsing %s as an IP: %w", ip, err)
+		}
+		addrs = append(addrs, addr)
+	}
+	return addrs, nil
+}