maxmind-geoip2 1.1.0 → 1.3.0

data/test/data/pkg/writer/decoder.go

@@ -0,0 +1,178 @@
+package writer
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"math"
+	"math/big"
+	"net/netip"
+
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+	"go4.org/netipx"
+)
+
+// WriteDecoderTestDB writes an mmdb file with all possible record value types.
+func (w *Writer) WriteDecoderTestDB() error {
+	dbWriter, err := mmdbwriter.New(
+		mmdbwriter.Options{
+			DatabaseType: "MaxMind DB Decoder Test",
+			Description: map[string]string{
+				"en": "MaxMind DB Decoder Test database - contains every MaxMind DB data type",
+			},
+			DisableIPv4Aliasing:     false,
+			IncludeReservedNetworks: true,
+			IPVersion:               6,
+			Languages:               []string{"en"},
+			RecordSize:              24,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("creating mmdbwriter: %w", err)
+	}
+
+	addrs, err := parseIPSlice(ipSample)
+	if err != nil {
+		return fmt.Errorf("parsing ip addresses: %w", err)
+	}
+	if err := insertAllTypes(dbWriter, addrs); err != nil {
+		return fmt.Errorf("inserting all types records: %w", err)
+	}
+
+	zeroAddr, err := netip.ParsePrefix("::0.0.0.0/128")
+	if err != nil {
+		return fmt.Errorf("parsing ip: %w", err)
+	}
+	if err := insertAllTypesZero(dbWriter, []netip.Prefix{zeroAddr}); err != nil {
+		return fmt.Errorf("inserting all types records: %w", err)
+	}
+
+	maxAddr, err := netip.ParsePrefix("::255.255.255.255/128")
+	if err != nil {
+		return fmt.Errorf("parsing ip: %w", err)
+	}
+	if err := insertNumericMax(dbWriter, []netip.Prefix{maxAddr}); err != nil {
+		return fmt.Errorf("inserting all types records: %w", err)
+	}
+
+	if err := w.write(dbWriter, "MaxMind-DB-test-decoder.mmdb"); err != nil {
+		return fmt.Errorf("writing database: %w", err)
+	}
+	return nil
+}
+
+// insertAllTypes inserts records with all possible value types.
+func insertAllTypes(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
+	buf := new(bytes.Buffer)
+	if err := binary.Write(buf, binary.BigEndian, uint32(42)); err != nil {
+		return fmt.Errorf("creating buffer for all types record: %w", err)
+	}
+
+	ui64 := big.Int{}
+	ui64.Lsh(big.NewInt(1), 60)
+
+	ui128 := big.Int{}
+	ui128.Lsh(big.NewInt(1), 120)
+	mmdbUint128 := mmdbtype.Uint128(ui128)
+
+	allTypes := mmdbtype.Map{
+		"array": mmdbtype.Slice{
+			mmdbtype.Uint32(1),
+			mmdbtype.Uint32(2),
+			mmdbtype.Uint32(3),
+		},
+		"bytes":   mmdbtype.Bytes(buf.Bytes()),
+		"boolean": mmdbtype.Bool(true),
+		"double":  mmdbtype.Float64(42.123456),
+		"float":   mmdbtype.Float32(1.1),
+		"int32":   mmdbtype.Int32(-1 * math.Pow(2, 28)),
+		"map": mmdbtype.Map{
+			"mapX": mmdbtype.Map{
+				"utf8_stringX": mmdbtype.String("hello"),
+				"arrayX": mmdbtype.Slice{
+					mmdbtype.Uint32(7),
+					mmdbtype.Uint32(8),
+					mmdbtype.Uint32(9),
+				},
+			},
+		},
+		"uint16":      mmdbtype.Uint16(100),
+		"uint32":      mmdbtype.Uint32(math.Pow(2, 28)),
+		"uint64":      mmdbtype.Uint64(ui64.Uint64()),
+		"uint128":     mmdbUint128.Copy(),
+		"utf8_string": mmdbtype.String("unicode! ☯ - ♫"),
+	}
+
+	for _, addr := range ipAddresses {
+		err := w.Insert(
+			netipx.PrefixIPNet(addr),
+			allTypes,
+		)
+		if err != nil {
+			return fmt.Errorf("inserting ip: %w", err)
+		}
+	}
+	return nil
+}
+
+// insertAllTypesZero inserts records with all possible value types with zero values.
+func insertAllTypesZero(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
+	var uint128 big.Int
+	mmdbUint128 := mmdbtype.Uint128(uint128)
+
+	zeroValues := mmdbtype.Map{
+		"array":       mmdbtype.Slice{},
+		"bytes":       mmdbtype.Bytes([]byte{}),
+		"boolean":     mmdbtype.Bool(false),
+		"double":      mmdbtype.Float64(0),
+		"float":       mmdbtype.Float32(0),
+		"int32":       mmdbtype.Int32(0),
+		"map":         mmdbtype.Map{},
+		"uint16":      mmdbtype.Uint16(0),
+		"uint32":      mmdbtype.Uint32(0),
+		"uint64":      mmdbtype.Uint64(0),
+		"uint128":     mmdbUint128.Copy(),
+		"utf8_string": mmdbtype.String(""),
+	}
+
+	for _, addr := range ipAddresses {
+		err := w.Insert(
+			netipx.PrefixIPNet(addr),
+			zeroValues,
+		)
+		if err != nil {
+			return fmt.Errorf("inserting ip: %w", err)
+		}
+	}
+	return nil
+}
+
+// insertNumericMax inserts records with numeric types maxed out.
+func insertNumericMax(w *mmdbwriter.Tree, ipAddresses []netip.Prefix) error {
+	var uint128Max big.Int
+	uint128Max.Exp(big.NewInt(2), big.NewInt(128), nil)
+	uint128Max.Sub(&uint128Max, big.NewInt(1))
+	mmdbUint128 := mmdbtype.Uint128(uint128Max)
+
+	numMax := mmdbtype.Map{
+		"double":  mmdbtype.Float64(math.Inf(1)),
+		"float":   mmdbtype.Float32(float32(math.Inf(1))),
+		"int32":   mmdbtype.Int32(1<<31 - 1),
+		"uint16":  mmdbtype.Uint16(0xffff),
+		"uint32":  mmdbtype.Uint32(0xffffffff),
+		"uint64":  mmdbtype.Uint64(0xffffffffffffffff),
+		"uint128": mmdbUint128.Copy(),
+	}
+
+	for _, addr := range ipAddresses {
+		err := w.Insert(
+			netipx.PrefixIPNet(addr),
+			numMax,
+		)
+		if err != nil {
+			return fmt.Errorf("inserting ip: %w", err)
+		}
+	}
+	return nil
+}

data/test/data/pkg/writer/geoip2.go

@@ -0,0 +1,184 @@
+package writer
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+	"go4.org/netipx"
+)
+
+// WriteGeoIP2TestDB writes GeoIP2 test mmdb files.
+func (w *Writer) WriteGeoIP2TestDB() error {
+	dbTypes := []string{
+		"GeoIP-Anonymous-Plus",
+		"GeoIP2-Anonymous-IP",
+		"GeoIP2-City",
+		"GeoIP2-Connection-Type",
+		"GeoIP2-Country",
+		"GeoIP2-DensityIncome",
+		"GeoIP2-Domain",
+		"GeoIP2-Enterprise",
+		"GeoIP2-IP-Risk",
+		"GeoIP2-ISP",
+		"GeoIP2-Precision-Enterprise",
+		"GeoIP2-Static-IP-Score",
+		"GeoIP2-User-Count",
+		"GeoLite2-ASN",
+		"GeoLite2-City",
+		"GeoLite2-Country",
+	}
+
+	for _, dbType := range dbTypes {
+		languages := []string{"en"}
+		description := map[string]string{
+			"en": strings.ReplaceAll(dbType, "-", " ") +
+				" Test Database (fake GeoIP2 data, for example purposes only)",
+		}
+
+		if dbType == "GeoIP2-City" {
+			languages = append(languages, "zh")
+			description["zh"] = "小型数据库"
+		}
+
+		dbWriter, err := mmdbwriter.New(
+			mmdbwriter.Options{
+				DatabaseType:        dbType,
+				Description:         description,
+				DisableIPv4Aliasing: false,
+				IPVersion:           6,
+				Languages:           languages,
+				RecordSize:          28,
+			},
+		)
+		if err != nil {
+			return fmt.Errorf("creating mmdbwriter: %w", err)
+		}
+
+		if dbType == "GeoIP2-Anonymous-IP" || dbType == "GeoIP-Anonymous-Plus" {
+			if err := populateAllNetworks(dbWriter); err != nil {
+				return fmt.Errorf("inserting all networks: %w", err)
+			}
+		}
+
+		jsonFileName := dbType + "-Test.json"
+		if err := w.insertJSON(dbWriter, jsonFileName); err != nil {
+			return fmt.Errorf("inserting json: %w", err)
+		}
+
+		dbFileName := dbType + "-Test.mmdb"
+		if err := w.write(dbWriter, dbFileName); err != nil {
+			return fmt.Errorf("writing database: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// insertJSON reads and parses a json file into mmdbtypes values and inserts
+// them into the mmdbwriter tree.
+func (w *Writer) insertJSON(dbWriter *mmdbwriter.Tree, fileName string) error {
+	file, err := os.Open(filepath.Clean(filepath.Join(w.source, fileName)))
+	if err != nil {
+		return fmt.Errorf("opening json file: %w", err)
+	}
+	defer file.Close()
+
+	var data []map[string]any
+	if err := json.NewDecoder(file).Decode(&data); err != nil {
+		return fmt.Errorf("decoding json file: %w", err)
+	}
+
+	for _, record := range data {
+		for k, v := range record {
+			prefix, err := netip.ParsePrefix(k)
+			if err != nil {
+				return fmt.Errorf("parsing ip: %w", err)
+			}
+
+			mmdbValue, err := toMMDBType(prefix.String(), v)
+			if err != nil {
+				return fmt.Errorf("converting value to mmdbtype: %w", err)
+			}
+
+			err = dbWriter.Insert(
+				netipx.PrefixIPNet(prefix),
+				mmdbValue,
+			)
+			if err != nil {
+				return fmt.Errorf("inserting ip: %w", err)
+			}
+		}
+	}
+	return nil
+}
+
+// toMMDBType key converts field values read from json into their corresponding mmdbtype.DataType.
+// It makes some assumptions for numeric types based on previous knowledge about field types.
+func toMMDBType(key string, value any) (mmdbtype.DataType, error) {
+	switch v := value.(type) {
+	case bool:
+		return mmdbtype.Bool(v), nil
+	case string:
+		return mmdbtype.String(v), nil
+	case map[string]any:
+		m := mmdbtype.Map{}
+		for innerKey, val := range v {
+			innerVal, err := toMMDBType(innerKey, val)
+			if err != nil {
+				return nil, fmt.Errorf("parsing mmdbtype.Map for key %q: %w", key, err)
+			}
+			m[mmdbtype.String(innerKey)] = innerVal
+		}
+		return m, nil
+	case []any:
+		s := mmdbtype.Slice{}
+		for _, val := range v {
+			innerVal, err := toMMDBType(key, val)
+			if err != nil {
+				return nil, fmt.Errorf("parsing mmdbtype.Slice for key %q: %w", key, err)
+			}
+			s = append(s, innerVal)
+		}
+		return s, nil
+	case float64:
+		switch key {
+		case "accuracy_radius", "anonymizer_confidence", "confidence", "metro_code":
+			return mmdbtype.Uint16(v), nil
+		case "autonomous_system_number", "average_income",
+			"geoname_id", "ipv4_24", "ipv4_32", "ipv6_32",
+			"ipv6_48", "ipv6_64", "population_density":
+			return mmdbtype.Uint32(v), nil
+		case "ip_risk", "latitude", "longitude", "score",
+			"static_ip_score":
+			return mmdbtype.Float64(v), nil
+		default:
+			return nil, fmt.Errorf("unsupported numeric type for key %q: %T", key, value)
+		}
+	default:
+		return nil, fmt.Errorf("unsupported type for key %q: %T", key, value)
+	}
+}
+
+// populate all networks inserts all networks into the writer with an empty map value.
+func populateAllNetworks(w *mmdbwriter.Tree) error {
+	defaultNet, err := netip.ParsePrefix("::/0")
+	if err != nil {
+		return fmt.Errorf("parsing ip: %w", err)
+	}
+
+	err = w.Insert(
+		netipx.PrefixIPNet(defaultNet),
+		mmdbtype.Map{},
+	)
+	if err != nil {
+		return fmt.Errorf("inserting ip: %w", err)
+	}
+
+	return nil
+}

data/test/data/pkg/writer/ip.go

@@ -0,0 +1,39 @@
+package writer
+
+import (
+	"fmt"
+	"net/netip"
+
+	"go4.org/netipx"
+)
+
+// parseIPRange takes IP addresses in string presentation form that represent a
+// range and returns an IP range.
+func parseIPRange(from, to string) (netipx.IPRange, error) {
+	startIP, err := netip.ParseAddr(from)
+	if err != nil {
+		return netipx.IPRange{}, fmt.Errorf("parsing %s as an IP: %w", from, err)
+	}
+	endIP, err := netip.ParseAddr(to)
+	if err != nil {
+		return netipx.IPRange{}, fmt.Errorf("parsing %s as an IP: %w", to, err)
+	}
+	ipRange := netipx.IPRangeFrom(startIP, endIP)
+	if !ipRange.IsValid() {
+		return netipx.IPRange{}, fmt.Errorf("%s-%s is an invalid IP range", startIP, endIP)
+	}
+	return ipRange, nil
+}
+
+// parseIPSlice parses a slice of IP address strings and returns a slice of netip.Prefix.
+func parseIPSlice(ipAddresses []string) ([]netip.Prefix, error) {
+	var addrs []netip.Prefix
+	for _, ip := range ipAddresses {
+		addr, err := netip.ParsePrefix(ip)
+		if err != nil {
+			return nil, fmt.Errorf("parsing %s as an IP: %w", ip, err)
+		}
+		addrs = append(addrs, addr)
+	}
+	return addrs, nil
+}

data/test/data/pkg/writer/maxmind.go

@@ -0,0 +1,246 @@
+package writer
+
+import (
+	"fmt"
+	"net/netip"
+
+	"github.com/maxmind/mmdbwriter"
+	"github.com/maxmind/mmdbwriter/mmdbtype"
+	"go4.org/netipx"
+)
+
+// WriteIPv4TestDB writes mmdb files for an ip range between 1.1.1.1 and 1.1.1.32
+// with various record sizes.
+func (w *Writer) WriteIPv4TestDB() error {
+	ipRange, err := parseIPRange("1.1.1.1", "1.1.1.32")
+	if err != nil {
+		return fmt.Errorf("parsing ip range: %w", err)
+	}
+
+	for _, recordSize := range []int{24, 28, 32} {
+		err := w.writeMaxMindTestDB(
+			recordSize,
+			[]netipx.IPRange{ipRange},
+			"ipv4",
+		)
+		if err != nil {
+			return fmt.Errorf("writing test database: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// WriteIPv6TestDB writes mmdb files for an ip range between ::1:ffff:ffff and ::2:0000:0059
+// with various record sizes.
+func (w *Writer) WriteIPv6TestDB() error {
+	ipRange, err := parseIPRange("::1:ffff:ffff", "::2:0000:0059")
+	if err != nil {
+		return fmt.Errorf("parsing ip range: %w", err)
+	}
+
+	for _, recordSize := range []int{24, 28, 32} {
+		err := w.writeMaxMindTestDB(
+			recordSize,
+			[]netipx.IPRange{ipRange},
+			"ipv6",
+		)
+		if err != nil {
+			return fmt.Errorf("writing test database: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// WriteMixedIPTestDB writes mmdb files for a mixed ip version range between ::1:ffff:ffff and
+// ::2:0000:0059
+// with various record sizes.
+func (w *Writer) WriteMixedIPTestDB() error {
+	ipv6Range, err := parseIPRange("::1:ffff:ffff", "::2:0000:0059")
+	if err != nil {
+		return fmt.Errorf("parsing ip range: %w", err)
+	}
+
+	ipv4Range, err := parseIPRange("1.1.1.1", "1.1.1.32")
+	if err != nil {
+		return fmt.Errorf("parsing ip range: %w", err)
+	}
+
+	for _, recordSize := range []int{24, 28, 32} {
+		err := w.writeMaxMindTestDB(
+			recordSize,
+			[]netipx.IPRange{ipv6Range, ipv4Range},
+			"mixed",
+		)
+		if err != nil {
+			return fmt.Errorf("writing test database: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// writeMaxMindTestDB writes test mmdb files.
+func (w *Writer) writeMaxMindTestDB(
+	recordSize int,
+	ipRange []netipx.IPRange,
+	ipVersionName string,
+) error {
+	ipVersion := 6
+	if ipRange[0].From().Is4() {
+		ipVersion = 4
+	}
+
+	metadata := map[string]string{}
+	metadata["en"] = "Test Database"
+	metadata["zh"] = "Test Database Chinese"
+
+	dbWriter, err := mmdbwriter.New(
+		mmdbwriter.Options{
+			DatabaseType:        "Test",
+			Description:         metadata,
+			DisableIPv4Aliasing: ipVersion == 4,
+			IPVersion:           ipVersion,
+			Languages:           []string{"en", "zh"},
+			RecordSize:          recordSize,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("creating mmdbwriter: %w", err)
+	}
+
+	for _, ir := range ipRange {
+		for _, prefix := range ir.Prefixes() {
+			ipString := prefix.Addr().String()
+			if ipVersion == 6 && prefix.Addr().Is4() {
+				ipString = "::" + ipString
+			}
+
+			err := dbWriter.Insert(
+				netipx.PrefixIPNet(prefix),
+				mmdbtype.Map{
+					"ip": mmdbtype.String(ipString),
+				},
+			)
+			if err != nil {
+				return fmt.Errorf("inserting ip: %w", err)
+			}
+		}
+	}
+
+	fileName := fmt.Sprintf("MaxMind-DB-test-%s-%d.mmdb", ipVersionName, recordSize)
+	if err := w.write(dbWriter, fileName); err != nil {
+		return fmt.Errorf("writing database: %w", err)
+	}
+
+	return nil
+}
+
+// WriteNoIPv4TestDB writes an mmdb file with no ipv4 records.
+func (w *Writer) WriteNoIPv4TestDB() error {
+	dbWriter, err := mmdbwriter.New(
+		mmdbwriter.Options{
+			DatabaseType: "MaxMind DB No IPv4 Search Tree",
+			Description: map[string]string{
+				"en": "MaxMind DB No IPv4 Search Tree",
+			},
+			DisableIPv4Aliasing:     true,
+			IncludeReservedNetworks: true,
+			IPVersion:               6,
+			Languages:               []string{"en"},
+			RecordSize:              24,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("creating mmdbwriter: %w", err)
+	}
+
+	addr, err := netip.ParsePrefix("::/64")
+	if err != nil {
+		return fmt.Errorf("parsing ip: %w", err)
+	}
+
+	err = dbWriter.Insert(
+		netipx.PrefixIPNet(addr),
+		mmdbtype.String(addr.String()),
+	)
+	if err != nil {
+		return fmt.Errorf("inserting ip: %w", err)
+	}
+
+	if err := w.write(dbWriter, "MaxMind-DB-no-ipv4-search-tree.mmdb"); err != nil {
+		return fmt.Errorf("writing database: %w", err)
+	}
+	return nil
+}
+
+// WriteNoMapTestDB writes an mmdb file where each record points to
+// a string value.
+func (w *Writer) WriteNoMapTestDB() error {
+	dbWriter, err := mmdbwriter.New(
+		mmdbwriter.Options{
+			DatabaseType: "MaxMind DB String Value Entries",
+			Description: map[string]string{
+				"en": "MaxMind DB String Value Entries (no maps or arrays as values)",
+			},
+			IPVersion:  4,
+			Languages:  []string{"en"},
+			RecordSize: 24,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("creating mmdbwriter: %w", err)
+	}
+
+	ipRange, err := parseIPRange("1.1.1.1", "1.1.1.32")
+	if err != nil {
+		return fmt.Errorf("parsing ip range: %w", err)
+	}
+
+	for _, prefix := range ipRange.Prefixes() {
+		err := dbWriter.Insert(
+			netipx.PrefixIPNet(prefix),
+			mmdbtype.String(prefix.String()),
+		)
+		if err != nil {
+			return fmt.Errorf("inserting ip: %w", err)
+		}
+	}
+
+	if err := w.write(dbWriter, "MaxMind-DB-string-value-entries.mmdb"); err != nil {
+		return fmt.Errorf("writing database: %w", err)
+	}
+	return nil
+}
+
+// WriteMetadataPointersTestDB writes an mmdb file with metadata pointers allowed.
+func (w *Writer) WriteMetadataPointersTestDB() error {
+	repeatedString := "Lots of pointers in metadata"
+	dbWriter, err := mmdbwriter.New(
+		mmdbwriter.Options{
+			DatabaseType: repeatedString,
+			Description: map[string]string{
+				"en": repeatedString,
+				"es": repeatedString,
+				"zh": repeatedString,
+			},
+			DisableIPv4Aliasing: true,
+			IPVersion:           6,
+			Languages:           []string{"en", "es", "zh"},
+			RecordSize:          24,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("creating mmdbwriter: %w", err)
+	}
+
+	if err := populateAllNetworks(dbWriter); err != nil {
+		return fmt.Errorf("inserting all networks: %w", err)
+	}
+
+	if err := w.write(dbWriter, "MaxMind-DB-test-metadata-pointers.mmdb"); err != nil {
+		return fmt.Errorf("writing database: %w", err)
+	}
+	return nil
+}