geoip2_c 0.3.2 → 0.3.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (33) hide show
  1. checksums.yaml +5 -5
  2. data/.travis.yml +3 -4
  3. data/README.md +21 -3
  4. data/docker-compose.yml +15 -1
  5. data/dockerfiles/{Dockerfile-ruby2.1 → Dockerfile-ruby2.5} +1 -1
  6. data/dockerfiles/{Dockerfile-ruby2.2 → Dockerfile-ruby2.6} +1 -1
  7. data/ext/geoip2/geoip2.c +5 -3
  8. data/ext/geoip2/libmaxminddb/.gitignore +3 -0
  9. data/ext/geoip2/libmaxminddb/.travis.yml +24 -2
  10. data/ext/geoip2/libmaxminddb/Changes.md +39 -0
  11. data/ext/geoip2/libmaxminddb/README.dev.md +41 -30
  12. data/ext/geoip2/libmaxminddb/README.md +3 -3
  13. data/ext/geoip2/libmaxminddb/bin/Makefile.am +5 -0
  14. data/ext/geoip2/libmaxminddb/bin/mmdblookup.c +333 -15
  15. data/ext/geoip2/libmaxminddb/configure.ac +5 -5
  16. data/ext/geoip2/libmaxminddb/dev-bin/ppa-release.sh +8 -5
  17. data/ext/geoip2/libmaxminddb/dev-bin/release.sh +7 -0
  18. data/ext/geoip2/libmaxminddb/dev-bin/valgrind-all.pl +10 -3
  19. data/ext/geoip2/libmaxminddb/include/maxminddb.h +11 -2
  20. data/ext/geoip2/libmaxminddb/projects/VS12/libmaxminddb.vcxproj +3 -1
  21. data/ext/geoip2/libmaxminddb/projects/VS12/libmaxminddb.vcxproj.filters +7 -1
  22. data/ext/geoip2/libmaxminddb/src/Makefile.am +18 -2
  23. data/ext/geoip2/libmaxminddb/src/data-pool.c +180 -0
  24. data/ext/geoip2/libmaxminddb/src/data-pool.h +52 -0
  25. data/ext/geoip2/libmaxminddb/src/maxminddb.c +58 -48
  26. data/ext/geoip2/libmaxminddb/t/Makefile.am +6 -2
  27. data/ext/geoip2/libmaxminddb/t/bad_databases_t.c +1 -0
  28. data/ext/geoip2/libmaxminddb/t/basic_lookup_t.c +35 -0
  29. data/ext/geoip2/libmaxminddb/t/data-pool-t.c +374 -0
  30. data/ext/geoip2/libmaxminddb/t/external_symbols_t.pl +106 -0
  31. data/lib/geoip2/version.rb +1 -1
  32. metadata +9 -7
  33. data/dockerfiles/Dockerfile-ruby2.3 +0 -8
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 7d4407340c5c98633cae71cbc6b2c69cfb85a315
4
- data.tar.gz: 35e0cc0e59278f9d6ce2f5c8079dff2a5a443ed0
2
+ SHA256:
3
+ metadata.gz: fa4074dbeab86f3881d576390460438107cd9d3d156d1a3da9813b2f5dc1f890
4
+ data.tar.gz: 94a44a7f48e4b69742c659b473b60baad893df2cf938db6e49d7ee7388a6fbe4
5
5
  SHA512:
6
- metadata.gz: 03a92ddb4b9d5097ea58a4a734a2bd4a7dc578e2afb01ad735211a6f71eeb139d2ce7cfab3022c90bdd0fded7e79799a89ece649c9886d32cc58d43ebed9bec5
7
- data.tar.gz: 92a70552ec7f8f3583e49574f88116a19f9575f6b2948889d2e0168097a71cfa9e9ee4bb6809abeb17bc5767b6108691fd3876040c778063f11c7d52b88847d2
6
+ metadata.gz: 2a338f192553f75914d5c5bfe72c26be5aea555df7c6416406751521f35445d36c06c99c0ea3aa58f11b9f635dc8f4e8f85a5ac2b03f402be8c9295d655ff41b
7
+ data.tar.gz: 758c468308389d460bdf3633afb824079188ce3ecc4cc05e2f76365c3adcbd3cbf158d6129cade3627177dda6f238b2e29d5b77bed044f2493b6587fcb577941
@@ -4,12 +4,11 @@ services:
4
4
 
5
5
  env:
6
6
  global:
7
- DOCKER_COMPOSE_VERSION: 1.8.1
7
+ DOCKER_COMPOSE_VERSION: 1.18.0
8
8
  matrix:
9
- - TARGET_RUBY_VERSION=2.1
10
- - TARGET_RUBY_VERSION=2.2
11
- - TARGET_RUBY_VERSION=2.3
12
9
  - TARGET_RUBY_VERSION=2.4
10
+ - TARGET_RUBY_VERSION=2.5
11
+ - TARGET_RUBY_VERSION=2.6
13
12
 
14
13
  before_install:
15
14
  #- sudo apt-get update
data/README.md CHANGED
@@ -15,6 +15,24 @@ This library uses `Hash#dig` and `Array#dig`.
15
15
  If you want to use this library with Ruby2.1 or Ruby2.2, you can install a gem which adds support `#dig` method.
16
16
  But I don't want to add extra dependency to this library, so I decided not to add extra dependency to support old Ruby versions.
17
17
 
18
+ ## Build requirements
19
+
20
+ Need ruby.h to build this extension, so you must install it before build this extension.
21
+
22
+ Debian GNU Linux / Ubuntu:
23
+
24
+ ```
25
+ $ sudo apt install -y build-essential automake autoconf libtool
26
+ ```
27
+
28
+ CentOS:
29
+
30
+ ```
31
+ $ sudo yum groupinstall -y "Development Tools"
32
+ ```
33
+
34
+ **NOTE**: If you want to use libmaxminddb provided as a deb/rpm package, you can install libmaxminddb-dev or libmaxminddb-devel at your own risk.
35
+
18
36
  ## Installation
19
37
 
20
38
  Add this line to your application's Gemfile:
@@ -36,10 +54,10 @@ Or install it yourself as:
36
54
  ```ruby
37
55
  require "geoip2"
38
56
 
39
- db = GeoIP2::Databaes.new("/path/to/GeoLite2-City.mmdb")
57
+ db = GeoIP2::Database.new("GeoLite2-City.mmdb")
40
58
  result = db.lookup("66.102.9.80")
41
- retult.get_value("city", "names", "en") # => "Mountain View"
42
- retult.dig("city", "names", "en") # => "Mountain View"
59
+ result.get_value("city", "names", "en") # => "Mountain View"
60
+ result.dig("city", "names", "en") # => "Mountain View"
43
61
  ```
44
62
 
45
63
  ## Development
@@ -1,5 +1,19 @@
1
- version: "2"
1
+ version: "3"
2
2
  services:
3
+ test-ruby2.6:
4
+ build:
5
+ context: .
6
+ dockerfile: dockerfiles/Dockerfile-ruby2.6
7
+ command: tail -f /dev/null
8
+ environment:
9
+ - BUNDLE_GEMFILE=Gemfile
10
+ test-ruby2.5:
11
+ build:
12
+ context: .
13
+ dockerfile: dockerfiles/Dockerfile-ruby2.5
14
+ command: tail -f /dev/null
15
+ environment:
16
+ - BUNDLE_GEMFILE=Gemfile
3
17
  test-ruby2.4:
4
18
  build:
5
19
  context: .
@@ -1,4 +1,4 @@
1
- FROM ruby:2.1-alpine
1
+ FROM ruby:2.5-alpine
2
2
 
3
3
  ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
4
 
@@ -1,4 +1,4 @@
1
- FROM ruby:2.2-alpine
1
+ FROM ruby:2.6-alpine
2
2
 
3
3
  ENV BUNDLE_GEMFILE=$BUNDLE_GEMFILE
4
4
 
@@ -30,13 +30,15 @@ static VALUE rb_geoip2_lr_to_h(VALUE self);
30
30
  static const rb_data_type_t rb_mmdb_type = {
31
31
  "geoip2/mmdb", {
32
32
  0, mmdb_free, 0,
33
- }, NULL, NULL
33
+ }, NULL, NULL,
34
+ RUBY_TYPED_FREE_IMMEDIATELY
34
35
  };
35
36
 
36
37
  static const rb_data_type_t rb_lookup_result_type = {
37
38
  "geoip2/lookup_result", {
38
- 0, 0, 0,
39
- }, NULL, NULL
39
+ 0, RUBY_DEFAULT_FREE, 0,
40
+ }, NULL, NULL,
41
+ RUBY_TYPED_FREE_IMMEDIATELY
40
42
  };
41
43
 
42
44
  static void
@@ -2,6 +2,7 @@
2
2
  *.lo
3
3
  *.o
4
4
  *.so
5
+ *.swp
5
6
  */.deps
6
7
  */.libs
7
8
  *~
@@ -24,9 +25,11 @@
24
25
  /man
25
26
  /missing
26
27
  /src/libmaxminddb.pc
28
+ /src/test-data-pool
27
29
  /t/*.log
28
30
  /t/*.trs
29
31
  /t/*_t
32
+ /t/*-t
30
33
  /test-driver
31
34
  Makefile
32
35
  Makefile.in
@@ -15,18 +15,40 @@ addons:
15
15
  language: c
16
16
  matrix:
17
17
  include:
18
+ # Why do we have -Wno-unused-parameter? Because libtap has unused
19
+ # parameters unfortunately.
20
+ #
21
+ # Why -Wno-unused-command-line-argument? Because Clang on OSX does not like
22
+ # -pthread to be given during linking (while gcc says it should be). We
23
+ # could alternatively work around this via conditionals in automake, but
24
+ # that seems uglier. Note we specify this for gcc on OSX as well as gcc
25
+ # there is a frontend for Clang.
18
26
  - os: linux
19
27
  compiler: gcc
20
28
  env: MMCFLAGS="-std=c99 -Wall -Wextra -Werror -Wno-unused-parameter"
21
29
  - os: osx
22
30
  compiler: gcc
23
- env: MMCFLAGS="-std=c99 -Wall -Wextra -Wno-unused-parameter"
31
+ env: MMCFLAGS="-std=c99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unused-command-line-argument"
24
32
  - os: linux
25
33
  compiler: clang
26
34
  env: MMCFLAGS="-std=c99 -Wall -Wextra -Werror -Wno-unused-parameter"
27
35
  - os: osx
28
36
  compiler: clang
29
- env: MMCFLAGS="-std=c99 -Wall -Wextra -Wno-unused-parameter"
37
+ env: MMCFLAGS="-std=c99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unused-command-line-argument"
38
+ # The same, but POSIX.1-2001. We default to POSIX.1-2008 but want to remain
39
+ # compatible with POSIX.1-2001.
40
+ - os: linux
41
+ compiler: gcc
42
+ env: MMCFLAGS="-std=c99 -Wall -Wextra -Werror -Wno-unused-parameter -D_POSIX_C_SOURCE=200112L"
43
+ - os: osx
44
+ compiler: gcc
45
+ env: MMCFLAGS="-std=c99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unused-command-line-argument -D_POSIX_C_SOURCE=200112L"
46
+ - os: linux
47
+ compiler: clang
48
+ env: MMCFLAGS="-std=c99 -Wall -Wextra -Werror -Wno-unused-parameter -D_POSIX_C_SOURCE=200112L"
49
+ - os: osx
50
+ compiler: clang
51
+ env: MMCFLAGS="-std=c99 -Wall -Wextra -Werror -Wno-unused-parameter -Wno-unused-command-line-argument -D_POSIX_C_SOURCE=200112L"
30
52
  env:
31
53
  global:
32
54
  - secure: a2pXNVW/lGrMdJTal+pzH0J5N69RdBErwe2dHU2xDnOeNcDYmiSxmU4Fw52KPYXFzdlR9GuEyZtHtesRQthSHpylcIvfJJCih7EvwbNQ5pfplpT5ri2PKPFWB11ebr6vG23Ucgc5lrqHdgIgv+QtqVmW1IDf0hq62itMpu4MwcQ=
@@ -1,3 +1,42 @@
1
+ ## 1.3.2 - 2018-01-17
2
+
3
+ * Allocate memory for `MMDB_entry_data_list_s` structs in separate chunks
4
+ rather than one large chunk. This simplifies accessing memory in
5
+ `MMDB_get_entry_data_list()` and increases performance. It builds on the
6
+ changes in 1.3.0 and 1.3.1.
7
+ * We no longer export `data_pool_*` symbols. These are internal functions
8
+ but we were previously exporting them. Pull request by Faidon Liambotis.
9
+ GitHub #162.
10
+ * Build with POSIX.1-2008 by default if the system supports it. This allows
11
+ use of `open()` with `O_CLOEXEC`. We retain support for systems that
12
+ provide only POSIX.1-2001.
13
+ * Open the database with the `O_CLOEXEC` flag if the system provides it.
14
+ This avoids cases where we could leak fds when called in multi-threaded
15
+ programs that `fork()` and `exec()`. Original report and PR by Brandon L
16
+ Black.
17
+ * Added a test to ensure we export only intended symbols (e.g. MMDB_*).
18
+
19
+
20
+ ## 1.3.1 - 2017-11-24
21
+
22
+ * Fix build problems related to `rpl_malloc()`. Pull request by Rainer
23
+ Gerhards. GitHub #152.
24
+ * Fix a race to set and read data in a field on the `MMDB_s` struct
25
+ (`ipv4_start_node`). GitHub #153.
26
+ * Fix cases of invalid memory access when using
27
+ `MMDB_get_entry_data_list()`. This was introduced in 1.3.0 and occurred
28
+ when performing large lookups. GitHub #153.
29
+
30
+
31
+ ## 1.3.0 - 2017-11-10
32
+
33
+ * Perform fewer memory allocations in `MMDB_get_entry_data_list()`. This
34
+ significantly improves its performance. GitHub #147.
35
+ * Fix `mmdblookup`'s build epoch reporting on some systems. Big endian
36
+ systems with a 32-bit `time_t` no longer show a database build date of
37
+ 1970-01-01 00:00:00. Pull request by Rainer Jung. GitHub #143.
38
+
39
+
1
40
  ## 1.2.1 - 2017-05-15
2
41
 
3
42
  * Use autoconf to check the system's endianness rather than trying to do this
@@ -4,23 +4,14 @@ We release by uploading the tarball to GitHub, uploading Ubuntu PPAs, and by
4
4
  updating the Homebrew recipe for this library.
5
5
 
6
6
  ## Creating the release tarball
7
+ You may want to refer to the section about prerequisites.
7
8
 
9
+ * Check whether there are any open issues to fix while you're doing this.
8
10
  * Update `Changes.md` to include specify the new version, today's date, and
9
- list relevant changes.
10
- * Run `./dev-bin/make-release.sh` to update various files in the distro, our
11
- GitHub pages, creates and pushes a tag, etc.
12
- * Run `make safedist` in the checkout directory to create a tarball.
13
-
14
- ## GitHub
15
-
16
- * Go to https://github.com/maxmind/libmaxminddb/releases
17
- * GitHub will already have a release for the tag you just created.
18
- * Click on the new tag new.
19
- * Click on "Edit tag".
20
- * Give the release a title summarizing the changes.
21
- * Paste in the changes from the `Changes.md` file as the body.
22
- * Attach the release tarball you just created.
23
- * Click "Update release"
11
+ list relevant changes. Commit this.
12
+ * Run `./dev-bin/release.sh` to update various files in the distro, our
13
+ GitHub pages, and creates a GitHub release with the tarball.
14
+ * Check the release looks good on both GitHub and launchpad.net.
24
15
 
25
16
  ## PPA
26
17
 
@@ -33,26 +24,46 @@ The PPA release script is at `dev-bin/ppa-release.sh`. Running it should
33
24
  guide you though the release, although it may require some changes to run on
34
25
  configurations different than Greg's machine.
35
26
 
36
- Pre-script PPA release process:
27
+ Check whether any new Ubuntu versions need to be listed in this script
28
+ before running it.
37
29
 
38
- 1. git co ubuntu-ppa
39
- 2. git merge `<TAG>`
40
- 3. dch -i
41
- * Add new entry for wily (or whatever the most recent Ubuntu release
42
- is. Follow existing PPA versioning style.
43
- 4. git commit to add the debian changelog
44
- 5. gbp buildpackage -S
45
- 6. dput ppa:maxmind/ppa ../libmaxminddb_`<TAG>`-`<DEB VERSION>`_source.changes
46
- 7. git push
47
-
48
- If 5 was successful, modify debian/changelog and repeat 5 & 6 for trusty and
49
- precise. Note that you can skip step #4 for subsequent uploads by adding
50
- `--git-ignore-new` when you call `gbp`.
30
+ You should run it from `master`.
51
31
 
52
32
  ## Homebrew
53
33
 
54
- * Go to https://github.com/Homebrew/homebrew/blob/master/Library/Formula/libmaxminddb.rb
34
+ * Go to https://github.com/Homebrew/homebrew-core/edit/master/Formula/libmaxminddb.rb
55
35
  * Edit the file to update the url and sha256. You can get the sha256 for the
56
36
  tarball with the `sha256sum` command line utility.
57
37
  * Make a commit with the summary `libmaxminddb <VERSION>`
58
38
  * Submit a PR with the changes you just made.
39
+
40
+ # Prerequisites for releasing
41
+
42
+ * Required packages (Ubuntu Artful): vim git-core dput build-essential
43
+ autoconf automake libtool git-buildpackage libfile-slurp-perl pandoc
44
+ dirmngr libfile-slurp-tiny-perl libdatetime-perl debhelper dh-autoreconf
45
+ libipc-run3-perl libtest-output-perl devscripts
46
+ * Install [hub](https://github.com/github/hub/releases). (Using `./install`
47
+ from the tarball is fine)
48
+ * GitHub ssh key (e.g. in `~/.ssh/id_rsa`)
49
+ * Git config (e.g. `~/.gitconfig`)
50
+ * Import your GPG secret key (or create one if you don't have a suitable
51
+ one)
52
+ * `gpg --import /path/to/key`
53
+ * `gpg --edit-key KEYID` and trust it ultimately
54
+ * Ensure it shows with `gpg --list-secret-keys`
55
+ * You need to be invited to the launchpad.net MaxMind organization on your
56
+ launchpad.net account.
57
+ * You need your GPG key listed on your launchpad.net account
58
+ * You can add it in the web interface. It wants the output of
59
+ `gpg --fingerprint`.
60
+ * Part of the instructions involve having your key published on the
61
+ Ubuntu keyserver:
62
+ `gpg --keyserver keyserver.ubuntu.com --send-keys KEYID`
63
+ * You'll get an email with an encrypted payload that you need to decrypt
64
+ and follow the link to confirm it.
65
+ * Ensure `dch` knows your name and email. Refer to its man page for how to
66
+ tell it this. One way is to set the `DEBFULLNAME` and `DEBEMAIL`
67
+ environment variables. These should match your GPG key's name and email
68
+ exactly. This is what gets used in the Debian changelog as well as
69
+ defines what GPG key to use.
@@ -20,7 +20,7 @@ This library is licensed under the Apache License, Version 2.
20
20
  ## From a Tarball
21
21
 
22
22
  This code is known to work with GCC 4.4+ and clang 3.2+. It should also work
23
- on other compilers that supports C99, POSIX 2011.11, and the `-fms-extensions
23
+ on other compilers that supports C99, POSIX.1-2001, and the `-fms-extensions
24
24
  flag` (or equivalent). The latter is needed to allow an anonymous union in a
25
25
  structure.
26
26
 
@@ -76,8 +76,8 @@ APT sources, run:
76
76
 
77
77
  Then install the packages by running:
78
78
 
79
- $ sudo aptitude update
80
- $ sudo aptitude install libmaxminddb0 libmaxminddb-dev mmdb-bin
79
+ $ sudo apt update
80
+ $ sudo apt install libmaxminddb0 libmaxminddb-dev mmdb-bin
81
81
 
82
82
  ## On OS X via Homebrew
83
83
 
@@ -3,3 +3,8 @@ include $(top_srcdir)/common.mk
3
3
  AM_LDFLAGS = $(top_builddir)/src/libmaxminddb.la
4
4
 
5
5
  bin_PROGRAMS = mmdblookup
6
+
7
+ if !WINDOWS
8
+ AM_CPPFLAGS += -pthread
9
+ AM_LDFLAGS += -pthread
10
+ endif
@@ -4,6 +4,10 @@
4
4
  #include "maxminddb.h"
5
5
  #include <errno.h>
6
6
  #include <getopt.h>
7
+ #ifndef _WIN32
8
+ #include <pthread.h>
9
+ #endif
10
+ #include <stdbool.h>
7
11
  #include <stdio.h>
8
12
  #include <stdlib.h>
9
13
  #include <string.h>
@@ -18,16 +22,27 @@
18
22
  #include <unistd.h>
19
23
  #endif
20
24
 
21
- #define LOCAL
25
+ #define LOCAL static
26
+ #define NO_PROTO
22
27
 
23
28
  /* *INDENT-OFF* */
24
29
  /* --prototypes automatically generated by dev-bin/regen-prototypes.pl - don't remove this comment */
25
30
  LOCAL void usage(char *program, int exit_code, const char *error);
26
- LOCAL const char **get_options(int argc, char **argv, char **mmdb_file,
27
- char **ip_address, int *verbose, int *iterations,
28
- int *lookup_path_length);
31
+ LOCAL const char **get_options(
32
+ int argc,
33
+ char **argv,
34
+ char **mmdb_file,
35
+ char **ip_address,
36
+ int *verbose,
37
+ int *iterations,
38
+ int *lookup_path_length,
39
+ int *const thread_count,
40
+ char **const ip_file);
29
41
  LOCAL MMDB_s open_or_die(const char *fname);
30
42
  LOCAL void dump_meta(MMDB_s *mmdb);
43
+ LOCAL bool lookup_from_file(MMDB_s *const mmdb,
44
+ char const *const ip_file,
45
+ bool const dump);
31
46
  LOCAL int lookup_and_print(MMDB_s *mmdb, const char *ip_address,
32
47
  const char **lookup_path,
33
48
  int lookup_path_length);
@@ -37,6 +52,17 @@ LOCAL void random_ipv4(char *ip);
37
52
  /* --prototypes end - don't remove this comment-- */
38
53
  /* *INDENT-ON* */
39
54
 
55
+ #ifndef _WIN32
56
+ // These aren't with the automatically generated prototypes as we'd lose the
57
+ // enclosing macros.
58
+ static bool start_threaded_benchmark(
59
+ MMDB_s *const mmdb,
60
+ int const thread_count,
61
+ int const iterations);
62
+ static long double get_time(void);
63
+ static void *thread(void *arg);
64
+ #endif
65
+
40
66
  int main(int argc, char **argv)
41
67
  {
42
68
  char *mmdb_file = NULL;
@@ -44,10 +70,12 @@ int main(int argc, char **argv)
44
70
  int verbose = 0;
45
71
  int iterations = 0;
46
72
  int lookup_path_length = 0;
73
+ int thread_count = 0;
74
+ char *ip_file = NULL;
47
75
 
48
76
  const char **lookup_path =
49
77
  get_options(argc, argv, &mmdb_file, &ip_address, &verbose, &iterations,
50
- &lookup_path_length);
78
+ &lookup_path_length, &thread_count, &ip_file);
51
79
 
52
80
  MMDB_s mmdb = open_or_die(mmdb_file);
53
81
 
@@ -55,12 +83,43 @@ int main(int argc, char **argv)
55
83
  dump_meta(&mmdb);
56
84
  }
57
85
 
86
+ // The benchmarking and lookup from file modes are hidden features mainly
87
+ // intended for development right now. This means there are several flags
88
+ // that exist but are intentionally not mentioned in the usage or man page.
89
+
90
+ // The lookup from file mode may be useful to expose publicly in the usage,
91
+ // but we should have it respect the lookup_path functionality if we do so.
92
+ if (ip_file) {
93
+ free(lookup_path);
94
+ if (!lookup_from_file(&mmdb, ip_file, verbose == 1)) {
95
+ MMDB_close(&mmdb);
96
+ return 1;
97
+ }
98
+ MMDB_close(&mmdb);
99
+ return 0;
100
+ }
101
+
58
102
  if (0 == iterations) {
59
103
  exit(lookup_and_print(&mmdb, ip_address, lookup_path,
60
104
  lookup_path_length));
61
- } else {
62
- exit(benchmark(&mmdb, iterations));
63
105
  }
106
+
107
+ free(lookup_path);
108
+
109
+ srand( time(NULL) );
110
+
111
+ #ifndef _WIN32
112
+ if (thread_count > 0) {
113
+ if (!start_threaded_benchmark(&mmdb, thread_count, iterations)) {
114
+ MMDB_close(&mmdb);
115
+ exit(1);
116
+ }
117
+ MMDB_close(&mmdb);
118
+ exit(0);
119
+ }
120
+ #endif
121
+
122
+ exit(benchmark(&mmdb, iterations));
64
123
  }
65
124
 
66
125
  LOCAL void usage(char *program, int exit_code, const char *error)
@@ -116,9 +175,16 @@ LOCAL void usage(char *program, int exit_code, const char *error)
116
175
  exit(exit_code);
117
176
  }
118
177
 
119
- LOCAL const char **get_options(int argc, char **argv, char **mmdb_file,
120
- char **ip_address, int *verbose, int *iterations,
121
- int *lookup_path_length)
178
+ LOCAL const char **get_options(
179
+ int argc,
180
+ char **argv,
181
+ char **mmdb_file,
182
+ char **ip_address,
183
+ int *verbose,
184
+ int *iterations,
185
+ int *lookup_path_length,
186
+ int *const thread_count,
187
+ char **const ip_file)
122
188
  {
123
189
  static int help = 0;
124
190
  static int version = 0;
@@ -130,13 +196,22 @@ LOCAL const char **get_options(int argc, char **argv, char **mmdb_file,
130
196
  { "verbose", no_argument, 0, 'v' },
131
197
  { "version", no_argument, 0, 'n' },
132
198
  { "benchmark", required_argument, 0, 'b' },
199
+ #ifndef _WIN32
200
+ { "threads", required_argument, 0, 't' },
201
+ #endif
202
+ { "ip-file", required_argument, 0, 'I' },
133
203
  { "help", no_argument, 0, 'h' },
134
204
  { "?", no_argument, 0, 1 },
135
205
  { 0, 0, 0, 0 }
136
206
  };
137
207
 
138
208
  int opt_index;
139
- int opt_char = getopt_long(argc, argv, "f:i:b:vnh?", options,
209
+ #ifdef _WIN32
210
+ char const * const optstring = "f:i:b:I:vnh?";
211
+ #else
212
+ char const * const optstring = "f:i:b:t:I:vnh?";
213
+ #endif
214
+ int opt_char = getopt_long(argc, argv, optstring, options,
140
215
  &opt_index);
141
216
 
142
217
  if (-1 == opt_char) {
@@ -155,6 +230,10 @@ LOCAL const char **get_options(int argc, char **argv, char **mmdb_file,
155
230
  *iterations = strtol(optarg, NULL, 10);
156
231
  } else if ('h' == opt_char || '?' == opt_char) {
157
232
  help = 1;
233
+ } else if (opt_char == 't') {
234
+ *thread_count = strtol(optarg, NULL, 10);
235
+ } else if (opt_char == 'I') {
236
+ *ip_file = optarg;
158
237
  }
159
238
  }
160
239
 
@@ -179,7 +258,7 @@ LOCAL const char **get_options(int argc, char **argv, char **mmdb_file,
179
258
  usage(program, 1, "You must provide a filename with --file");
180
259
  }
181
260
 
182
- if (NULL == *ip_address && *iterations == 0) {
261
+ if (*ip_address == NULL && *iterations == 0 && !*ip_file) {
183
262
  usage(program, 1, "You must provide an IP address with --ip");
184
263
  }
185
264
 
@@ -229,8 +308,8 @@ LOCAL void dump_meta(MMDB_s *mmdb)
229
308
  " Languages: ";
230
309
 
231
310
  char date[40];
232
- strftime(date, 40, "%F %T UTC",
233
- gmtime((const time_t *)&mmdb->metadata.build_epoch));
311
+ const time_t epoch = (const time_t)mmdb->metadata.build_epoch;
312
+ strftime(date, 40, "%F %T UTC", gmtime(&epoch));
234
313
 
235
314
  fprintf(stdout, meta_dump,
236
315
  mmdb->metadata.node_count,
@@ -259,6 +338,109 @@ LOCAL void dump_meta(MMDB_s *mmdb)
259
338
  fprintf(stdout, "\n");
260
339
  }
261
340
 
341
+ // The input file should have one IP per line.
342
+ //
343
+ // We look up each IP.
344
+ //
345
+ // If dump is true, we dump the data for each IP to stderr. This is useful for
346
+ // comparison in that you can dump out the data for the IPs before and after
347
+ // making changes. It goes to stderr rather than stdout so that the report does
348
+ // not get included in what you will compare (since it will almost always be
349
+ // different).
350
+ //
351
+ // In addition to being useful for comparisons, this function provides a way to
352
+ // have a more deterministic set of lookups for benchmarking.
353
+ LOCAL bool lookup_from_file(MMDB_s *const mmdb,
354
+ char const *const ip_file,
355
+ bool const dump)
356
+ {
357
+ FILE *const fh = fopen(ip_file, "r");
358
+ if (!fh) {
359
+ fprintf(stderr, "fopen(): %s: %s\n", ip_file, strerror(errno));
360
+ return false;
361
+ }
362
+
363
+ clock_t const clock_start = clock();
364
+ char buf[1024] = { 0 };
365
+ // I'd normally use uint64_t, but support for it is optional in C99.
366
+ unsigned long long i = 0;
367
+ while (1) {
368
+ if (fgets(buf, sizeof(buf), fh) == NULL) {
369
+ if (!feof(fh)) {
370
+ fprintf(stderr, "fgets(): %s\n", strerror(errno));
371
+ fclose(fh);
372
+ return false;
373
+ }
374
+ if (fclose(fh) != 0) {
375
+ fprintf(stderr, "fclose(): %s\n", strerror(errno));
376
+ return false;
377
+ }
378
+ break;
379
+ }
380
+
381
+ char *ptr = buf;
382
+ while (*ptr != '\0') {
383
+ if (*ptr == '\n') {
384
+ *ptr = '\0';
385
+ break;
386
+ }
387
+ ptr++;
388
+ }
389
+ if (strlen(buf) == 0) {
390
+ continue;
391
+ }
392
+
393
+ i++;
394
+
395
+ MMDB_lookup_result_s result = lookup_or_die(mmdb, buf);
396
+ if (!result.found_entry) {
397
+ continue;
398
+ }
399
+
400
+ MMDB_entry_data_list_s *entry_data_list = NULL;
401
+ int const status = MMDB_get_entry_data_list(&result.entry,
402
+ &entry_data_list);
403
+ if (status != MMDB_SUCCESS) {
404
+ fprintf(stderr, "MMDB_get_entry_data_list(): %s\n",
405
+ MMDB_strerror(status));
406
+ fclose(fh);
407
+ MMDB_free_entry_data_list(entry_data_list);
408
+ return false;
409
+ }
410
+
411
+ if (!entry_data_list) {
412
+ fprintf(stderr, "entry_data_list is NULL\n");
413
+ fclose(fh);
414
+ return false;
415
+ }
416
+
417
+ if (dump) {
418
+ fprintf(stdout, "%s:\n", buf);
419
+ int const status = MMDB_dump_entry_data_list(stderr,
420
+ entry_data_list, 0);
421
+ if (status != MMDB_SUCCESS) {
422
+ fprintf(stderr, "MMDB_dump_entry_data_list(): %s\n",
423
+ MMDB_strerror(status));
424
+ fclose(fh);
425
+ MMDB_free_entry_data_list(entry_data_list);
426
+ return false;
427
+ }
428
+ }
429
+
430
+ MMDB_free_entry_data_list(entry_data_list);
431
+ }
432
+
433
+ clock_t const clock_diff = clock() - clock_start;
434
+ double const seconds = (double)clock_diff / CLOCKS_PER_SEC;
435
+
436
+ fprintf(
437
+ stdout,
438
+ "Looked up %llu addresses in %.2f seconds. %.2f lookups per second.\n",
439
+ i, seconds, i / seconds);
440
+
441
+ return true;
442
+ }
443
+
262
444
  LOCAL int lookup_and_print(MMDB_s *mmdb, const char *ip_address,
263
445
  const char **lookup_path,
264
446
  int lookup_path_length)
@@ -324,7 +506,6 @@ LOCAL int benchmark(MMDB_s *mmdb, int iterations)
324
506
  char ip_address[16];
325
507
  int exit_code = 0;
326
508
 
327
- srand( time(NULL) );
328
509
  clock_t time = clock();
329
510
 
330
511
  for (int i = 0; i < iterations; i++) {
@@ -395,3 +576,140 @@ LOCAL void random_ipv4(char *ip)
395
576
  snprintf(ip, 16, "%u.%u.%u.%u",
396
577
  *bytes, *(bytes + 1), *(bytes + 2), *(bytes + 3));
397
578
  }
579
+
580
+ #ifndef _WIN32
581
+ struct thread_info {
582
+ pthread_t id;
583
+ int num;
584
+ MMDB_s *mmdb;
585
+ int iterations;
586
+ };
587
+
588
+ NO_PROTO static bool start_threaded_benchmark(
589
+ MMDB_s *const mmdb,
590
+ int const thread_count,
591
+ int const iterations)
592
+ {
593
+ struct thread_info *const tinfo = calloc(thread_count,
594
+ sizeof(struct thread_info));
595
+ if (!tinfo) {
596
+ fprintf(stderr, "calloc(): %s\n", strerror(errno));
597
+ return false;
598
+ }
599
+
600
+ // Using clock() isn't appropriate for multiple threads. It's CPU time, not
601
+ // wall time.
602
+ long double const start_time = get_time();
603
+ if (start_time == -1) {
604
+ return false;
605
+ }
606
+
607
+ for (int i = 0; i < thread_count; i++) {
608
+ tinfo[i].num = i;
609
+ tinfo[i].mmdb = mmdb;
610
+ tinfo[i].iterations = iterations;
611
+
612
+ if (pthread_create(&tinfo[i].id, NULL, &thread, &tinfo[i]) != 0) {
613
+ fprintf(stderr, "pthread_create() failed\n");
614
+ free(tinfo);
615
+ return false;
616
+ }
617
+ }
618
+
619
+ for (int i = 0; i < thread_count; i++) {
620
+ if (pthread_join(tinfo[i].id, NULL) != 0) {
621
+ fprintf(stderr, "pthread_join() failed\n");
622
+ free(tinfo);
623
+ return false;
624
+ }
625
+ }
626
+
627
+ free(tinfo);
628
+
629
+ long double const end_time = get_time();
630
+ if (end_time == -1) {
631
+ return false;
632
+ }
633
+
634
+ long double const elapsed = end_time - start_time;
635
+ unsigned long long const total_ips = iterations * thread_count;
636
+ long double rate = total_ips;
637
+ if (elapsed != 0) {
638
+ rate = total_ips / elapsed;
639
+ }
640
+
641
+ fprintf(
642
+ stdout,
643
+ "Looked up %llu addresses using %d threads in %.2Lf seconds. %.2Lf lookups per second.\n",
644
+ total_ips, thread_count, elapsed, rate);
645
+
646
+ return true;
647
+ }
648
+
649
+ NO_PROTO static long double get_time(void)
650
+ {
651
+ // clock_gettime() is not present on OSX until 10.12.
652
+ #ifdef HAVE_CLOCK_GETTIME
653
+ struct timespec tp = {
654
+ .tv_sec = 0,
655
+ .tv_nsec = 0,
656
+ };
657
+ clockid_t clk_id = CLOCK_REALTIME;
658
+ #ifdef _POSIX_MONOTONIC_CLOCK
659
+ clk_id = CLOCK_MONOTONIC;
660
+ #endif
661
+ if (clock_gettime(clk_id, &tp) != 0) {
662
+ fprintf(stderr, "clock_gettime(): %s\n", strerror(errno));
663
+ return -1;
664
+ }
665
+ return tp.tv_sec + ((float)tp.tv_nsec / 1e9);
666
+ #else
667
+ time_t t = time(NULL);
668
+ if (t == (time_t)-1) {
669
+ fprintf(stderr, "time(): %s\n", strerror(errno));
670
+ return -1;
671
+ }
672
+ return (long double)t;
673
+ #endif
674
+ }
675
+
676
+ NO_PROTO static void *thread(void *arg)
677
+ {
678
+ const struct thread_info *const tinfo = arg;
679
+ if (!tinfo) {
680
+ fprintf(stderr, "thread(): %s\n", strerror(EINVAL));
681
+ return NULL;
682
+ }
683
+
684
+ char ip_address[16] = { 0 };
685
+
686
+ for (int i = 0; i < tinfo->iterations; i++) {
687
+ memset(ip_address, 0, 16);
688
+ random_ipv4(ip_address);
689
+
690
+ MMDB_lookup_result_s result = lookup_or_die(tinfo->mmdb, ip_address);
691
+ if (!result.found_entry) {
692
+ continue;
693
+ }
694
+
695
+ MMDB_entry_data_list_s *entry_data_list = NULL;
696
+ int const status = MMDB_get_entry_data_list(&result.entry,
697
+ &entry_data_list);
698
+ if (status != MMDB_SUCCESS) {
699
+ fprintf(stderr, "MMDB_get_entry_data_list(): %s\n",
700
+ MMDB_strerror(status));
701
+ MMDB_free_entry_data_list(entry_data_list);
702
+ return NULL;
703
+ }
704
+
705
+ if (!entry_data_list) {
706
+ fprintf(stderr, "entry_data_list is NULL\n");
707
+ return NULL;
708
+ }
709
+
710
+ MMDB_free_entry_data_list(entry_data_list);
711
+ }
712
+
713
+ return NULL;
714
+ }
715
+ #endif