ip2region_ext 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +34 -0
- data/Rakefile +8 -0
- data/db/ip2region.xdb +0 -0
- data/ext/ip2region/extconf.rb +8 -0
- data/ext/ip2region/xdb_searcher.c +428 -0
- data/ext/ip2region/xdb_searcher.h +149 -0
- data/lib/ip2region_ext/ip2region_ffi.rb +53 -0
- data/lib/ip2region_ext/version.rb +5 -0
- data/lib/ip2region_ext/xdb.rb +70 -0
- data/lib/ip2region_ext.rb +26 -0
- data/sig/ip2region.rbs +4 -0
- metadata +72 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 29c808556500f049f3e21908c35fbb809b992874f21f9cf81478c862d541f0f0
|
4
|
+
data.tar.gz: e40d4f499e75a22adea750e529dbaee7be7ad25b0549ba9b4c0d703596593c60
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: '048a8fdd7e4942b348ce08f1d3d962c2bdb3d64535b86471200b569ef427bb82be6a9913f6aa83462247acc6af78c8a475d8bba7ae0d15fe4492a93b3d80293c'
|
7
|
+
data.tar.gz: c89112c8ea4797e9e5288ad8c906042dd532a1e1440937ed04dba94a84877cc3c073fb78c4c0597744c5a46b5b59edfe5268b8ab936cf44543fc1c9b0657fd01
|
data/README.md
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
|
2
|
+
# Ip2regionExt
|
3
|
+
|
4
|
+
## 介绍
|
5
|
+
|
6
|
+
这是一个ip2region库c客户端的ruby ffi扩展, 基于 [lionsoul2014/ip2region](https://github.com/lionsoul2014/ip2region) 最新版本开发
|
7
|
+
|
8
|
+
直接调用c原生api,充分发挥IO操作的性能优势
|
9
|
+
|
10
|
+
## 安装
|
11
|
+
`gem install ip2region_ext`
|
12
|
+
## 使用
|
13
|
+
|
14
|
+
```ruby
|
15
|
+
require 'ip2region_ext'
|
16
|
+
|
17
|
+
# 查询
|
18
|
+
#db_type = :file|:index|:cache
|
19
|
+
#Ip2regionExt.connect({db_type: :index, db_path: "/var/ip2region.xdb"})
|
20
|
+
Ip2regionExt.connect({db_type: :index})
|
21
|
+
p Ip2regionExt.search('114.114.114.114')
|
22
|
+
Ip2regionExt.close
|
23
|
+
# => "中国|0|辽宁省|丹东市|联通"
|
24
|
+
|
25
|
+
```
|
26
|
+
|
27
|
+
## 如何下载外挂 xdb 文件
|
28
|
+
|
29
|
+
下载这个文件 [https://github.com/lionsoul2014/ip2region/blob/master/data/ip2region.xdb](https://github.com/lionsoul2014/ip2region/blob/master/data/ip2region.xdb)
|
30
|
+
|
31
|
+
|
32
|
+
## 协议
|
33
|
+
|
34
|
+
MIT 协议
|
data/Rakefile
ADDED
data/db/ip2region.xdb
ADDED
Binary file
|
@@ -0,0 +1,428 @@
|
|
1
|
+
// Copyright 2022 The Ip2Region Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a Apache2.0-style
|
3
|
+
// license that can be found in the LICENSE file.
|
4
|
+
|
5
|
+
// ---
|
6
|
+
// @Author Lion <chenxin619315@gmail.com>
|
7
|
+
// @Date 2022/06/27
|
8
|
+
|
9
|
+
#include "xdb_searcher.h"
|
10
|
+
|
11
|
+
// for Linux
|
12
|
+
#ifdef XDB_LINUX
|
13
|
+
#include "sys/time.h"
|
14
|
+
#endif
|
15
|
+
|
16
|
+
// @Note: since 2023/10/13 to compatible with the windows system
|
17
|
+
#ifdef XDB_WINDOWS
|
18
|
+
#include <windows.h>
|
19
|
+
XDB_PRIVATE(int) gettimeofday(struct timeval* tp, void* tzp) {
|
20
|
+
FILETIME ft;
|
21
|
+
ULARGE_INTEGER uint64;
|
22
|
+
long long ns100 = 116444736000000000LL;
|
23
|
+
|
24
|
+
GetSystemTimeAsFileTime(&ft);
|
25
|
+
uint64.LowPart = ft.dwLowDateTime;
|
26
|
+
uint64.HighPart = ft.dwHighDateTime;
|
27
|
+
uint64.QuadPart -= ns100;
|
28
|
+
uint64.QuadPart /= 10;
|
29
|
+
|
30
|
+
tp->tv_sec = uint64.QuadPart / 1000000;
|
31
|
+
tp->tv_usec = uint64.QuadPart % 1000000;
|
32
|
+
|
33
|
+
return 0;
|
34
|
+
}
|
35
|
+
#endif
|
36
|
+
|
37
|
+
|
38
|
+
// internal function prototype define
|
39
|
+
XDB_PRIVATE(int) read(xdb_searcher_t *, long offset, char *, size_t length);
|
40
|
+
|
41
|
+
XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) {
|
42
|
+
memset(xdb, 0x00, sizeof(xdb_searcher_t));
|
43
|
+
|
44
|
+
// check the content buffer first
|
45
|
+
if (c_buffer != NULL) {
|
46
|
+
xdb->v_index = NULL;
|
47
|
+
xdb->content = c_buffer;
|
48
|
+
return 0;
|
49
|
+
}
|
50
|
+
|
51
|
+
// open the xdb binary file
|
52
|
+
FILE *handle = fopen(db_path, "rb");
|
53
|
+
if (handle == NULL) {
|
54
|
+
return 1;
|
55
|
+
}
|
56
|
+
|
57
|
+
xdb->handle = handle;
|
58
|
+
xdb->v_index = v_index;
|
59
|
+
|
60
|
+
return 0;
|
61
|
+
}
|
62
|
+
|
63
|
+
// xdb searcher new api define
|
64
|
+
XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) {
|
65
|
+
return xdb_new_base(xdb, db_path, NULL, NULL);
|
66
|
+
}
|
67
|
+
|
68
|
+
XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) {
|
69
|
+
return xdb_new_base(xdb, db_path, v_index, NULL);
|
70
|
+
}
|
71
|
+
|
72
|
+
XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) {
|
73
|
+
return xdb_new_base(xdb, NULL, NULL, c_buffer);
|
74
|
+
}
|
75
|
+
|
76
|
+
XDB_PUBLIC(void) xdb_close(void *ptr) {
|
77
|
+
xdb_searcher_t *xdb = (xdb_searcher_t *) ptr;
|
78
|
+
if (xdb->handle != NULL) {
|
79
|
+
fclose(xdb->handle);
|
80
|
+
xdb->handle = NULL;
|
81
|
+
}
|
82
|
+
}
|
83
|
+
|
84
|
+
// --- xdb searcher search api define
|
85
|
+
|
86
|
+
XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, char *region_buffer, size_t length) {
|
87
|
+
unsigned int ip = 0;
|
88
|
+
int errcode = xdb_check_ip(str_ip, &ip);
|
89
|
+
if (errcode != 0) {
|
90
|
+
return 10 + errcode;
|
91
|
+
} else {
|
92
|
+
return xdb_search(xdb, ip, region_buffer, length);
|
93
|
+
}
|
94
|
+
}
|
95
|
+
|
96
|
+
XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_buffer, size_t length) {
|
97
|
+
int il0, il1, idx, err, l, h, m, data_len;
|
98
|
+
unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr;
|
99
|
+
char vector_buffer[xdb_vector_index_size], segment_buffer[xdb_segment_index_size];
|
100
|
+
|
101
|
+
// reset the io counter
|
102
|
+
xdb->io_count = 0;
|
103
|
+
|
104
|
+
// locate the segment index block based on the vector index
|
105
|
+
il0 = ((int) (ip >> 24)) & 0xFF;
|
106
|
+
il1 = ((int) (ip >> 16)) & 0xFF;
|
107
|
+
idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size;
|
108
|
+
if (xdb->v_index != NULL) {
|
109
|
+
s_ptr = xdb_get_uint(xdb->v_index->buffer, idx);
|
110
|
+
e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4);
|
111
|
+
} else if (xdb->content != NULL) {
|
112
|
+
s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx);
|
113
|
+
e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4);
|
114
|
+
} else {
|
115
|
+
err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer));
|
116
|
+
if (err != 0) {
|
117
|
+
return 10 + err;
|
118
|
+
}
|
119
|
+
|
120
|
+
s_ptr = xdb_get_uint(vector_buffer, 0);
|
121
|
+
e_ptr = xdb_get_uint(vector_buffer, 4);
|
122
|
+
}
|
123
|
+
|
124
|
+
// printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr);
|
125
|
+
// binary search to get the final region info
|
126
|
+
data_len = 0, data_ptr = 0;
|
127
|
+
l = 0, h = ((int) (e_ptr - s_ptr)) / xdb_segment_index_size;
|
128
|
+
while (l <= h) {
|
129
|
+
m = (l + h) >> 1;
|
130
|
+
p = s_ptr + m * xdb_segment_index_size;
|
131
|
+
|
132
|
+
// read the segment index item
|
133
|
+
err = read(xdb, p, segment_buffer, sizeof(segment_buffer));
|
134
|
+
if (err != 0) {
|
135
|
+
return 20 + err;
|
136
|
+
}
|
137
|
+
|
138
|
+
// decode the data fields as needed
|
139
|
+
sip = xdb_get_uint(segment_buffer, 0);
|
140
|
+
if (ip < sip) {
|
141
|
+
h = m - 1;
|
142
|
+
} else {
|
143
|
+
eip = xdb_get_uint(segment_buffer, 4);
|
144
|
+
if (ip > eip) {
|
145
|
+
l = m + 1;
|
146
|
+
} else {
|
147
|
+
data_len = xdb_get_ushort(segment_buffer, 8);
|
148
|
+
data_ptr = xdb_get_uint(segment_buffer, 10);
|
149
|
+
break;
|
150
|
+
}
|
151
|
+
}
|
152
|
+
}
|
153
|
+
|
154
|
+
// printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr);
|
155
|
+
if (data_len == 0) {
|
156
|
+
region_buffer[0] = '\0';
|
157
|
+
return 0;
|
158
|
+
}
|
159
|
+
|
160
|
+
// buffer length checking
|
161
|
+
if (data_len >= (int) length) {
|
162
|
+
return 1;
|
163
|
+
}
|
164
|
+
|
165
|
+
err = read(xdb, data_ptr, region_buffer, data_len);
|
166
|
+
if (err != 0) {
|
167
|
+
return 30 + err;
|
168
|
+
}
|
169
|
+
|
170
|
+
// auto append a NULL-end
|
171
|
+
region_buffer[data_len] = '\0';
|
172
|
+
return 0;
|
173
|
+
}
|
174
|
+
|
175
|
+
XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) {
|
176
|
+
// check the xdb content cache first
|
177
|
+
if (xdb->content != NULL) {
|
178
|
+
memcpy(buffer, xdb->content->buffer + offset, length);
|
179
|
+
return 0;
|
180
|
+
}
|
181
|
+
|
182
|
+
// seek to the offset
|
183
|
+
if (fseek(xdb->handle, offset, SEEK_SET) == -1) {
|
184
|
+
return 1;
|
185
|
+
}
|
186
|
+
|
187
|
+
xdb->io_count++;
|
188
|
+
if (fread(buffer, 1, length, xdb->handle) != length) {
|
189
|
+
return 2;
|
190
|
+
}
|
191
|
+
|
192
|
+
return 0;
|
193
|
+
}
|
194
|
+
|
195
|
+
XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) {
|
196
|
+
return xdb->io_count;
|
197
|
+
}
|
198
|
+
|
199
|
+
|
200
|
+
// --- buffer load util functions
|
201
|
+
|
202
|
+
XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *handle) {
|
203
|
+
xdb_header_t *header;
|
204
|
+
unsigned int size = xdb_header_info_length;
|
205
|
+
|
206
|
+
// entry alloc
|
207
|
+
header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t));
|
208
|
+
if (header == NULL) {
|
209
|
+
return NULL;
|
210
|
+
}
|
211
|
+
|
212
|
+
if (fseek(handle, 0, SEEK_SET) == -1) {
|
213
|
+
xdb_free(header);
|
214
|
+
return NULL;
|
215
|
+
}
|
216
|
+
|
217
|
+
if (fread(header->buffer, 1,size, handle) != size) {
|
218
|
+
xdb_free(header);
|
219
|
+
return NULL;
|
220
|
+
}
|
221
|
+
|
222
|
+
// fill the fields
|
223
|
+
header->length = size;
|
224
|
+
header->version = (unsigned short) xdb_get_ushort(header->buffer, 0);
|
225
|
+
header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2);
|
226
|
+
header->created_at = xdb_get_uint(header->buffer, 4);
|
227
|
+
header->start_index_ptr = xdb_get_uint(header->buffer, 8);
|
228
|
+
header->end_index_ptr = xdb_get_uint(header->buffer,12);
|
229
|
+
|
230
|
+
return header;
|
231
|
+
}
|
232
|
+
|
233
|
+
XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *db_path) {
|
234
|
+
xdb_header_t *header;
|
235
|
+
FILE *handle = fopen(db_path, "rb");
|
236
|
+
if (handle == NULL) {
|
237
|
+
return NULL;
|
238
|
+
}
|
239
|
+
|
240
|
+
header = xdb_load_header(handle);
|
241
|
+
fclose(handle);
|
242
|
+
return header;
|
243
|
+
}
|
244
|
+
|
245
|
+
XDB_PUBLIC(void) xdb_close_header(void *ptr) {
|
246
|
+
xdb_header_t *header = (xdb_header_t *) ptr;
|
247
|
+
if (header->length > 0) {
|
248
|
+
header->length = 0;
|
249
|
+
xdb_free(header);
|
250
|
+
}
|
251
|
+
}
|
252
|
+
|
253
|
+
// --- vector index
|
254
|
+
|
255
|
+
XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *handle) {
|
256
|
+
xdb_vector_index_t *v_index;
|
257
|
+
unsigned int size = xdb_vector_index_length;
|
258
|
+
|
259
|
+
// seek to the vector index offset
|
260
|
+
if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) {
|
261
|
+
return NULL;
|
262
|
+
}
|
263
|
+
|
264
|
+
// do the buffer read
|
265
|
+
v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t));
|
266
|
+
if (v_index == NULL) {
|
267
|
+
return NULL;
|
268
|
+
}
|
269
|
+
|
270
|
+
v_index->length = size;
|
271
|
+
if (fread(v_index->buffer, 1, size, handle) != size) {
|
272
|
+
xdb_free(v_index);
|
273
|
+
return NULL;
|
274
|
+
}
|
275
|
+
|
276
|
+
return v_index;
|
277
|
+
}
|
278
|
+
|
279
|
+
XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *db_path) {
|
280
|
+
xdb_vector_index_t *v_index;
|
281
|
+
FILE *handle = fopen(db_path, "rb");
|
282
|
+
if (handle == NULL) {
|
283
|
+
return NULL;
|
284
|
+
}
|
285
|
+
|
286
|
+
v_index = xdb_load_vector_index(handle);
|
287
|
+
fclose(handle);
|
288
|
+
return v_index;
|
289
|
+
}
|
290
|
+
|
291
|
+
XDB_PUBLIC(void) xdb_close_vector_index(void *ptr) {
|
292
|
+
xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr;
|
293
|
+
if (v_index->length > 0) {
|
294
|
+
v_index->length = 0;
|
295
|
+
xdb_free(v_index);
|
296
|
+
}
|
297
|
+
}
|
298
|
+
|
299
|
+
// --- content buffer
|
300
|
+
|
301
|
+
XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *handle) {
|
302
|
+
unsigned int size;
|
303
|
+
xdb_content_t *content;
|
304
|
+
|
305
|
+
// determine the file size
|
306
|
+
if (fseek(handle, 0, SEEK_END) == -1) {
|
307
|
+
return NULL;
|
308
|
+
}
|
309
|
+
|
310
|
+
size = (unsigned int) ftell(handle);
|
311
|
+
if (fseek(handle, 0, SEEK_SET) == -1) {
|
312
|
+
return NULL;
|
313
|
+
}
|
314
|
+
|
315
|
+
// do the file read
|
316
|
+
content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t));
|
317
|
+
if (content == NULL) {
|
318
|
+
return NULL;
|
319
|
+
}
|
320
|
+
|
321
|
+
// do the buffer alloc
|
322
|
+
content->buffer = (char *) xdb_malloc(size);
|
323
|
+
if (content->buffer == NULL) {
|
324
|
+
xdb_free(content);
|
325
|
+
return NULL;
|
326
|
+
}
|
327
|
+
|
328
|
+
// read the content into the buffer
|
329
|
+
content->length = size;
|
330
|
+
if (fread(content->buffer, 1, size, handle) != size) {
|
331
|
+
xdb_free(content);
|
332
|
+
return NULL;
|
333
|
+
}
|
334
|
+
|
335
|
+
return content;
|
336
|
+
}
|
337
|
+
|
338
|
+
XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *db_path) {
|
339
|
+
xdb_content_t *content;
|
340
|
+
FILE *handle = fopen(db_path, "rb");
|
341
|
+
if (handle == NULL) {
|
342
|
+
return NULL;
|
343
|
+
}
|
344
|
+
|
345
|
+
content = xdb_load_content(handle);
|
346
|
+
fclose(handle);
|
347
|
+
return content;
|
348
|
+
}
|
349
|
+
|
350
|
+
XDB_PUBLIC(void) xdb_close_content(void *ptr) {
|
351
|
+
xdb_content_t *content = (xdb_content_t *) ptr;
|
352
|
+
if (content->length > 0) {
|
353
|
+
content->length = 0;
|
354
|
+
xdb_free(content->buffer);
|
355
|
+
content->buffer = NULL;
|
356
|
+
xdb_free(content);
|
357
|
+
}
|
358
|
+
}
|
359
|
+
|
360
|
+
// --- End
|
361
|
+
|
362
|
+
// get unsigned long (4bytes) from a specified buffer start from the specified offset
|
363
|
+
XDB_PUBLIC(unsigned int) xdb_get_uint(const char *buffer, int offset) {
|
364
|
+
return (
|
365
|
+
((buffer[offset ]) & 0x000000FF) |
|
366
|
+
((buffer[offset+1] << 8) & 0x0000FF00) |
|
367
|
+
((buffer[offset+2] << 16) & 0x00FF0000) |
|
368
|
+
((buffer[offset+3] << 24) & 0xFF000000)
|
369
|
+
);
|
370
|
+
}
|
371
|
+
|
372
|
+
// get unsigned short (2bytes) from a specified buffer start from the specified offset
|
373
|
+
XDB_PUBLIC(int) xdb_get_ushort(const char *buffer, int offset) {
|
374
|
+
return (
|
375
|
+
((buffer[offset ]) & 0x000000FF) |
|
376
|
+
((buffer[offset+1] << 8) & 0x0000FF00)
|
377
|
+
);
|
378
|
+
}
|
379
|
+
|
380
|
+
// string ip to unsigned int
|
381
|
+
static int shiftIndex[4] = {24, 16, 8, 0};
|
382
|
+
XDB_PUBLIC(int) xdb_check_ip(const char *src_ip, unsigned int *dst_ip) {
|
383
|
+
char c;
|
384
|
+
int i, n, ip = 0;
|
385
|
+
const char *ptr = src_ip;
|
386
|
+
for (i = 0; i < 4; i++) {
|
387
|
+
n = 0;
|
388
|
+
while (1) {
|
389
|
+
c = *ptr;
|
390
|
+
ptr++;
|
391
|
+
if (c >= '0' && c <= '9') {
|
392
|
+
n *= 10;
|
393
|
+
n += c - '0';
|
394
|
+
} else if ((i < 3 && c == '.') || i == 3) {
|
395
|
+
// stopping at the '.' but ignore the tailing chars
|
396
|
+
// after the 3rd one (auto clean the tailing none-integer ?).
|
397
|
+
break;
|
398
|
+
} else {
|
399
|
+
return 1;
|
400
|
+
}
|
401
|
+
}
|
402
|
+
|
403
|
+
if (n > 0xFF) {
|
404
|
+
return 2;
|
405
|
+
}
|
406
|
+
|
407
|
+
ip |= (n << shiftIndex[i]);
|
408
|
+
}
|
409
|
+
|
410
|
+
*dst_ip = ip;
|
411
|
+
return 0;
|
412
|
+
}
|
413
|
+
|
414
|
+
// unsigned int ip to string ip
|
415
|
+
XDB_PUBLIC(void) xdb_long2ip(unsigned int ip, char *buffer) {
|
416
|
+
sprintf(buffer, "%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8) & 0xFF, ip & 0xFF);
|
417
|
+
}
|
418
|
+
|
419
|
+
// get the middle ip of a and b
|
420
|
+
XDB_PUBLIC(unsigned int) xdb_mip(unsigned long a, unsigned long b) {
|
421
|
+
return (unsigned int) ((a + b) >> 1);
|
422
|
+
}
|
423
|
+
|
424
|
+
XDB_PUBLIC(long) xdb_now() {
|
425
|
+
struct timeval c_time;
|
426
|
+
gettimeofday(&c_time, NULL);
|
427
|
+
return c_time.tv_sec * (int)1e6 + c_time.tv_usec;
|
428
|
+
}
|
@@ -0,0 +1,149 @@
|
|
1
|
+
// Copyright 2022 The Ip2Region Authors. All rights reserved.
|
2
|
+
// Use of this source code is governed by a Apache2.0-style
|
3
|
+
// license that can be found in the LICENSE file.
|
4
|
+
|
5
|
+
// ---
|
6
|
+
// @Author Lion <chenxin619315@gmail.com>
|
7
|
+
// @Date 2022/06/27
|
8
|
+
|
9
|
+
#ifndef C_XDB_SEARCHER_H
|
10
|
+
#define C_XDB_SEARCHER_H
|
11
|
+
|
12
|
+
#include <stdio.h>
|
13
|
+
#include <stdlib.h>
|
14
|
+
#include <string.h>
|
15
|
+
|
16
|
+
#if ( defined(WIN32) || defined(_WIN32) || defined(__WINDOWS_) || defined(WINNT) )
|
17
|
+
# define XDB_PUBLIC(type) extern __declspec(dllexport) type
|
18
|
+
# define XDB_PRIVATE(type) static type
|
19
|
+
# define XDB_WINDOWS
|
20
|
+
#elif ( defined(linux) || defined(_UNIX) || defined(__APPLE__) )
|
21
|
+
# define XDB_PUBLIC(type) extern type
|
22
|
+
# define XDB_PRIVATE(type) static inline type
|
23
|
+
# define XDB_LINUX
|
24
|
+
#endif
|
25
|
+
|
26
|
+
#define xdb_calloc( _blocks, _bytes ) calloc( _blocks, _bytes )
|
27
|
+
#define xdb_malloc( _bytes ) malloc( _bytes )
|
28
|
+
#define xdb_free( _ptr ) free( _ptr )
|
29
|
+
|
30
|
+
// public constants define
|
31
|
+
#define xdb_header_info_length 256
|
32
|
+
#define xdb_vector_index_rows 256
|
33
|
+
#define xdb_vector_index_cols 256
|
34
|
+
#define xdb_vector_index_size 8
|
35
|
+
#define xdb_segment_index_size 14
|
36
|
+
|
37
|
+
// cache of vector_index_row × vector_index_rows × vector_index_size
|
38
|
+
#define xdb_vector_index_length 524288
|
39
|
+
|
40
|
+
|
41
|
+
// --- buffer load util functions
|
42
|
+
|
43
|
+
// use the following buffer struct to wrap the binary buffer data
|
44
|
+
// since the buffer data could not be operated with the string API.
|
45
|
+
struct xdb_header {
|
46
|
+
unsigned short version;
|
47
|
+
unsigned short index_policy;
|
48
|
+
unsigned int created_at;
|
49
|
+
unsigned int start_index_ptr;
|
50
|
+
unsigned int end_index_ptr;
|
51
|
+
|
52
|
+
// the original buffer
|
53
|
+
unsigned int length;
|
54
|
+
char buffer[xdb_header_info_length];
|
55
|
+
};
|
56
|
+
typedef struct xdb_header xdb_header_t;
|
57
|
+
|
58
|
+
XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *);
|
59
|
+
|
60
|
+
XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *);
|
61
|
+
|
62
|
+
XDB_PUBLIC(void) xdb_close_header(void *);
|
63
|
+
|
64
|
+
|
65
|
+
// --- vector index buffer
|
66
|
+
struct xdb_vector_index {
|
67
|
+
unsigned int length;
|
68
|
+
char buffer[xdb_vector_index_length];
|
69
|
+
};
|
70
|
+
typedef struct xdb_vector_index xdb_vector_index_t;
|
71
|
+
|
72
|
+
XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *);
|
73
|
+
|
74
|
+
XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *);
|
75
|
+
|
76
|
+
XDB_PUBLIC(void) xdb_close_vector_index(void *);
|
77
|
+
|
78
|
+
|
79
|
+
// --- content buffer
|
80
|
+
struct xdb_content {
|
81
|
+
unsigned int length;
|
82
|
+
char *buffer;
|
83
|
+
};
|
84
|
+
typedef struct xdb_content xdb_content_t;
|
85
|
+
|
86
|
+
XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *);
|
87
|
+
|
88
|
+
XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *);
|
89
|
+
|
90
|
+
XDB_PUBLIC(void) xdb_close_content(void *);
|
91
|
+
|
92
|
+
// --- End buffer load
|
93
|
+
|
94
|
+
// xdb searcher structure
|
95
|
+
struct xdb_searcher_entry {
|
96
|
+
FILE *handle;
|
97
|
+
|
98
|
+
// header info
|
99
|
+
const char *header;
|
100
|
+
int io_count;
|
101
|
+
|
102
|
+
// vector index buffer cache.
|
103
|
+
// preload the vector index will reduce the number of IO operations
|
104
|
+
// thus speedup the search process.
|
105
|
+
const xdb_vector_index_t *v_index;
|
106
|
+
|
107
|
+
// content buffer.
|
108
|
+
// cache the whole xdb content.
|
109
|
+
const xdb_content_t *content;
|
110
|
+
};
|
111
|
+
typedef struct xdb_searcher_entry xdb_searcher_t;
|
112
|
+
|
113
|
+
// xdb searcher new api define
|
114
|
+
XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *, const char *);
|
115
|
+
|
116
|
+
XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *);
|
117
|
+
|
118
|
+
XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *);
|
119
|
+
|
120
|
+
XDB_PUBLIC(void) xdb_close(void *);
|
121
|
+
|
122
|
+
// xdb searcher search api define
|
123
|
+
XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t);
|
124
|
+
|
125
|
+
XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, unsigned int, char *, size_t);
|
126
|
+
|
127
|
+
XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *);
|
128
|
+
|
129
|
+
|
130
|
+
// get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian
|
131
|
+
XDB_PUBLIC(unsigned int) xdb_get_uint(const char *, int);
|
132
|
+
|
133
|
+
// get unsigned short (2bytes) from a specified buffer start from the specified offset with little-endian
|
134
|
+
XDB_PUBLIC(int) xdb_get_ushort(const char *, int);
|
135
|
+
|
136
|
+
// check the specified string ip and convert it to an unsigned int
|
137
|
+
XDB_PUBLIC(int) xdb_check_ip(const char *, unsigned int *);
|
138
|
+
|
139
|
+
// unsigned int ip to string ip
|
140
|
+
XDB_PUBLIC(void) xdb_long2ip(unsigned int, char *);
|
141
|
+
|
142
|
+
// get the middle ip of a and b
|
143
|
+
XDB_PUBLIC(unsigned int) xdb_mip(unsigned long, unsigned long);
|
144
|
+
|
145
|
+
// get the current time in microseconds
|
146
|
+
XDB_PUBLIC(long) xdb_now();
|
147
|
+
|
148
|
+
|
149
|
+
#endif //C_XDB_SEARCHER_H
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'ffi'
|
2
|
+
|
3
|
+
module Ip2regionExt
|
4
|
+
extend FFI::Library
|
5
|
+
ffi_lib File.join(__dir__, '../','libip2region_ext.so')
|
6
|
+
|
7
|
+
XDB_HEADER_INFO_LENGTH = 256
|
8
|
+
XDB_VECTOR_INDEX_LENGTH = 524288
|
9
|
+
|
10
|
+
class XdbHeaderT < FFI::Struct
|
11
|
+
layout :version, :ushort,
|
12
|
+
:index_policy, :ushort,
|
13
|
+
:created_at, :uint,
|
14
|
+
:start_index_ptr, :uint,
|
15
|
+
:end_index_ptr, :uint,
|
16
|
+
:length, :uint,
|
17
|
+
:buffer, [:char, XDB_HEADER_INFO_LENGTH] # xdb_header_info_length 需要替换为实际的值
|
18
|
+
end
|
19
|
+
|
20
|
+
class XdbVectorIndexT < FFI::Struct
|
21
|
+
layout :length, :uint,
|
22
|
+
:buffer, [:char, XDB_VECTOR_INDEX_LENGTH]
|
23
|
+
end
|
24
|
+
|
25
|
+
class XdbContentT < FFI::Struct
|
26
|
+
layout :length, :uint,
|
27
|
+
:buffer, :pointer
|
28
|
+
end
|
29
|
+
|
30
|
+
class XdbSearcherT < FFI::Struct
|
31
|
+
layout :handle, :pointer,
|
32
|
+
:header, :string,
|
33
|
+
:io_count, :int,
|
34
|
+
:v_index, :pointer, # 指向 xdb_vector_index_t 结构体的指针
|
35
|
+
:content, :pointer # 指向 xdb_content_t 结构体的指针
|
36
|
+
end
|
37
|
+
|
38
|
+
#attach_function :xdb_load_content_from_file, [:string], XdbContentT.ptr
|
39
|
+
attach_function :xdb_new_with_file_only, [:pointer, :string], :int
|
40
|
+
attach_function :xdb_check_ip, [:string, :pointer], :int
|
41
|
+
attach_function :xdb_search_by_string, [ :pointer, :string, :pointer, :size_t ], :int
|
42
|
+
#attach_function :xdb_search, [:pointer, :uint, :string, :size_t], :int
|
43
|
+
attach_function :xdb_close, [:pointer], :void
|
44
|
+
attach_function :xdb_load_vector_index_from_file, [:string], XdbVectorIndexT.ptr
|
45
|
+
attach_function :xdb_new_with_vector_index, [:pointer, :string, :pointer], :int
|
46
|
+
attach_function :xdb_close_vector_index, [:pointer], :void
|
47
|
+
attach_function :xdb_load_content_from_file, [:string], :pointer
|
48
|
+
attach_function :xdb_new_with_buffer, [:pointer, :pointer], :int
|
49
|
+
attach_function :xdb_close_content, [:pointer], :void
|
50
|
+
|
51
|
+
|
52
|
+
end
|
53
|
+
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'singleton'
|
3
|
+
module Ip2regionExt
|
4
|
+
class Xdb
|
5
|
+
include Singleton
|
6
|
+
|
7
|
+
def connect(opt = {})
|
8
|
+
@option = {db_path: nil, db_type: :file}.merge(opt)
|
9
|
+
@searcher = XdbSearcherT.new()
|
10
|
+
#@vector_index = XdbVectorIndexT.new()
|
11
|
+
#@content = XdbContentT.new()
|
12
|
+
|
13
|
+
case @option[:db_type]
|
14
|
+
when :file
|
15
|
+
load_by_file
|
16
|
+
when :index
|
17
|
+
load_by_index
|
18
|
+
when :cache
|
19
|
+
load_by_cache
|
20
|
+
else
|
21
|
+
load_by_file
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def load_by_file
|
26
|
+
err = Ip2regionExt.xdb_new_with_file_only(@searcher, @option[:db_path])
|
27
|
+
raise Error.new("failed to create xdb searcher from `#{@option[:db_path]}` with errno=#{err}\n") if err != 0
|
28
|
+
end
|
29
|
+
|
30
|
+
def load_by_index
|
31
|
+
@vector_index = Ip2regionExt.xdb_load_vector_index_from_file(@option[:db_path])
|
32
|
+
raise Error.new("failed to load vector index from `#{@option[:db_path]}`\n") unless @vector_index
|
33
|
+
|
34
|
+
err = Ip2regionExt.xdb_new_with_vector_index(@searcher.pointer, @option[:db_path], @vector_index)
|
35
|
+
raise Error.new("failed to create vector index cached searcher with errno=#{err}\n") if err != 0
|
36
|
+
end
|
37
|
+
|
38
|
+
def load_by_cache
|
39
|
+
@content = Ip2regionExt.xdb_load_content_from_file(@option[:db_path])
|
40
|
+
raise Error.new("failed to load xdb content from `#{}`\n") unless @content
|
41
|
+
|
42
|
+
err = Ip2regionExt.xdb_new_with_buffer(@searcher, @content)
|
43
|
+
raise Error.new("failed to create content cached searcher with errno=#{err}\n") if err != 0
|
44
|
+
end
|
45
|
+
|
46
|
+
def query(ip_address)
|
47
|
+
region_buffer = FFI::MemoryPointer.new(:char, 256)
|
48
|
+
err = Ip2regionExt.xdb_search_by_string(@searcher, ip_address, region_buffer, region_buffer.size)
|
49
|
+
raise Error.new("failed search(#{ip_address}) with errno=#{err}\n") if err != 0
|
50
|
+
region_buffer.read_string.force_encoding('UTF-8')
|
51
|
+
end
|
52
|
+
|
53
|
+
def close
|
54
|
+
Ip2regionExt.xdb_close(@searcher)
|
55
|
+
Ip2regionExt.xdb_close_vector_index(@vector_index) if @option[:db_type] == :index
|
56
|
+
Ip2regionExt.xdb_close_content(@content) if @option[:db_type] == :cache
|
57
|
+
end
|
58
|
+
|
59
|
+
def check_ip(ip_address)
|
60
|
+
result = FFI::MemoryPointer.new(:int)
|
61
|
+
err = Ip2regionExt.xdb_check_ip(ip_address, result)
|
62
|
+
raise Error.new("failed check ip(#{ip_address}) with errno=#{err}\n") if err != 0
|
63
|
+
result.get_int(0)
|
64
|
+
end
|
65
|
+
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "ip2region_ext/version"
|
4
|
+
require_relative 'ip2region_ext/ip2region_ffi'
|
5
|
+
require_relative 'ip2region_ext/xdb'
|
6
|
+
|
7
|
+
module Ip2regionExt
|
8
|
+
class Error < StandardError; end
|
9
|
+
|
10
|
+
@@xdb_path = File.join(__dir__, '../db','ip2region.xdb')
|
11
|
+
|
12
|
+
def self.connect(option = {})
|
13
|
+
option[:db_path] ||= @@xdb_path
|
14
|
+
Xdb.instance.connect(option)
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.search(ip_address)
|
18
|
+
Xdb.instance.query(ip_address)
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.close
|
22
|
+
Xdb.instance.close
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
end
|
data/sig/ip2region.rbs
ADDED
metadata
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ip2region_ext
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- songgz
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-06-10 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: ffi
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
description: 直接调用c原生api,充分发挥IO操作的性能优势
|
28
|
+
email:
|
29
|
+
- sgzhe@163.com
|
30
|
+
executables: []
|
31
|
+
extensions:
|
32
|
+
- ext/ip2region/extconf.rb
|
33
|
+
extra_rdoc_files: []
|
34
|
+
files:
|
35
|
+
- README.md
|
36
|
+
- Rakefile
|
37
|
+
- db/ip2region.xdb
|
38
|
+
- ext/ip2region/extconf.rb
|
39
|
+
- ext/ip2region/xdb_searcher.c
|
40
|
+
- ext/ip2region/xdb_searcher.h
|
41
|
+
- lib/ip2region_ext.rb
|
42
|
+
- lib/ip2region_ext/ip2region_ffi.rb
|
43
|
+
- lib/ip2region_ext/version.rb
|
44
|
+
- lib/ip2region_ext/xdb.rb
|
45
|
+
- sig/ip2region.rbs
|
46
|
+
homepage: https://github.com/songgz/ip2region_ext
|
47
|
+
licenses: []
|
48
|
+
metadata:
|
49
|
+
allowed_push_host: https://rubygems.org
|
50
|
+
homepage_uri: https://github.com/songgz/ip2region_ext
|
51
|
+
source_code_uri: https://github.com/songgz/ip2region_ext
|
52
|
+
changelog_uri: https://github.com/songgz/ip2region_ext
|
53
|
+
post_install_message:
|
54
|
+
rdoc_options: []
|
55
|
+
require_paths:
|
56
|
+
- lib
|
57
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 3.0.0
|
62
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '0'
|
67
|
+
requirements: []
|
68
|
+
rubygems_version: 3.5.10
|
69
|
+
signing_key:
|
70
|
+
specification_version: 4
|
71
|
+
summary: 这是一个ip2region库c客户端的ruby ffi扩展
|
72
|
+
test_files: []
|