gnista 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/HISTORY.md +5 -0
- data/LICENSE.txt +22 -0
- data/README.md +73 -0
- data/Rakefile +20 -0
- data/ext/gnista/extconf.rb +9 -0
- data/ext/gnista/gnista.c +534 -0
- data/lib/gnista/version.rb +3 -0
- data/lib/gnista.rb +12 -0
- data/test/test_commands.rb +134 -0
- data/test/test_gnista.rb +57 -0
- metadata +88 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: bab3ebb294b61581358c5cec1b5c35d58278068f
|
4
|
+
data.tar.gz: 46ce784095d0f7ebfa35ed4292c8666572ba6f71
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b6758ef75988f46af845a974ab7e50398fc87251c1b070dfa4d2fd96679d0de075df5f3fb3bc2446b506b591811317943536173805ae0a1afc0a316abe9f9ccc
|
7
|
+
data.tar.gz: dc6f0af77d91f73180f466556b1275e9c7b86650a849aedf4e4cb524ff4b16f8ac48b5eec1c710d0230fa7b6b84c8f3aba8436f4fff8de0e3392726c84aa6672
|
data/HISTORY.md
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
(The MIT License)
|
2
|
+
|
3
|
+
Copyright (c) 2013 Emanuel Andersson
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
'Software'), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
19
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
20
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
21
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
22
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
# Gnista
|
2
|
+
|
3
|
+
Gnista is a wrapper for the database/hashstore [Sparkey](http://github.com/spotify/sparkey) written for Ruby. Gnista packs all the native features of Sparkey into a shiny red package, easy to use.
|
4
|
+
|
5
|
+
## Setup
|
6
|
+
|
7
|
+
You will need Sparkey, obviously. It's really easy to install. Head over to [http://github.com/spotify/sparkey](http://github.com/spotify/sparkey) and install it, I'll wait..
|
8
|
+
|
9
|
+
Now, add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
gem 'gnista'
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install gnista
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
You can read about Sparkey [here](http://github.com/spotify/sparkey).
|
24
|
+
|
25
|
+
```ruby
|
26
|
+
require 'gnista'
|
27
|
+
|
28
|
+
logwriter = Gnista::Logwriter.new "mylog.log" # no compression
|
29
|
+
logwriter = Gnista::Logwriter.new "mylog.log", 4000 # 4k compression block size
|
30
|
+
logwriter = Gnista::Logwriter.new "mylog.log", :append # append to existing log
|
31
|
+
|
32
|
+
logwriter.put "key", "value" # put entry
|
33
|
+
logwriter.del "key" # delete entry
|
34
|
+
|
35
|
+
logwriter.flush
|
36
|
+
|
37
|
+
logreader = Gnista::Logreader.new "mylog.log"
|
38
|
+
logreader.each {|key,value| puts key, value }
|
39
|
+
|
40
|
+
Gnista::Hash.write "mylog.hash", "mylog.log" # no preferred hash size
|
41
|
+
Gnista::Hash.write "mylog.hash", "mylog.log", 4 # 32 bit murmurhash3_x86_32
|
42
|
+
Gnista::Hash.write "mylog.hash", "mylog.log", 8 # lower 64-bit part of murmurhash3_x64_128
|
43
|
+
|
44
|
+
hash = Gnista::Hash.new "mylog.hash", "mylog.log"
|
45
|
+
hash.each {|key,value| puts key, value }
|
46
|
+
hash.get "key" # => "value" or nil
|
47
|
+
|
48
|
+
hash.maxkeylen # largest key length
|
49
|
+
hash.maxkeylen # largest value length
|
50
|
+
hash.length
|
51
|
+
hash.collisions
|
52
|
+
|
53
|
+
# Don't forget to close!
|
54
|
+
logwriter.close
|
55
|
+
logreader.close
|
56
|
+
hash.close
|
57
|
+
```
|
58
|
+
|
59
|
+
|
60
|
+
## Contributing
|
61
|
+
|
62
|
+
Use the following commands to build the native extensions and test:
|
63
|
+
|
64
|
+
$ rake make
|
65
|
+
$ rake test
|
66
|
+
|
67
|
+
Then:
|
68
|
+
|
69
|
+
1. Fork it
|
70
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
71
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
72
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
73
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
|
3
|
+
require 'rake/testtask'
|
4
|
+
Rake::TestTask.new(:test) do |test|
|
5
|
+
test.libs << 'ext' << 'test'
|
6
|
+
test.pattern = 'test/**/test_*'
|
7
|
+
test.verbose = false
|
8
|
+
end
|
9
|
+
|
10
|
+
task :make do |t|
|
11
|
+
sh "cd ext/gnista && ruby extconf.rb && make"
|
12
|
+
end
|
13
|
+
|
14
|
+
task :clean do |t|
|
15
|
+
sh "rm -f ext/gnista/*.bundle"
|
16
|
+
sh "rm -f ext/gnista/Makefile"
|
17
|
+
sh "rm -f ext/gnista/*.log"
|
18
|
+
sh "rm -f ext/gnista/*.o"
|
19
|
+
sh "rm -rf pkg"
|
20
|
+
end
|
data/ext/gnista/gnista.c
ADDED
@@ -0,0 +1,534 @@
|
|
1
|
+
#include <ruby.h>
|
2
|
+
#include <sparkey/sparkey.h>
|
3
|
+
|
4
|
+
typedef int bool;
|
5
|
+
#define true 1
|
6
|
+
#define false 0
|
7
|
+
|
8
|
+
|
9
|
+
typedef struct instance_logwriter {
|
10
|
+
sparkey_logwriter *logwriter;
|
11
|
+
int open;
|
12
|
+
} instance_logwriter;
|
13
|
+
|
14
|
+
typedef struct instance_logreader {
|
15
|
+
sparkey_logreader *logreader;
|
16
|
+
int open;
|
17
|
+
} instance_logreader;
|
18
|
+
|
19
|
+
typedef struct instance_hashreader {
|
20
|
+
sparkey_hashreader *hashreader;
|
21
|
+
int open;
|
22
|
+
} instance_hashreader;
|
23
|
+
|
24
|
+
/********************************************************************************/
|
25
|
+
/**************** ERROR HANDLING ************************************************/
|
26
|
+
|
27
|
+
VALUE GnistaException = Qnil;
|
28
|
+
|
29
|
+
static void raise_sparkey(sparkey_returncode returncode) {
|
30
|
+
const char *error_msg = sparkey_errstring(returncode);
|
31
|
+
rb_raise(GnistaException, "%s", error_msg);
|
32
|
+
}
|
33
|
+
|
34
|
+
static void check_open(bool open) {
|
35
|
+
if (!open) {
|
36
|
+
rb_raise(GnistaException, "Closed");
|
37
|
+
}
|
38
|
+
}
|
39
|
+
|
40
|
+
/********************************************************************************/
|
41
|
+
/**************** LOGWRITER *****************************************************/
|
42
|
+
|
43
|
+
static void dealloc_logwriter(void *p) {
|
44
|
+
instance_logwriter *i_logwriter = p;
|
45
|
+
|
46
|
+
if (i_logwriter->open) {
|
47
|
+
sparkey_returncode returncode;
|
48
|
+
returncode = sparkey_logwriter_close(&i_logwriter->logwriter);
|
49
|
+
|
50
|
+
if (returncode != SPARKEY_SUCCESS) {
|
51
|
+
raise_sparkey(returncode);
|
52
|
+
}
|
53
|
+
}
|
54
|
+
|
55
|
+
free(i_logwriter);
|
56
|
+
}
|
57
|
+
|
58
|
+
static VALUE alloc_logwriter(VALUE klass) {
|
59
|
+
instance_logwriter *i_logwriter = ALLOC(instance_logwriter);
|
60
|
+
return Data_Wrap_Struct(klass, 0, dealloc_logwriter, i_logwriter);
|
61
|
+
}
|
62
|
+
|
63
|
+
static instance_logwriter* get_logwriter(VALUE self) {
|
64
|
+
instance_logwriter *i_logwriter;
|
65
|
+
Data_Get_Struct(self, instance_logwriter, i_logwriter);
|
66
|
+
return i_logwriter;
|
67
|
+
}
|
68
|
+
|
69
|
+
static VALUE method_logwriter_initialize(VALUE self, VALUE args) {
|
70
|
+
sparkey_returncode returncode;
|
71
|
+
instance_logwriter *i_logwriter = get_logwriter(self);
|
72
|
+
int len = RARRAY_LEN(args);
|
73
|
+
|
74
|
+
if (len > 2 || len == 0) {
|
75
|
+
rb_raise(rb_eArgError, "Wrong number of arguments");
|
76
|
+
}
|
77
|
+
|
78
|
+
Check_Type(rb_ary_entry(args, 0), T_STRING);
|
79
|
+
|
80
|
+
if (len == 2) {
|
81
|
+
if (TYPE(rb_ary_entry(args, 1)) == T_SYMBOL && (rb_to_id(rb_ary_entry(args, 1)) == rb_intern("append"))) {
|
82
|
+
returncode = sparkey_logwriter_append(&i_logwriter->logwriter, RSTRING_PTR(rb_ary_entry(args, 0)));
|
83
|
+
} else if (TYPE(rb_ary_entry(args, 1)) == T_FIXNUM) {
|
84
|
+
returncode = sparkey_logwriter_create(&i_logwriter->logwriter, RSTRING_PTR(rb_ary_entry(args, 0)), SPARKEY_COMPRESSION_SNAPPY, NUM2INT(rb_ary_entry(args, 1)));
|
85
|
+
} else {
|
86
|
+
rb_raise(rb_eArgError, "Invalid arguments");
|
87
|
+
}
|
88
|
+
} else {
|
89
|
+
returncode = sparkey_logwriter_create(&i_logwriter->logwriter, RSTRING_PTR(rb_ary_entry(args, 0)), SPARKEY_COMPRESSION_NONE, 0);
|
90
|
+
}
|
91
|
+
|
92
|
+
if (returncode != SPARKEY_SUCCESS) {
|
93
|
+
raise_sparkey(returncode);
|
94
|
+
i_logwriter->open = false;
|
95
|
+
} else {
|
96
|
+
i_logwriter->open = true;
|
97
|
+
}
|
98
|
+
|
99
|
+
return self;
|
100
|
+
}
|
101
|
+
|
102
|
+
static VALUE method_logwriter_close(VALUE self) {
|
103
|
+
sparkey_returncode returncode;
|
104
|
+
instance_logwriter *i_logwriter = get_logwriter(self);
|
105
|
+
check_open(i_logwriter->open);
|
106
|
+
|
107
|
+
returncode = sparkey_logwriter_close(&i_logwriter->logwriter);
|
108
|
+
|
109
|
+
if (returncode != SPARKEY_SUCCESS) {
|
110
|
+
raise_sparkey(returncode);
|
111
|
+
}
|
112
|
+
|
113
|
+
i_logwriter->open = false;
|
114
|
+
|
115
|
+
return Qnil;
|
116
|
+
}
|
117
|
+
|
118
|
+
static VALUE method_logwriter_put(VALUE self, VALUE key, VALUE val) {
|
119
|
+
sparkey_returncode returncode;
|
120
|
+
instance_logwriter *i_logwriter = get_logwriter(self);
|
121
|
+
check_open(i_logwriter->open);
|
122
|
+
|
123
|
+
returncode = sparkey_logwriter_put(i_logwriter->logwriter, RSTRING_LEN(key), (uint8_t*)RSTRING_PTR(key), RSTRING_LEN(val), (uint8_t*)RSTRING_PTR(val));
|
124
|
+
|
125
|
+
if (returncode != SPARKEY_SUCCESS) {
|
126
|
+
raise_sparkey(returncode);
|
127
|
+
}
|
128
|
+
|
129
|
+
return Qnil;
|
130
|
+
}
|
131
|
+
|
132
|
+
static VALUE method_logwriter_delete(VALUE self, VALUE key) {
|
133
|
+
sparkey_returncode returncode;
|
134
|
+
instance_logwriter *i_logwriter = get_logwriter(self);
|
135
|
+
check_open(i_logwriter->open);
|
136
|
+
|
137
|
+
returncode = sparkey_logwriter_delete(i_logwriter->logwriter, RSTRING_LEN(key), (uint8_t*)RSTRING_PTR(key));
|
138
|
+
|
139
|
+
if (returncode != SPARKEY_SUCCESS) {
|
140
|
+
raise_sparkey(returncode);
|
141
|
+
}
|
142
|
+
|
143
|
+
return Qnil;
|
144
|
+
}
|
145
|
+
|
146
|
+
static VALUE method_logwriter_flush(VALUE self) {
|
147
|
+
sparkey_returncode returncode;
|
148
|
+
instance_logwriter *i_logwriter = get_logwriter(self);
|
149
|
+
check_open(i_logwriter->open);
|
150
|
+
|
151
|
+
returncode = sparkey_logwriter_flush(i_logwriter->logwriter);
|
152
|
+
|
153
|
+
if (returncode != SPARKEY_SUCCESS) {
|
154
|
+
raise_sparkey(returncode);
|
155
|
+
}
|
156
|
+
|
157
|
+
return Qnil;
|
158
|
+
}
|
159
|
+
|
160
|
+
static VALUE method_logwriter_open(VALUE self) {
|
161
|
+
instance_logwriter *i_logwriter = get_logwriter(self);
|
162
|
+
|
163
|
+
if (i_logwriter->open) {
|
164
|
+
return Qtrue;
|
165
|
+
} else {
|
166
|
+
return Qfalse;
|
167
|
+
}
|
168
|
+
}
|
169
|
+
|
170
|
+
/********************************************************************************/
|
171
|
+
/**************** LOGREADER *****************************************************/
|
172
|
+
|
173
|
+
static void dealloc_logreader(void *p) {
|
174
|
+
instance_logreader *i_logreader = p;
|
175
|
+
|
176
|
+
if (i_logreader->open) {
|
177
|
+
sparkey_logreader_close(&i_logreader->logreader);
|
178
|
+
}
|
179
|
+
|
180
|
+
free(i_logreader);
|
181
|
+
}
|
182
|
+
|
183
|
+
static VALUE alloc_logreader(VALUE klass) {
|
184
|
+
instance_logreader *i_logreader = ALLOC(instance_logreader);
|
185
|
+
return Data_Wrap_Struct(klass, 0, dealloc_logreader, i_logreader);
|
186
|
+
}
|
187
|
+
|
188
|
+
static instance_logreader* get_logreader(VALUE self) {
|
189
|
+
instance_logreader *i_logreader;
|
190
|
+
Data_Get_Struct(self, instance_logreader, i_logreader);
|
191
|
+
return i_logreader;
|
192
|
+
}
|
193
|
+
|
194
|
+
static VALUE method_logreader_initialize(VALUE self, VALUE filename) {
|
195
|
+
sparkey_returncode returncode;
|
196
|
+
instance_logreader *i_logreader = get_logreader(self);
|
197
|
+
Check_Type(filename, T_STRING);
|
198
|
+
|
199
|
+
returncode = sparkey_logreader_open(&i_logreader->logreader, RSTRING_PTR(filename));
|
200
|
+
|
201
|
+
if (returncode != SPARKEY_SUCCESS) {
|
202
|
+
raise_sparkey(returncode);
|
203
|
+
i_logreader->open = false;
|
204
|
+
} else {
|
205
|
+
i_logreader->open = true;
|
206
|
+
}
|
207
|
+
|
208
|
+
return self;
|
209
|
+
}
|
210
|
+
|
211
|
+
static VALUE method_logreader_close(VALUE self) {
|
212
|
+
instance_logreader *i_logreader = get_logreader(self);
|
213
|
+
check_open(i_logreader->open);
|
214
|
+
sparkey_logreader_close(&i_logreader->logreader);
|
215
|
+
i_logreader->open = false;
|
216
|
+
|
217
|
+
return Qnil;
|
218
|
+
}
|
219
|
+
|
220
|
+
static VALUE method_logreader_each(VALUE self) {
|
221
|
+
sparkey_returncode returncode;
|
222
|
+
sparkey_logiter *logiter;
|
223
|
+
instance_logreader *i_logreader = get_logreader(self);
|
224
|
+
check_open(i_logreader->open);
|
225
|
+
|
226
|
+
returncode = sparkey_logiter_create(&logiter, i_logreader->logreader);
|
227
|
+
|
228
|
+
if (returncode != SPARKEY_SUCCESS) {
|
229
|
+
raise_sparkey(returncode);
|
230
|
+
}
|
231
|
+
|
232
|
+
while (1) {
|
233
|
+
returncode = sparkey_logiter_next(logiter, i_logreader->logreader);
|
234
|
+
|
235
|
+
if (sparkey_logiter_state(logiter) != SPARKEY_ITER_ACTIVE) {
|
236
|
+
break;
|
237
|
+
}
|
238
|
+
|
239
|
+
uint64_t wanted_keylen = sparkey_logiter_keylen(logiter);
|
240
|
+
uint8_t *keybuf = malloc(wanted_keylen);
|
241
|
+
uint64_t actual_keylen;
|
242
|
+
returncode = sparkey_logiter_fill_key(logiter, i_logreader->logreader, wanted_keylen, keybuf, &actual_keylen);
|
243
|
+
|
244
|
+
if (returncode != SPARKEY_SUCCESS) {
|
245
|
+
raise_sparkey(returncode);
|
246
|
+
} else if (wanted_keylen != actual_keylen) {
|
247
|
+
rb_raise(GnistaException, "Corrupted read in logreader.");
|
248
|
+
}
|
249
|
+
|
250
|
+
uint64_t wanted_valuelen = sparkey_logiter_valuelen(logiter);
|
251
|
+
uint8_t *valuebuf = malloc(wanted_valuelen);
|
252
|
+
uint64_t actual_valuelen;
|
253
|
+
returncode = sparkey_logiter_fill_value(logiter, i_logreader->logreader, wanted_valuelen, valuebuf, &actual_valuelen);
|
254
|
+
|
255
|
+
if (returncode != SPARKEY_SUCCESS) {
|
256
|
+
raise_sparkey(returncode);
|
257
|
+
} else if (wanted_valuelen != actual_valuelen) {
|
258
|
+
rb_raise(GnistaException, "Corrupted read in logreader.");
|
259
|
+
}
|
260
|
+
|
261
|
+
rb_yield_values(2, rb_str_new((char *)keybuf, actual_keylen), rb_str_new((char *)valuebuf, actual_valuelen));
|
262
|
+
|
263
|
+
free(keybuf);
|
264
|
+
free(valuebuf);
|
265
|
+
}
|
266
|
+
|
267
|
+
sparkey_logiter_close(&logiter);
|
268
|
+
|
269
|
+
return Qnil;
|
270
|
+
}
|
271
|
+
|
272
|
+
static VALUE method_logreader_open(VALUE self) {
|
273
|
+
instance_logreader *i_logreader = get_logreader(self);
|
274
|
+
|
275
|
+
if (i_logreader->open) {
|
276
|
+
return Qtrue;
|
277
|
+
} else {
|
278
|
+
return Qfalse;
|
279
|
+
}
|
280
|
+
}
|
281
|
+
|
282
|
+
/********************************************************************************/
|
283
|
+
/**************** HASH **********************************************************/
|
284
|
+
|
285
|
+
static void dealloc_hashreader(void *p) {
|
286
|
+
instance_hashreader *i_hashreader = p;
|
287
|
+
|
288
|
+
if (i_hashreader->open) {
|
289
|
+
sparkey_hash_close(&i_hashreader->hashreader);
|
290
|
+
}
|
291
|
+
|
292
|
+
free(i_hashreader);
|
293
|
+
}
|
294
|
+
|
295
|
+
static VALUE alloc_hashreader(VALUE klass) {
|
296
|
+
instance_hashreader *i_hashreader = ALLOC(instance_hashreader);
|
297
|
+
return Data_Wrap_Struct(klass, 0, dealloc_hashreader, i_hashreader);
|
298
|
+
}
|
299
|
+
|
300
|
+
static instance_hashreader* get_hashreader(VALUE self) {
|
301
|
+
instance_hashreader *i_hashreader;
|
302
|
+
Data_Get_Struct(self, instance_hashreader, i_hashreader);
|
303
|
+
return i_hashreader;
|
304
|
+
}
|
305
|
+
|
306
|
+
static VALUE method_hash_write(VALUE klass, VALUE args) {
|
307
|
+
sparkey_returncode returncode;
|
308
|
+
int len = RARRAY_LEN(args);
|
309
|
+
int hash_size = 0;
|
310
|
+
|
311
|
+
if (len > 3 || len < 2) {
|
312
|
+
rb_raise(rb_eArgError, "Wrong number of arguments");
|
313
|
+
}
|
314
|
+
|
315
|
+
Check_Type(rb_ary_entry(args, 0), T_STRING);
|
316
|
+
Check_Type(rb_ary_entry(args, 1), T_STRING);
|
317
|
+
|
318
|
+
if (len == 3) {
|
319
|
+
Check_Type(rb_ary_entry(args, 2), T_FIXNUM);
|
320
|
+
hash_size = NUM2INT(rb_ary_entry(args, 2));
|
321
|
+
}
|
322
|
+
|
323
|
+
returncode = sparkey_hash_write(RSTRING_PTR(rb_ary_entry(args, 0)), RSTRING_PTR(rb_ary_entry(args, 1)), hash_size);
|
324
|
+
|
325
|
+
if (returncode != SPARKEY_SUCCESS) {
|
326
|
+
raise_sparkey(returncode);
|
327
|
+
}
|
328
|
+
|
329
|
+
return Qnil;
|
330
|
+
}
|
331
|
+
|
332
|
+
static VALUE method_hash_initialize(VALUE self, VALUE hash_filename, VALUE log_filename) {
|
333
|
+
sparkey_returncode returncode;
|
334
|
+
instance_hashreader *i_hashreader = get_hashreader(self);
|
335
|
+
|
336
|
+
Check_Type(hash_filename, T_STRING);
|
337
|
+
Check_Type(log_filename, T_STRING);
|
338
|
+
|
339
|
+
returncode = sparkey_hash_open(&i_hashreader->hashreader, RSTRING_PTR(hash_filename), RSTRING_PTR(log_filename));
|
340
|
+
|
341
|
+
if (returncode != SPARKEY_SUCCESS) {
|
342
|
+
raise_sparkey(returncode);
|
343
|
+
i_hashreader->open = false;
|
344
|
+
} else {
|
345
|
+
i_hashreader->open = true;
|
346
|
+
}
|
347
|
+
|
348
|
+
return Qnil;
|
349
|
+
}
|
350
|
+
|
351
|
+
static VALUE method_hash_close(VALUE self) {
|
352
|
+
instance_hashreader *i_hashreader = get_hashreader(self);
|
353
|
+
check_open(i_hashreader->open);
|
354
|
+
sparkey_hash_close(&i_hashreader->hashreader);
|
355
|
+
i_hashreader->open = false;
|
356
|
+
|
357
|
+
return Qnil;
|
358
|
+
}
|
359
|
+
|
360
|
+
static VALUE method_hash_each(VALUE self) {
|
361
|
+
sparkey_returncode returncode;
|
362
|
+
instance_hashreader *i_hashreader = get_hashreader(self);
|
363
|
+
sparkey_logiter *logiter;
|
364
|
+
check_open(i_hashreader->open);
|
365
|
+
|
366
|
+
returncode = sparkey_logiter_create(&logiter, sparkey_hash_getreader(i_hashreader->hashreader));
|
367
|
+
|
368
|
+
if (returncode != SPARKEY_SUCCESS) {
|
369
|
+
raise_sparkey(returncode);
|
370
|
+
}
|
371
|
+
|
372
|
+
while (1) {
|
373
|
+
returncode = sparkey_logiter_hashnext(logiter, i_hashreader->hashreader);
|
374
|
+
|
375
|
+
if (sparkey_logiter_state(logiter) != SPARKEY_ITER_ACTIVE) {
|
376
|
+
break;
|
377
|
+
}
|
378
|
+
|
379
|
+
uint64_t wanted_keylen = sparkey_logiter_keylen(logiter);
|
380
|
+
uint8_t *keybuf = malloc(wanted_keylen);
|
381
|
+
uint64_t actual_keylen;
|
382
|
+
returncode = sparkey_logiter_fill_key(logiter, sparkey_hash_getreader(i_hashreader->hashreader), wanted_keylen, keybuf, &actual_keylen);
|
383
|
+
|
384
|
+
if (returncode != SPARKEY_SUCCESS) {
|
385
|
+
raise_sparkey(returncode);
|
386
|
+
} else if (wanted_keylen != actual_keylen) {
|
387
|
+
rb_raise(GnistaException, "Corrupt entry in logreader.");
|
388
|
+
}
|
389
|
+
|
390
|
+
uint64_t wanted_valuelen = sparkey_logiter_valuelen(logiter);
|
391
|
+
uint8_t *valuebuf = malloc(wanted_valuelen);
|
392
|
+
uint64_t actual_valuelen;
|
393
|
+
returncode = sparkey_logiter_fill_value(logiter, sparkey_hash_getreader(i_hashreader->hashreader), wanted_valuelen, valuebuf, &actual_valuelen);
|
394
|
+
|
395
|
+
if (returncode != SPARKEY_SUCCESS) {
|
396
|
+
raise_sparkey(returncode);
|
397
|
+
} else if (wanted_valuelen != actual_valuelen) {
|
398
|
+
rb_raise(GnistaException, "Corrupt entry in logreader.");
|
399
|
+
}
|
400
|
+
|
401
|
+
rb_yield_values(2, rb_str_new((char *)keybuf, actual_keylen), rb_str_new((char *)valuebuf, actual_valuelen));
|
402
|
+
|
403
|
+
free(keybuf);
|
404
|
+
free(valuebuf);
|
405
|
+
}
|
406
|
+
|
407
|
+
sparkey_logiter_close(&logiter);
|
408
|
+
|
409
|
+
return Qnil;
|
410
|
+
}
|
411
|
+
|
412
|
+
static VALUE method_hash_get(VALUE self, VALUE key) {
|
413
|
+
sparkey_returncode returncode;
|
414
|
+
instance_hashreader *i_hashreader = get_hashreader(self);
|
415
|
+
sparkey_logiter *logiter;
|
416
|
+
check_open(i_hashreader->open);
|
417
|
+
|
418
|
+
returncode = sparkey_logiter_create(&logiter, sparkey_hash_getreader(i_hashreader->hashreader));
|
419
|
+
|
420
|
+
if (returncode != SPARKEY_SUCCESS) {
|
421
|
+
raise_sparkey(returncode);
|
422
|
+
}
|
423
|
+
|
424
|
+
key = StringValue(key);
|
425
|
+
|
426
|
+
returncode = sparkey_hash_get(i_hashreader->hashreader, (uint8_t*)RSTRING_PTR(key), RSTRING_LEN(key), logiter);
|
427
|
+
|
428
|
+
if (sparkey_logiter_state(logiter) != SPARKEY_ITER_ACTIVE) {
|
429
|
+
return Qnil;
|
430
|
+
}
|
431
|
+
|
432
|
+
uint64_t wanted_valuelen = sparkey_logiter_valuelen(logiter);
|
433
|
+
uint8_t *valuebuf = malloc(wanted_valuelen);
|
434
|
+
uint64_t actual_valuelen;
|
435
|
+
returncode = sparkey_logiter_fill_value(logiter, sparkey_hash_getreader(i_hashreader->hashreader), wanted_valuelen, valuebuf, &actual_valuelen);
|
436
|
+
|
437
|
+
if (returncode != SPARKEY_SUCCESS) {
|
438
|
+
raise_sparkey(returncode);
|
439
|
+
} else if (wanted_valuelen != actual_valuelen) {
|
440
|
+
rb_raise(GnistaException, "Corrupt entry in hash.");
|
441
|
+
}
|
442
|
+
|
443
|
+
sparkey_logiter_close(&logiter);
|
444
|
+
|
445
|
+
return rb_str_new((char *)valuebuf, actual_valuelen);
|
446
|
+
}
|
447
|
+
|
448
|
+
static VALUE method_hash_maxkeylen(VALUE self) {
|
449
|
+
instance_hashreader *i_hashreader = get_hashreader(self);
|
450
|
+
uint64_t maxkeylen;
|
451
|
+
check_open(i_hashreader->open);
|
452
|
+
|
453
|
+
maxkeylen = sparkey_logreader_maxkeylen(sparkey_hash_getreader(i_hashreader->hashreader));
|
454
|
+
|
455
|
+
return INT2NUM(maxkeylen);
|
456
|
+
}
|
457
|
+
|
458
|
+
static VALUE method_hash_maxvaluelen(VALUE self) {
|
459
|
+
instance_hashreader *i_hashreader = get_hashreader(self);
|
460
|
+
uint64_t maxvaluelen;
|
461
|
+
check_open(i_hashreader->open);
|
462
|
+
|
463
|
+
maxvaluelen = sparkey_logreader_maxvaluelen(sparkey_hash_getreader(i_hashreader->hashreader));
|
464
|
+
|
465
|
+
return INT2NUM(maxvaluelen);
|
466
|
+
}
|
467
|
+
|
468
|
+
static VALUE method_hash_length(VALUE self) {
|
469
|
+
instance_hashreader *i_hashreader = get_hashreader(self);
|
470
|
+
uint64_t numentries;
|
471
|
+
check_open(i_hashreader->open);
|
472
|
+
|
473
|
+
numentries = sparkey_hash_numentries(i_hashreader->hashreader);
|
474
|
+
|
475
|
+
return INT2NUM(numentries);
|
476
|
+
}
|
477
|
+
|
478
|
+
static VALUE method_hash_collisions(VALUE self) {
|
479
|
+
instance_hashreader *i_hashreader = get_hashreader(self);
|
480
|
+
uint64_t collisions;
|
481
|
+
check_open(i_hashreader->open);
|
482
|
+
|
483
|
+
collisions = sparkey_hash_numcollisions(i_hashreader->hashreader);
|
484
|
+
|
485
|
+
return INT2NUM(collisions);
|
486
|
+
}
|
487
|
+
|
488
|
+
static VALUE method_hash_open(VALUE self) {
|
489
|
+
instance_hashreader *i_hashreader = get_hashreader(self);
|
490
|
+
|
491
|
+
if (i_hashreader->open) {
|
492
|
+
return Qtrue;
|
493
|
+
} else {
|
494
|
+
return Qfalse;
|
495
|
+
}
|
496
|
+
}
|
497
|
+
|
498
|
+
/********************************************************************************/
|
499
|
+
/**************** INITIALIZE GNISTA *********************************************/
|
500
|
+
|
501
|
+
void Init_gnista() {
|
502
|
+
VALUE Gnista = rb_define_module("Gnista");
|
503
|
+
VALUE Logwriter = rb_define_class_under(Gnista, "Logwriter", rb_cObject);
|
504
|
+
VALUE Logreader = rb_define_class_under(Gnista, "Logreader", rb_cObject);
|
505
|
+
VALUE Hash = rb_define_class_under(Gnista, "Hash", rb_cObject);
|
506
|
+
|
507
|
+
rb_define_alloc_func(Logwriter, alloc_logwriter);
|
508
|
+
rb_define_method(Logwriter, "initialize", method_logwriter_initialize, -2);
|
509
|
+
rb_define_method(Logwriter, "close", method_logwriter_close, 0);
|
510
|
+
rb_define_method(Logwriter, "put", method_logwriter_put, 2);
|
511
|
+
rb_define_method(Logwriter, "delete", method_logwriter_delete, 1);
|
512
|
+
rb_define_method(Logwriter, "flush", method_logwriter_flush, 0);
|
513
|
+
rb_define_method(Logwriter, "open?", method_logwriter_open, 0);
|
514
|
+
|
515
|
+
rb_define_alloc_func(Logreader, alloc_logreader);
|
516
|
+
rb_define_method(Logreader, "initialize", method_logreader_initialize, 1);
|
517
|
+
rb_define_method(Logreader, "close", method_logreader_close, 0);
|
518
|
+
rb_define_method(Logreader, "each", method_logreader_each, 0);
|
519
|
+
rb_define_method(Logreader, "open?", method_logreader_open, 0);
|
520
|
+
|
521
|
+
rb_define_alloc_func(Hash, alloc_hashreader);
|
522
|
+
rb_define_singleton_method(Hash, "write", method_hash_write, -2);
|
523
|
+
rb_define_method(Hash, "initialize", method_hash_initialize, 2);
|
524
|
+
rb_define_method(Hash, "close", method_hash_close, 0);
|
525
|
+
rb_define_method(Hash, "each", method_hash_each, 0);
|
526
|
+
rb_define_method(Hash, "get", method_hash_get, 1);
|
527
|
+
rb_define_method(Hash, "maxkeylen", method_hash_maxkeylen, 0);
|
528
|
+
rb_define_method(Hash, "maxvaluelen", method_hash_maxvaluelen, 0);
|
529
|
+
rb_define_method(Hash, "length", method_hash_length, 0);
|
530
|
+
rb_define_method(Hash, "collisions", method_hash_collisions, 0);
|
531
|
+
rb_define_method(Hash, "open?", method_hash_open, 0);
|
532
|
+
|
533
|
+
GnistaException = rb_define_class("GnistaException", rb_eException);
|
534
|
+
}
|
data/lib/gnista.rb
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
require 'minitest/spec'
|
2
|
+
require 'minitest/autorun'
|
3
|
+
require 'fileutils'
|
4
|
+
require File.expand_path("../../lib/gnista", __FILE__)
|
5
|
+
|
6
|
+
describe Gnista do
|
7
|
+
|
8
|
+
log_path = '/tmp/test.log'
|
9
|
+
hash_path = '/tmp/test.hash'
|
10
|
+
|
11
|
+
before do
|
12
|
+
@logwriter = Gnista::Logwriter.new log_path
|
13
|
+
@logreader = Gnista::Logreader.new log_path
|
14
|
+
end
|
15
|
+
|
16
|
+
after do
|
17
|
+
@logwriter.close
|
18
|
+
@logreader.close
|
19
|
+
FileUtils.rm log_path
|
20
|
+
end
|
21
|
+
|
22
|
+
it "can use put command" do
|
23
|
+
res = @logwriter.put "key1", "value1"
|
24
|
+
res.must_be_nil
|
25
|
+
end
|
26
|
+
|
27
|
+
it "can use delete command" do
|
28
|
+
res = @logwriter.delete "key10"
|
29
|
+
res.must_be_nil
|
30
|
+
end
|
31
|
+
|
32
|
+
it "can use flush command" do
|
33
|
+
@logwriter.flush.must_be_nil
|
34
|
+
end
|
35
|
+
|
36
|
+
it "can iterate each log entry" do
|
37
|
+
@logwriter.put "key1", "value1"
|
38
|
+
@logwriter.put "key2", "value2"
|
39
|
+
|
40
|
+
a = true
|
41
|
+
@logreader.each do |key,value|
|
42
|
+
if a
|
43
|
+
key.must_equal "key1"
|
44
|
+
value.must_equal "value1"
|
45
|
+
a = false
|
46
|
+
else
|
47
|
+
key.must_equal "key2"
|
48
|
+
value.must_equal "value2"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
it "can read hash max key/value length" do
|
55
|
+
@logwriter.put "abc", "a"
|
56
|
+
@logwriter.put "a", "abcd"
|
57
|
+
@logwriter.flush
|
58
|
+
Gnista::Hash.write hash_path, log_path
|
59
|
+
|
60
|
+
hash = Gnista::Hash.new hash_path, log_path
|
61
|
+
|
62
|
+
hash.maxkeylen.must_equal 3
|
63
|
+
hash.maxvaluelen.must_equal 4
|
64
|
+
FileUtils.rm hash_path
|
65
|
+
end
|
66
|
+
|
67
|
+
it "can iterate each hash entry" do
|
68
|
+
@logwriter.put "key1", "value1"
|
69
|
+
@logwriter.put "key2", "value2"
|
70
|
+
@logwriter.flush
|
71
|
+
Gnista::Hash.write hash_path, log_path
|
72
|
+
|
73
|
+
hash = Gnista::Hash.new hash_path, log_path
|
74
|
+
|
75
|
+
a = true
|
76
|
+
hash.each do |key,value|
|
77
|
+
if a
|
78
|
+
key.must_equal "key1"
|
79
|
+
value.must_equal "value1"
|
80
|
+
a = false
|
81
|
+
else
|
82
|
+
key.must_equal "key2"
|
83
|
+
value.must_equal "value2"
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
hash.close
|
88
|
+
FileUtils.rm hash_path
|
89
|
+
end
|
90
|
+
|
91
|
+
it "can get from hash" do
|
92
|
+
@logwriter.put "key1", "value1"
|
93
|
+
@logwriter.put "key2", "value2"
|
94
|
+
@logwriter.delete "key2"
|
95
|
+
@logwriter.flush
|
96
|
+
Gnista::Hash.write hash_path, log_path
|
97
|
+
|
98
|
+
hash = Gnista::Hash.new hash_path, log_path
|
99
|
+
ret = hash.get "key1"
|
100
|
+
ret.must_equal "value1"
|
101
|
+
|
102
|
+
ret = hash.get "key2"
|
103
|
+
ret.must_be_nil
|
104
|
+
FileUtils.rm hash_path
|
105
|
+
end
|
106
|
+
|
107
|
+
it "can get the hash length" do
|
108
|
+
@logwriter.put "key1", "value1"
|
109
|
+
@logwriter.put "key2", "value2"
|
110
|
+
@logwriter.put "key3", "value3"
|
111
|
+
@logwriter.flush
|
112
|
+
Gnista::Hash.write hash_path, log_path
|
113
|
+
|
114
|
+
hash = Gnista::Hash.new hash_path, log_path
|
115
|
+
hash.length.must_equal 3
|
116
|
+
FileUtils.rm hash_path
|
117
|
+
end
|
118
|
+
|
119
|
+
it "can get the hash collisions count" do
|
120
|
+
# This is hard to test
|
121
|
+
end
|
122
|
+
|
123
|
+
it "can be open or closed" do
|
124
|
+
@logwriter.open?.must_equal true
|
125
|
+
@logreader.open?.must_equal true
|
126
|
+
Gnista::Hash.write hash_path, log_path
|
127
|
+
hash = Gnista::Hash.new hash_path, log_path
|
128
|
+
hash.open?.must_equal true
|
129
|
+
hash.close
|
130
|
+
hash.open?.must_equal false
|
131
|
+
FileUtils.rm hash_path
|
132
|
+
end
|
133
|
+
|
134
|
+
end
|
data/test/test_gnista.rb
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
require 'minitest/spec'
|
2
|
+
require 'minitest/autorun'
|
3
|
+
require 'fileutils'
|
4
|
+
require File.expand_path("../../lib/gnista", __FILE__)
|
5
|
+
|
6
|
+
describe Gnista do
|
7
|
+
|
8
|
+
log_path = '/tmp/test.log'
|
9
|
+
hash_path = '/tmp/test.hash'
|
10
|
+
|
11
|
+
it "can create a Logwriter with only a filename" do
|
12
|
+
@logwriter = Gnista::Logwriter.new log_path
|
13
|
+
@logwriter.must_be_instance_of Gnista::Logwriter
|
14
|
+
@logwriter.close
|
15
|
+
FileUtils.rm log_path
|
16
|
+
end
|
17
|
+
|
18
|
+
it "can create a Logwriter with a filename and a compression number" do
|
19
|
+
@logwriter = Gnista::Logwriter.new log_path, 4000
|
20
|
+
@logwriter.must_be_instance_of Gnista::Logwriter
|
21
|
+
@logwriter.close
|
22
|
+
FileUtils.rm log_path
|
23
|
+
end
|
24
|
+
|
25
|
+
it "can append a Logwriter" do
|
26
|
+
@logwriter = Gnista::Logwriter.new log_path
|
27
|
+
@logwriter.close
|
28
|
+
|
29
|
+
@logwriter = Gnista::Logwriter.new log_path, :append
|
30
|
+
@logwriter.must_be_instance_of Gnista::Logwriter
|
31
|
+
@logwriter.close
|
32
|
+
FileUtils.rm log_path
|
33
|
+
end
|
34
|
+
|
35
|
+
it "can create a Logreader with a filename" do
|
36
|
+
@logwriter = Gnista::Logwriter.new log_path
|
37
|
+
@logwriter.close
|
38
|
+
|
39
|
+
@logreader = Gnista::Logreader.new log_path
|
40
|
+
@logreader.must_be_instance_of Gnista::Logreader
|
41
|
+
@logreader.close
|
42
|
+
FileUtils.rm log_path
|
43
|
+
end
|
44
|
+
|
45
|
+
it "can write and read a Hash with a log-file and a hash-file" do
|
46
|
+
@logwriter = Gnista::Logwriter.new log_path
|
47
|
+
@logwriter.close
|
48
|
+
|
49
|
+
Gnista::Hash.write hash_path, log_path
|
50
|
+
@hash = Gnista::Hash.new hash_path, log_path
|
51
|
+
@hash.must_be_instance_of Gnista::Hash
|
52
|
+
@hash.close
|
53
|
+
FileUtils.rm hash_path
|
54
|
+
FileUtils.rm log_path
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
metadata
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: gnista
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Emanuel Andersson
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-09-11 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.3'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.3'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
description: Gnista is a wrapper for the database/hashstore Sparkey written for Ruby.
|
42
|
+
Gnista packs all the native features of Sparkey into a shiny red package, easy to
|
43
|
+
use.
|
44
|
+
email:
|
45
|
+
- manusdude@gmail.com
|
46
|
+
executables: []
|
47
|
+
extensions:
|
48
|
+
- ext/gnista/extconf.rb
|
49
|
+
extra_rdoc_files: []
|
50
|
+
files:
|
51
|
+
- README.md
|
52
|
+
- LICENSE.txt
|
53
|
+
- HISTORY.md
|
54
|
+
- Rakefile
|
55
|
+
- ext/gnista/extconf.rb
|
56
|
+
- ext/gnista/gnista.c
|
57
|
+
- test/test_gnista.rb
|
58
|
+
- test/test_commands.rb
|
59
|
+
- lib/gnista.rb
|
60
|
+
- lib/gnista/version.rb
|
61
|
+
homepage: http://github.com/emnl/gnista
|
62
|
+
licenses:
|
63
|
+
- MIT
|
64
|
+
metadata: {}
|
65
|
+
post_install_message:
|
66
|
+
rdoc_options: []
|
67
|
+
require_paths:
|
68
|
+
- ext
|
69
|
+
- lib
|
70
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - '>='
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - '>='
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0'
|
80
|
+
requirements: []
|
81
|
+
rubyforge_project:
|
82
|
+
rubygems_version: 2.0.0
|
83
|
+
signing_key:
|
84
|
+
specification_version: 4
|
85
|
+
summary: Ruby wrapper for Spotify's Sparkey
|
86
|
+
test_files:
|
87
|
+
- test/test_gnista.rb
|
88
|
+
- test/test_commands.rb
|