localmemcache 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +18 -11
- data/Rakefile +22 -7
- data/VERSION +1 -1
- data/bench/common.rb +7 -0
- data/bench/gdbm_vs_lmc +10 -0
- data/bench/gdbm_vs_lmc.rb +29 -0
- data/bench/lmc_bench +11 -0
- data/bench/lmc_bench.rb +27 -0
- data/bench/memcached_bench +8 -0
- data/bench/memcached_bench.rb +19 -0
- data/bench/tyrant_bench +10 -0
- data/bench/tyrant_bench.rb +19 -0
- data/example/hello.c +1 -1
- data/site/doc/classes/LocalMemCache/ArgError.html +113 -0
- data/site/doc/classes/LocalMemCache/InitError.html +113 -0
- data/site/doc/classes/LocalMemCache/LocalMemCacheError.html +111 -0
- data/site/doc/classes/LocalMemCache/LockError.html +113 -0
- data/site/doc/classes/LocalMemCache/LockTimedOut.html +113 -0
- data/site/doc/classes/LocalMemCache/MemoryPoolClosed.html +113 -0
- data/site/doc/classes/LocalMemCache/MemoryPoolFull.html +113 -0
- data/site/doc/classes/LocalMemCache/OutOfMemoryError.html +113 -0
- data/site/doc/classes/LocalMemCache/RecoveryFailed.html +113 -0
- data/site/doc/classes/LocalMemCache/ShmError.html +113 -0
- data/site/doc/classes/LocalMemCache/ShmLockFailed.html +113 -0
- data/site/doc/classes/LocalMemCache/ShmUnlockFailed.html +113 -0
- data/site/doc/classes/LocalMemCache.html +515 -0
- data/site/doc/classes/LocalMemCache.src/M000001.html +19 -0
- data/site/doc/classes/LocalMemCache.src/M000002.html +18 -0
- data/site/doc/classes/LocalMemCache.src/M000003.html +18 -0
- data/site/doc/classes/LocalMemCache.src/M000004.html +39 -0
- data/site/doc/classes/LocalMemCache.src/M000005.html +29 -0
- data/site/doc/classes/LocalMemCache.src/M000006.html +23 -0
- data/site/doc/classes/LocalMemCache.src/M000007.html +23 -0
- data/site/doc/classes/LocalMemCache.src/M000008.html +22 -0
- data/site/doc/classes/LocalMemCache.src/M000009.html +24 -0
- data/site/doc/classes/LocalMemCache.src/M000010.html +24 -0
- data/site/doc/classes/LocalMemCache.src/M000011.html +22 -0
- data/site/doc/classes/LocalMemCache.src/M000012.html +22 -0
- data/site/doc/created.rid +1 -0
- data/site/doc/files/__/src/ruby-binding/extconf_rb.html +108 -0
- data/site/doc/files/__/src/ruby-binding/localmemcache_rb.html +108 -0
- data/site/doc/files/__/src/ruby-binding/rblocalmemcache_c.html +101 -0
- data/site/doc/fr_class_index.html +39 -0
- data/site/doc/fr_file_index.html +28 -0
- data/site/doc/fr_method_index.html +38 -0
- data/site/doc/index.html +24 -0
- data/site/doc/rdoc-style.css +208 -0
- data/site/index.html +50 -46
- data/src/lmc_common.c +22 -0
- data/src/lmc_common.h +4 -0
- data/src/lmc_hashtable.h +1 -1
- data/src/lmc_lock.c +17 -3
- data/src/lmc_shm.c +4 -2
- data/src/lmc_valloc.c +6 -5
- data/src/lmc_valloc.h +1 -0
- data/src/localmemcache.c +56 -35
- data/src/localmemcache.h +161 -4
- data/src/ruby-binding/localmemcache.rb +48 -16
- data/src/ruby-binding/rblocalmemcache.c +131 -24
- data/src/tests/bench.rb +1 -1
- data/src/tests/lmc.rb +11 -2
- metadata +48 -7
- data/INTERNALS +0 -26
- data/example/hello.bin +0 -0
data/src/localmemcache.c
CHANGED
@@ -33,30 +33,35 @@ int lmc_release_lock_flag(void *base, lmc_error_t *e) {
|
|
33
33
|
return 1;
|
34
34
|
}
|
35
35
|
|
36
|
-
|
37
|
-
|
38
|
-
if (
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
if (
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
36
|
+
int lmc_namespace_or_filename(char *result, const char* ons, const char *ofn,
|
37
|
+
lmc_error_t *e) {
|
38
|
+
if (ons) {
|
39
|
+
lmc_clean_string(result, ons);
|
40
|
+
return 1;
|
41
|
+
}
|
42
|
+
if (ofn) {
|
43
|
+
size_t n = strlen(ofn);
|
44
|
+
if (n > 1010) { n = 1010; }
|
45
|
+
char *d = result;
|
46
|
+
if (!lmc_is_filename(ofn)) {
|
47
|
+
strcpy(d, "./");
|
48
|
+
d += 2;
|
49
49
|
}
|
50
|
+
strcpy(d, ofn);
|
51
|
+
return 1;
|
50
52
|
}
|
51
|
-
|
53
|
+
lmc_handle_error_with_err_string("lmc_namespace_or_filename",
|
54
|
+
"Need to supply either namespace or filename argument", "ArgError", e);
|
55
|
+
return 0;
|
52
56
|
}
|
53
57
|
|
54
|
-
int local_memcache_clear_namespace(const char *namespace,
|
55
|
-
lmc_error_t *e) {
|
58
|
+
int local_memcache_clear_namespace(const char *namespace, const char *filename,
|
59
|
+
int repair, lmc_error_t *e) {
|
56
60
|
char clean_ns[1024];
|
57
|
-
|
61
|
+
if (!lmc_namespace_or_filename((char *)clean_ns, namespace, filename, e))
|
62
|
+
return 1;
|
58
63
|
lmc_clean_namespace((char *)clean_ns, e);
|
59
|
-
if (repair) {
|
64
|
+
if (repair) {
|
60
65
|
lmc_lock_t *l = lmc_lock_init((char *)clean_ns, 1, e);
|
61
66
|
lmc_lock_repair(l);
|
62
67
|
free(l);
|
@@ -70,6 +75,8 @@ int local_memcache_clear_namespace(const char *namespace, int repair,
|
|
70
75
|
return 1;
|
71
76
|
}
|
72
77
|
|
78
|
+
int __local_memcache_check_namespace(const char *clean_ns, lmc_error_t *e);
|
79
|
+
|
73
80
|
local_memcache_t *__local_memcache_create(const char *namespace, size_t size,
|
74
81
|
int force, int *ok, lmc_error_t* e) {
|
75
82
|
int d;
|
@@ -89,11 +96,11 @@ retry:
|
|
89
96
|
}
|
90
97
|
if (!lmc_is_lock_working(lmc->lock, e)) {
|
91
98
|
if (!force) {
|
92
|
-
if (
|
99
|
+
if (__local_memcache_check_namespace(namespace, e)) goto retry;
|
93
100
|
lmc_handle_error_with_err_string("local_memcache_create",
|
94
101
|
"Failed to repair shared memory!", "ShmLockFailed", e);
|
95
102
|
goto failed;
|
96
|
-
}
|
103
|
+
}
|
97
104
|
*ok = 0;
|
98
105
|
}
|
99
106
|
{
|
@@ -133,26 +140,26 @@ failed:
|
|
133
140
|
return NULL;
|
134
141
|
}
|
135
142
|
|
136
|
-
local_memcache_t *local_memcache_create(const char *namespace,
|
137
|
-
lmc_error_t* e) {
|
143
|
+
local_memcache_t *local_memcache_create(const char *namespace,
|
144
|
+
const char *filename, double size_mb, lmc_error_t* e) {
|
138
145
|
char clean_ns[1024];
|
139
146
|
double s = size_mb == 0.0 ? 1024.0 : size_mb;
|
140
147
|
size_t si = s * 1024 * 1024;
|
141
148
|
//printf("size: %f, s: %f, si: %zd\n", size_mb, s, si);
|
142
|
-
|
149
|
+
if (!lmc_namespace_or_filename((char *)clean_ns, namespace, filename, e))
|
150
|
+
return 0;
|
143
151
|
return __local_memcache_create((char *)clean_ns, si, 0, 0, e);
|
144
152
|
}
|
145
153
|
|
146
|
-
int
|
147
|
-
char clean_ns[1024];
|
148
|
-
lmc_clean_namespace_string((char *)clean_ns, namespace);
|
154
|
+
int __local_memcache_check_namespace(const char *clean_ns, lmc_error_t *e) {
|
149
155
|
char check_lock_name[1024];
|
150
156
|
snprintf((char *)check_lock_name, 1023, "%s-check", (char *)clean_ns);
|
151
157
|
|
152
158
|
if (!lmc_does_namespace_exist((char *)clean_ns)) {
|
153
159
|
lmc_clear_namespace_lock(check_lock_name);
|
154
|
-
lmc_clear_namespace_lock(
|
155
|
-
|
160
|
+
lmc_clear_namespace_lock(clean_ns);
|
161
|
+
fprintf(stderr, "[localmemcache] namespace '%s' does not exist!\n",
|
162
|
+
(char *)clean_ns);
|
156
163
|
return 1;
|
157
164
|
}
|
158
165
|
|
@@ -170,12 +177,14 @@ int local_memcache_check_namespace(const char *namespace, lmc_error_t *e) {
|
|
170
177
|
local_memcache_t *lmc = __local_memcache_create((char *)clean_ns, ns_size,
|
171
178
|
1, &ok, e);
|
172
179
|
if (!lmc) {
|
173
|
-
|
180
|
+
lmc_handle_error_with_err_string("__local_memcache_create",
|
181
|
+
"Unable to attach memory pool", "InitError", e);
|
174
182
|
goto failed;
|
175
183
|
}
|
176
184
|
md = lmc->base;
|
177
185
|
if (!ok) {
|
178
|
-
|
186
|
+
fprintf(stderr, "[localmemcache] Auto repairing namespace '%s'\n",
|
187
|
+
clean_ns);
|
179
188
|
if (!md->locked) goto release;
|
180
189
|
if (md->log.op_id == 0) goto unlock_and_release;
|
181
190
|
if (ht_redo(lmc->base, md->va_hash, &md->log, e)) goto unlock_and_release;
|
@@ -208,24 +217,36 @@ failed:
|
|
208
217
|
lmc_lock_release("local_memcache_check_namespace", check_l, e);
|
209
218
|
check_lock_failed:
|
210
219
|
free(check_l);
|
211
|
-
|
220
|
+
fprintf(stderr, "[localmemcache] Failed to repair namespace '%s'\n",
|
221
|
+
clean_ns);
|
212
222
|
return 0;
|
213
223
|
}
|
214
224
|
|
225
|
+
int local_memcache_check_namespace(const char *namespace, const char *filename,
|
226
|
+
lmc_error_t *e) {
|
227
|
+
char clean_ns[1024];
|
228
|
+
if (!lmc_namespace_or_filename((char *)clean_ns, namespace, filename, e))
|
229
|
+
return 0;
|
230
|
+
return __local_memcache_check_namespace(clean_ns, e);
|
231
|
+
}
|
232
|
+
|
233
|
+
|
215
234
|
|
216
235
|
int lmc_lock_shm_region(const char *who, local_memcache_t *lmc) {
|
217
236
|
int r;
|
218
237
|
int retry_counter = 0;
|
219
238
|
retry:
|
220
239
|
if (retry_counter++ > 10) {
|
221
|
-
|
222
|
-
lmc->namespace);
|
240
|
+
fprintf(stderr, "[localmemcache] Too many retries: "
|
241
|
+
"Cannot repair namespace '%s'\n", lmc->namespace);
|
223
242
|
return 0;
|
224
243
|
}
|
225
244
|
r = lmc_lock_obtain(who, lmc->lock, &lmc->error);
|
226
245
|
if (!r && (strcmp(lmc->error.error_type, "LockTimedOut") == 0)) {
|
227
|
-
if (
|
228
|
-
|
246
|
+
if (__local_memcache_check_namespace(lmc->namespace,
|
247
|
+
&lmc->error)) goto retry;
|
248
|
+
fprintf(stderr, "[localmemcache] Cannot repair namespace '%s'\n",
|
249
|
+
lmc->namespace);
|
229
250
|
}
|
230
251
|
if (!r) return 0;
|
231
252
|
if (!lmc_set_lock_flag(lmc->base, &lmc->error)) {
|
data/src/localmemcache.h
CHANGED
@@ -15,6 +15,70 @@
|
|
15
15
|
#define LOCAL_MEMCACHE_FAILED 0
|
16
16
|
#define LOCAL_MEMCACHE_SUCCESS 1
|
17
17
|
|
18
|
+
/*
|
19
|
+
* LocalMemCache provides for a Hashtable of strings in shared memory (via a
|
20
|
+
* memory mapped file), which thus can be shared between processes on a
|
21
|
+
* computer. Here is an example of its usage:
|
22
|
+
*
|
23
|
+
* #include <stdio.h>
|
24
|
+
* #include <localmemcache.h>
|
25
|
+
*
|
26
|
+
* int main() {
|
27
|
+
* lmc_error_t e;
|
28
|
+
* // To use a filename instead of a namespace:
|
29
|
+
* // lmc = local_memcache_create(0, "filename.lmc", 0, &e);
|
30
|
+
* local_memcache_t *lmc = local_memcache_create("viewcounters", 0, 0, &e);
|
31
|
+
* if (!lmc) {
|
32
|
+
* fprintf(stderr, "Couldn't create localmemcache: %s\n", e.error_str);
|
33
|
+
* return 1;
|
34
|
+
* }
|
35
|
+
* if (!local_memcache_set(lmc, "foo", 3, "1", 1)) goto failed;
|
36
|
+
* size_t n_value;
|
37
|
+
* char *value = local_memcache_get_new(lmc, "foo", 3, &n_value);
|
38
|
+
* if (!value) goto failed;
|
39
|
+
* free(value);
|
40
|
+
* if (!local_memcache_delete(lmc, "foo", 3)) goto failed;
|
41
|
+
* if (!local_memcache_free(lmc, &e)) {
|
42
|
+
* fprintf(stderr, "Failed to release localmemcache: %s\n", e.error_str);
|
43
|
+
* return 1;
|
44
|
+
* }
|
45
|
+
*
|
46
|
+
* return 0;
|
47
|
+
*
|
48
|
+
* failed:
|
49
|
+
* fprintf(stderr, "%s\n", lmc->error.error_str);
|
50
|
+
* return 1;
|
51
|
+
*
|
52
|
+
* }
|
53
|
+
*
|
54
|
+
* == Default sizes of memory pools
|
55
|
+
*
|
56
|
+
* The default size for memory pools is 1024 (MB). It cannot be changed later,
|
57
|
+
* so choose a size that will provide enough space for all your data. You
|
58
|
+
* might consider setting this size to the maximum filesize of your
|
59
|
+
* filesystem. Also note that while these memory pools may look large on your
|
60
|
+
* disk, they really aren't, because with sparse files only those parts of the
|
61
|
+
* file which contain non-null data actually use disk space.
|
62
|
+
*
|
63
|
+
* == Automatic recovery from crashes
|
64
|
+
*
|
65
|
+
* In case a process is terminated while accessing a memory pool, other
|
66
|
+
* processes will wait for the lock up to 2 seconds, and will then try to
|
67
|
+
* resume the aborted operation. This can also be done explicitly by using
|
68
|
+
* LocalMemCache.check(options).
|
69
|
+
*
|
70
|
+
* == Clearing memory pools
|
71
|
+
*
|
72
|
+
* Removing memory pools can be done with LocalMemCache.clear(options).
|
73
|
+
*
|
74
|
+
* == Environment
|
75
|
+
*
|
76
|
+
* If you use the :namespace parameter, the .lmc file for your namespace will
|
77
|
+
* reside in /var/tmp/localmemcache. This can be overriden by setting the
|
78
|
+
* LMC_NAMESPACES_ROOT_PATH variable in the environment.
|
79
|
+
*
|
80
|
+
*/
|
81
|
+
|
18
82
|
typedef struct {
|
19
83
|
char *namespace;
|
20
84
|
size_t size;
|
@@ -26,22 +90,115 @@ typedef struct {
|
|
26
90
|
lmc_error_t error;
|
27
91
|
} local_memcache_t;
|
28
92
|
|
29
|
-
|
30
|
-
|
93
|
+
/*
|
94
|
+
* Creates a new handle for accessing a shared memory region.
|
95
|
+
*
|
96
|
+
* lmc_error_t e;
|
97
|
+
* // open via namespace
|
98
|
+
* local_memcache_t *lmc = local_memcache_create("viewcounters", 0, 0, &e);
|
99
|
+
* // open via filename
|
100
|
+
* local_memcache_t *lmc = local_memcache_create(0, "./foo.lmc", 0, &e);
|
101
|
+
*
|
102
|
+
* You must supply at least a namespace or filename parameter
|
103
|
+
*
|
104
|
+
* The size_mb defaults to 1024 (1 GB).
|
105
|
+
*
|
106
|
+
* If you use the namespace parameter, the .lmc file for your namespace will
|
107
|
+
* reside in /var/tmp/localmemcache. This can be overriden by setting the
|
108
|
+
* LMC_NAMESPACES_ROOT_PATH variable in the environment.
|
109
|
+
*
|
110
|
+
* When you first call .new for a previously not existing memory pool, a
|
111
|
+
* sparse file will be created and memory and disk space will be allocated to
|
112
|
+
* hold the empty hashtable (about 100K), so the size_mb refers
|
113
|
+
* only to the maximum size of the memory pool. .new for an already existing
|
114
|
+
* memory pool will only map the already previously allocated RAM into the
|
115
|
+
* virtual address space of your process.
|
116
|
+
*/
|
117
|
+
local_memcache_t *local_memcache_create(const char *namespace,
|
118
|
+
const char *filename, double size_mb, lmc_error_t* e);
|
119
|
+
|
120
|
+
/*
|
121
|
+
* Retrieve string value from hashtable.
|
122
|
+
*
|
123
|
+
* It will return a newly allocated string which you need to free() after use.
|
124
|
+
*/
|
31
125
|
char *local_memcache_get_new(local_memcache_t *lmc, const char *key,
|
32
126
|
size_t n_key, size_t *n_value);
|
127
|
+
|
128
|
+
/*
|
129
|
+
* Set string value in hashtable.
|
130
|
+
*/
|
33
131
|
int local_memcache_set(local_memcache_t *lmc, const char *key, size_t n_key,
|
34
132
|
const char* value, size_t n_value);
|
133
|
+
|
134
|
+
/*
|
135
|
+
* Deletes key from hashtable.
|
136
|
+
*/
|
35
137
|
int local_memcache_delete(local_memcache_t *lmc, char *key, size_t n_key);
|
138
|
+
|
139
|
+
/*
|
140
|
+
* Releases memory pool handle.
|
141
|
+
*/
|
36
142
|
int local_memcache_free(local_memcache_t *lmc, lmc_error_t *e);
|
143
|
+
|
144
|
+
/*
|
145
|
+
* Iterate over key value pairs in memory pool
|
146
|
+
*
|
147
|
+
* example:
|
148
|
+
* typedef struct {
|
149
|
+
* ...
|
150
|
+
* } collector_t;
|
151
|
+
*
|
152
|
+
* int my_collect(void *ctx, const char* key, const char* value) {
|
153
|
+
* collector_t *c = ctx;
|
154
|
+
* ....
|
155
|
+
* }
|
156
|
+
*
|
157
|
+
* local_memcache_t *lmc;
|
158
|
+
* collector_t c;
|
159
|
+
* local_memcache_iterate(lmc, (void *) &c, my_collect);
|
160
|
+
*
|
161
|
+
* The memory pool will be locked while iteration takes place, so try to make
|
162
|
+
* sure you can iterate within under 2 seconds otherwise other waiting
|
163
|
+
* processes will try to remove the lock (2 seconds is the timeout for
|
164
|
+
* triggering the automatic recovery.)
|
165
|
+
*
|
166
|
+
*/
|
37
167
|
int local_memcache_iterate(local_memcache_t *lmc, void *ctx,
|
38
168
|
LMC_ITERATOR_P(iter));
|
39
|
-
|
169
|
+
|
170
|
+
/*
|
171
|
+
* Deletes a memory pool. If repair is 1, locked semaphores are
|
172
|
+
* removed as well.
|
173
|
+
*
|
174
|
+
* WARNING: Do only call this method with the repair option if you are sure
|
175
|
+
* that you really want to remove this memory pool and no more processes are
|
176
|
+
* still using it.
|
177
|
+
*
|
178
|
+
* If you delete a pool and other processes still have handles open on it, the
|
179
|
+
* status of these handles becomes undefined. There's no way for a process to
|
180
|
+
* know when a handle is not valid anymore, so only delete a memory pool if
|
181
|
+
* you are sure that all handles are closed.
|
182
|
+
*
|
183
|
+
* The memory pool must be specified by either setting the filename or
|
184
|
+
* namespace parameter.
|
185
|
+
*/
|
186
|
+
int local_memcache_clear_namespace(const char *namespace, const char *filename,
|
187
|
+
int repair, lmc_error_t *e);
|
188
|
+
/*
|
189
|
+
* Tries to repair a corrupt namespace. Usually one doesn't call this method
|
190
|
+
* directly, it's invoked automatically when operations time out.
|
191
|
+
*
|
192
|
+
* The memory pool must be specified by either setting the filename or
|
193
|
+
* namespace parameter.
|
194
|
+
*/
|
195
|
+
int local_memcache_check_namespace(const char *namespace, const char *filename,
|
40
196
|
lmc_error_t *e);
|
41
|
-
int local_memcache_check_namespace(const char *namespace, lmc_error_t *e);
|
42
197
|
|
198
|
+
/* internal, do not use */
|
43
199
|
const char *__local_memcache_get(local_memcache_t *lmc,
|
44
200
|
const char *key, size_t n_key, size_t *n_value);
|
45
201
|
|
202
|
+
/* internal, do not use */
|
46
203
|
int lmc_unlock_shm_region(const char *who, local_memcache_t *lmc);
|
47
204
|
#endif
|
@@ -1,12 +1,5 @@
|
|
1
1
|
require 'rblocalmemcache'
|
2
2
|
|
3
|
-
# == Overview
|
4
|
-
# TestRDocUsage: A useless file
|
5
|
-
#
|
6
|
-
# == Example
|
7
|
-
#
|
8
|
-
# Usage: ruby testRDocUsage.rb [options]
|
9
|
-
#
|
10
3
|
class LocalMemCache
|
11
4
|
|
12
5
|
class LocalMemCacheError < StandardError; end
|
@@ -15,29 +8,68 @@ class LocalMemCache
|
|
15
8
|
class LockError < LocalMemCacheError; end
|
16
9
|
class LockTimedOut < LocalMemCacheError; end
|
17
10
|
class OutOfMemoryError < LocalMemCacheError; end
|
11
|
+
class ArgError < LocalMemCacheError; end
|
12
|
+
class InitError < LocalMemCacheError; end
|
18
13
|
class RecoveryFailed < LocalMemCacheError; end
|
19
14
|
class ShmLockFailed < LocalMemCacheError; end
|
20
15
|
class ShmUnlockFailed < LocalMemCacheError; end
|
21
16
|
class MemoryPoolClosed < LocalMemCacheError; end
|
22
17
|
|
23
18
|
# Creates a new handle for accessing a shared memory region.
|
24
|
-
#
|
19
|
+
#
|
25
20
|
# LocalMemCache.new :namespace=>"foo", :size_mb=> 1
|
26
|
-
#
|
27
|
-
#
|
21
|
+
#
|
22
|
+
# LocalMemCache.new :filename=>"./foo.lmc"
|
23
|
+
#
|
24
|
+
# You must supply at least a :namespace or :filename parameter
|
28
25
|
# The size_mb defaults to 1024 (1 GB).
|
29
26
|
#
|
27
|
+
# If you use the :namespace parameter, the .lmc file for your namespace will
|
28
|
+
# reside in /var/tmp/localmemcache. This can be overriden by setting the
|
29
|
+
# LMC_NAMESPACES_ROOT_PATH variable in the environment.
|
30
|
+
#
|
31
|
+
# When you first call .new for a previously not existing memory pool, a
|
32
|
+
# sparse file will be created and memory and disk space will be allocated to
|
33
|
+
# hold the empty hashtable (about 100K), so the size_mb refers
|
34
|
+
# only to the maximum size of the memory pool. .new for an already existing
|
35
|
+
# memory pool will only map the already previously allocated RAM into the
|
36
|
+
# virtual address space of your process.
|
37
|
+
#
|
38
|
+
#
|
30
39
|
def self.new(options)
|
31
40
|
o = { :size_mb => 0 }.update(options || {})
|
32
|
-
|
33
|
-
_new(o[:namespace].to_s, o[:size_mb].to_f);
|
41
|
+
_new(o);
|
34
42
|
end
|
35
43
|
|
36
|
-
#
|
37
|
-
#
|
38
|
-
#
|
44
|
+
# NOTE: This method is deprecated, use LocalMemCache.clear(*args) instead.
|
45
|
+
#
|
46
|
+
# Deletes a memory pool. If the repair flag is set, locked semaphores are
|
47
|
+
# removed as well.
|
39
48
|
#
|
49
|
+
# If you delete a pool and other processes still have handles open on it, the
|
50
|
+
# status of these handles becomes undefined. There's no way for a process to
|
51
|
+
# know when a handle is not valid anymore, so only delete a memory pool if
|
52
|
+
# you are sure that all handles are closed.
|
53
|
+
#
|
54
|
+
# WARNING: Do only call this method with the repair=true flag if you are sure
|
55
|
+
# that you really want to remove this memory pool and no more processes are
|
56
|
+
# still using it.
|
40
57
|
def self.clear_namespace(namespace, repair = false)
|
41
|
-
|
58
|
+
clear :namespace => namespace.to_s, :repair => repair
|
59
|
+
end
|
60
|
+
|
61
|
+
# NOTE: This method is deprecated, use LocalMemCache.check(*args) instead.
|
62
|
+
#
|
63
|
+
# Tries to repair a corrupt namespace. Usually one doesn't call this method
|
64
|
+
# directly, it's invoked automatically when operations time out.
|
65
|
+
#
|
66
|
+
# valid options are
|
67
|
+
# [:namespace]
|
68
|
+
# [:filename]
|
69
|
+
#
|
70
|
+
# The memory pool must be specified by either setting the :filename or
|
71
|
+
# :namespace option. The default for :repair is false.
|
72
|
+
def self.check_namespace(namespace)
|
73
|
+
check :namespace => namespace.to_s
|
42
74
|
end
|
43
75
|
end
|
@@ -15,7 +15,6 @@
|
|
15
15
|
#define RSTRING_PTR(x) RSTRING(x)->ptr
|
16
16
|
#endif
|
17
17
|
|
18
|
-
|
19
18
|
#if RUBY_VERSION_CODE >= 190
|
20
19
|
#define ruby_errinfo rb_errinfo()
|
21
20
|
#endif
|
@@ -32,6 +31,11 @@ char *rstring_ptr(VALUE s) {
|
|
32
31
|
return r ? r : "nil";
|
33
32
|
}
|
34
33
|
|
34
|
+
char *rstring_ptr_null(VALUE s) {
|
35
|
+
char* r = NIL_P(s) ? NULL : RSTRING_PTR(rb_String(s));
|
36
|
+
return r ? r : NULL;
|
37
|
+
}
|
38
|
+
|
35
39
|
/* :nodoc: */
|
36
40
|
size_t rstring_length(VALUE s) {
|
37
41
|
size_t r = NIL_P(s) ? 0 : RSTRING_LEN(rb_String(s));
|
@@ -58,6 +62,10 @@ typedef struct {
|
|
58
62
|
} rb_lmc_handle_t;
|
59
63
|
|
60
64
|
static VALUE LocalMemCache;
|
65
|
+
static VALUE lmc_rb_sym_namespace;
|
66
|
+
static VALUE lmc_rb_sym_filename;
|
67
|
+
static VALUE lmc_rb_sym_size_mb;
|
68
|
+
static VALUE lmc_rb_sym_repair;
|
61
69
|
|
62
70
|
/* :nodoc: */
|
63
71
|
void __rb_lmc_raise_exception(const char *error_type, const char *m) {
|
@@ -86,12 +94,20 @@ static void rb_lmc_free_handle(rb_lmc_handle_t *h) {
|
|
86
94
|
local_memcache_free(rb_lmc_check_handle_access(h), &e);
|
87
95
|
}
|
88
96
|
|
97
|
+
void lmc_check_dict(VALUE o) {
|
98
|
+
if (TYPE(o) != T_HASH) {
|
99
|
+
rb_raise(rb_eArgError, "expected a Hash");
|
100
|
+
}
|
101
|
+
}
|
89
102
|
|
90
103
|
/* :nodoc: */
|
91
|
-
static VALUE LocalMemCache__new2(VALUE klass, VALUE
|
104
|
+
static VALUE LocalMemCache__new2(VALUE klass, VALUE o) {
|
105
|
+
lmc_check_dict(o);
|
92
106
|
lmc_error_t e;
|
93
|
-
local_memcache_t *l = local_memcache_create(
|
94
|
-
|
107
|
+
local_memcache_t *l = local_memcache_create(
|
108
|
+
rstring_ptr_null(rb_hash_aref(o, lmc_rb_sym_namespace)),
|
109
|
+
rstring_ptr_null(rb_hash_aref(o, lmc_rb_sym_filename)),
|
110
|
+
double_value(rb_hash_aref(o, lmc_rb_sym_size_mb)), &e);
|
95
111
|
if (!l) rb_lmc_raise_exception(&e);
|
96
112
|
rb_lmc_handle_t *h = calloc(1, sizeof(rb_lmc_handle_t));
|
97
113
|
if (!h) rb_raise(rb_eRuntimeError, "memory allocation error");
|
@@ -107,21 +123,61 @@ local_memcache_t *get_LocalMemCache(VALUE obj) {
|
|
107
123
|
return rb_lmc_check_handle_access(h);
|
108
124
|
}
|
109
125
|
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
126
|
+
/*
|
127
|
+
* call-seq: LocalMemCache.clear(*args)
|
128
|
+
*
|
129
|
+
* Deletes a memory pool. If the :repair option is set, locked semaphores are
|
130
|
+
* removed as well.
|
131
|
+
*
|
132
|
+
* WARNING: Do only call this method with the :repair option if you are sure
|
133
|
+
* that you really want to remove this memory pool and no more processes are
|
134
|
+
* still using it.
|
135
|
+
*
|
136
|
+
* If you delete a pool and other processes still have handles open on it, the
|
137
|
+
* status of these handles becomes undefined. There's no way for a process to
|
138
|
+
* know when a handle is not valid anymore, so only delete a memory pool if
|
139
|
+
* you are sure that all handles are closed.
|
140
|
+
*
|
141
|
+
* valid options for clear are
|
142
|
+
* [:namespace]
|
143
|
+
* [:filename]
|
144
|
+
* [:repair]
|
145
|
+
*
|
146
|
+
* The memory pool must be specified by either setting the :filename or
|
147
|
+
* :namespace option. The default for :repair is false.
|
148
|
+
*/
|
149
|
+
static VALUE LocalMemCache__clear(VALUE klass, VALUE o) {
|
150
|
+
lmc_check_dict(o);
|
114
151
|
lmc_error_t e;
|
115
|
-
if (!local_memcache_clear_namespace(
|
152
|
+
if (!local_memcache_clear_namespace(
|
153
|
+
rstring_ptr_null(rb_hash_aref(o, lmc_rb_sym_namespace)),
|
154
|
+
rstring_ptr_null(rb_hash_aref(o, lmc_rb_sym_filename)),
|
155
|
+
bool_value(rb_hash_aref(o, lmc_rb_sym_repair)), &e)) {
|
116
156
|
rb_lmc_raise_exception(&e);
|
117
157
|
}
|
118
158
|
return Qnil;
|
119
159
|
}
|
120
160
|
|
121
|
-
/*
|
122
|
-
|
161
|
+
/*
|
162
|
+
* call-seq: LocalMemCache.check(*args)
|
163
|
+
*
|
164
|
+
* Tries to repair a corrupt namespace. Usually one doesn't call this method
|
165
|
+
* directly, it's invoked automatically when operations time out.
|
166
|
+
*
|
167
|
+
* valid options are
|
168
|
+
* [:namespace]
|
169
|
+
* [:filename]
|
170
|
+
*
|
171
|
+
* The memory pool must be specified by either setting the :filename or
|
172
|
+
* :namespace option.
|
173
|
+
*/
|
174
|
+
static VALUE LocalMemCache__check(VALUE klass, VALUE o) {
|
175
|
+
lmc_check_dict(o);
|
123
176
|
lmc_error_t e;
|
124
|
-
if (!local_memcache_check_namespace(
|
177
|
+
if (!local_memcache_check_namespace(
|
178
|
+
rstring_ptr_null(rb_hash_aref(o, lmc_rb_sym_namespace)),
|
179
|
+
rstring_ptr_null(rb_hash_aref(o, lmc_rb_sym_filename)),
|
180
|
+
&e)) {
|
125
181
|
rb_lmc_raise_exception(&e);
|
126
182
|
}
|
127
183
|
return Qnil;
|
@@ -140,13 +196,12 @@ static VALUE LocalMemCache__disable_test_crash(VALUE klass) {
|
|
140
196
|
return Qnil;
|
141
197
|
}
|
142
198
|
|
143
|
-
|
144
199
|
/*
|
145
200
|
* call-seq:
|
146
|
-
* lmc.get(key) ->
|
147
|
-
* lmc[key] ->
|
201
|
+
* lmc.get(key) -> string value or nil
|
202
|
+
* lmc[key] -> string value or nil
|
148
203
|
*
|
149
|
-
* Retrieve value from hashtable.
|
204
|
+
* Retrieve string value from hashtable.
|
150
205
|
*/
|
151
206
|
static VALUE LocalMemCache__get(VALUE obj, VALUE key) {
|
152
207
|
size_t l;
|
@@ -162,9 +217,9 @@ static VALUE LocalMemCache__get(VALUE obj, VALUE key) {
|
|
162
217
|
* lmc.set(key, value) -> Qnil
|
163
218
|
* lmc[key]=value -> Qnil
|
164
219
|
*
|
165
|
-
* Set value for key in hashtable.
|
220
|
+
* Set value for key in hashtable. Value and key will be converted to
|
221
|
+
* string.
|
166
222
|
*/
|
167
|
-
|
168
223
|
static VALUE LocalMemCache__set(VALUE obj, VALUE key, VALUE value) {
|
169
224
|
local_memcache_t *lmc = get_LocalMemCache(obj);
|
170
225
|
if (!local_memcache_set(lmc, rstring_ptr(key), rstring_length(key),
|
@@ -178,7 +233,7 @@ static VALUE LocalMemCache__set(VALUE obj, VALUE key, VALUE value) {
|
|
178
233
|
* call-seq:
|
179
234
|
* lmc.delete(key) -> Qnil
|
180
235
|
*
|
181
|
-
* Deletes key from hashtable.
|
236
|
+
* Deletes key from hashtable. The key is converted to string.
|
182
237
|
*/
|
183
238
|
static VALUE LocalMemCache__delete(VALUE obj, VALUE key) {
|
184
239
|
return local_memcache_delete(get_LocalMemCache(obj),
|
@@ -244,13 +299,60 @@ static VALUE LocalMemCache__keys(VALUE obj) {
|
|
244
299
|
return rb_ary_entry(d, 1);
|
245
300
|
}
|
246
301
|
|
302
|
+
/*
|
303
|
+
* Document-class: LocalMemCache
|
304
|
+
*
|
305
|
+
* <code>LocalMemCache</code> provides for a Hashtable of strings in shared
|
306
|
+
* memory (via a memory mapped file), which thus can be shared between
|
307
|
+
* processes on a computer. Here is an example of its usage:
|
308
|
+
*
|
309
|
+
* $lm = LocalMemCache.new :namespace => "viewcounters"
|
310
|
+
* $lm[:foo] = 1
|
311
|
+
* $lm[:foo] # -> "1"
|
312
|
+
* $lm.delete(:foo)
|
313
|
+
*
|
314
|
+
* <code>LocalMemCache</code> can also be used as a persistent key value
|
315
|
+
* database, just use the :filename instead of the :namespace parameter.
|
316
|
+
*
|
317
|
+
* $lm = LocalMemCache.new :filename => "my-database.lmc"
|
318
|
+
* $lm[:foo] = 1
|
319
|
+
* $lm[:foo] # -> "1"
|
320
|
+
* $lm.delete(:foo)
|
321
|
+
*
|
322
|
+
* == Default sizes of memory pools
|
323
|
+
*
|
324
|
+
* The default size for memory pools is 1024 (MB). It cannot be changed later,
|
325
|
+
* so choose a size that will provide enough space for all your data. You
|
326
|
+
* might consider setting this size to the maximum filesize of your
|
327
|
+
* filesystem. Also note that while these memory pools may look large on your
|
328
|
+
* disk, they really aren't, because with sparse files only those parts of the
|
329
|
+
* file which contain non-null data actually use disk space.
|
330
|
+
*
|
331
|
+
* == Automatic recovery from crashes
|
332
|
+
*
|
333
|
+
* In case a process is terminated while accessing a memory pool, other
|
334
|
+
* processes will wait for the lock up to 2 seconds, and will then try to
|
335
|
+
* resume the aborted operation. This can also be done explicitly by using
|
336
|
+
* LocalMemCache.check(options).
|
337
|
+
*
|
338
|
+
* == Clearing memory pools
|
339
|
+
*
|
340
|
+
* Removing memory pools can be done with LocalMemCache.clear(options).
|
341
|
+
*
|
342
|
+
* == Environment
|
343
|
+
*
|
344
|
+
* If you use the :namespace parameter, the .lmc file for your namespace will
|
345
|
+
* reside in /var/tmp/localmemcache. This can be overriden by setting the
|
346
|
+
* LMC_NAMESPACES_ROOT_PATH variable in the environment.
|
347
|
+
*
|
348
|
+
*/
|
247
349
|
void Init_rblocalmemcache() {
|
248
350
|
LocalMemCache = rb_define_class("LocalMemCache", rb_cObject);
|
249
|
-
rb_define_singleton_method(LocalMemCache, "_new", LocalMemCache__new2,
|
250
|
-
rb_define_singleton_method(LocalMemCache, "
|
251
|
-
|
252
|
-
rb_define_singleton_method(LocalMemCache, "
|
253
|
-
|
351
|
+
rb_define_singleton_method(LocalMemCache, "_new", LocalMemCache__new2, 1);
|
352
|
+
rb_define_singleton_method(LocalMemCache, "clear",
|
353
|
+
LocalMemCache__clear, 1);
|
354
|
+
rb_define_singleton_method(LocalMemCache, "check",
|
355
|
+
LocalMemCache__check, 1);
|
254
356
|
rb_define_singleton_method(LocalMemCache, "disable_test_crash",
|
255
357
|
LocalMemCache__disable_test_crash, 0);
|
256
358
|
rb_define_singleton_method(LocalMemCache, "enable_test_crash",
|
@@ -262,4 +364,9 @@ void Init_rblocalmemcache() {
|
|
262
364
|
rb_define_method(LocalMemCache, "[]=", LocalMemCache__set, 2);
|
263
365
|
rb_define_method(LocalMemCache, "keys", LocalMemCache__keys, 0);
|
264
366
|
rb_define_method(LocalMemCache, "close", LocalMemCache__close, 0);
|
367
|
+
|
368
|
+
lmc_rb_sym_namespace = ID2SYM(rb_intern("namespace"));
|
369
|
+
lmc_rb_sym_filename = ID2SYM(rb_intern("filename"));
|
370
|
+
lmc_rb_sym_size_mb = ID2SYM(rb_intern("size_mb"));
|
371
|
+
lmc_rb_sym_repair = ID2SYM(rb_intern("repair"));
|
265
372
|
}
|