rochefort 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rochefort.rb +96 -11
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c15eb785404e7b278a6f3b33f71fa245fa705c4e
|
|
4
|
+
data.tar.gz: 2e5388be09ceb14991c5ba2d42c6634fa116fbfc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 802ec64a724befb9a4d60a1185d09a32451b4c3617037e6292c5e0272b006d9e1cb14da1a13058fa75227b6ba77863620eae63ada91378c7d0c08c4b5e381efc
|
|
7
|
+
data.tar.gz: 1e9f9be8803852c36c0bcb494ee2b3930fd028508b5bc022b33ef10e472316b2c40ad091b03be0ae4415f47194e3979873f31fd4cad5b3f974db88009996e54b
|
data/lib/rochefort.rb
CHANGED
|
@@ -2,40 +2,125 @@ require 'rest-client'
|
|
|
2
2
|
require 'uri'
|
|
3
3
|
require 'json'
|
|
4
4
|
|
|
5
|
+
# client for https://github.com/jackdoe/rochefort - disk speed append + offset service (poor man's kafka),
|
|
5
6
|
# example usage
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
# r = new Rochefort("http://localhost:8001")
|
|
8
|
+
# offset = r.append("example-namespace", "example-id","example-data")
|
|
9
|
+
# p r.get("example-namespace",offset)
|
|
12
10
|
class Rochefort
|
|
11
|
+
# @param url - the rochefort url (String)
|
|
13
12
|
def initialize(url)
|
|
14
13
|
@urlSet = URI::join(url,'append').to_s
|
|
15
14
|
@urlGet = URI::join(url,'get').to_s
|
|
15
|
+
@urlScan = URI::join(url,'scan').to_s
|
|
16
16
|
@urlGetMulti = URI::join(url,'getMulti').to_s
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
+
# append to rochefort, and returns the stored offset
|
|
20
|
+
# r = Rochefort.new(url)
|
|
21
|
+
# r.append(
|
|
22
|
+
# namespace: "ns", # default nil (nil means the default namespace)
|
|
23
|
+
# data: "some data", # the data you want to append
|
|
24
|
+
# read_timeout: 1, # default 1
|
|
25
|
+
# open_timeout: 1, # default 1
|
|
26
|
+
# )
|
|
27
|
+
# @return the offset at which the data was stored
|
|
19
28
|
def append(opts)
|
|
20
|
-
data = RestClient::Request.execute(method: :post,
|
|
29
|
+
data = RestClient::Request.execute(method: :post,
|
|
30
|
+
url: "#{@urlSet}?namespace=#{opts[:namespace]}&id=#{opts[:id]}",
|
|
31
|
+
payload: opts[:data],
|
|
32
|
+
read_timeout: opts[:read_timeout] || 1,
|
|
33
|
+
open_timeout: opts[:open_timeout] || 1)
|
|
21
34
|
out = JSON.parse(data.body)
|
|
22
35
|
return out["offset"]
|
|
23
36
|
end
|
|
24
37
|
|
|
38
|
+
# get data from rochefort
|
|
39
|
+
# r = Rochefort.new(url)
|
|
40
|
+
# r.get(
|
|
41
|
+
# namespace: "ns", # default nil (nil means the default namespace)
|
|
42
|
+
# offset: 0, # the offset returned from append()
|
|
43
|
+
# read_timeout: 1, # default 1
|
|
44
|
+
# open_timeout: 1, # default 1
|
|
45
|
+
# )
|
|
46
|
+
# @return the stored data (String)
|
|
25
47
|
def get(opts)
|
|
26
|
-
data = RestClient::Request.execute(method: :get,
|
|
48
|
+
data = RestClient::Request.execute(method: :get,
|
|
49
|
+
url: "#{@urlGet}?namespace=#{opts[:namespace]}&offset=#{opts[:offset]}",
|
|
50
|
+
read_timeout: opts[:read_timeout] || 1,
|
|
51
|
+
open_timeout: opts[:open_timeout] || 1).body
|
|
27
52
|
return data
|
|
28
|
-
end
|
|
53
|
+
end
|
|
29
54
|
|
|
55
|
+
# get multiple items from rochefort, (@see #get) (@see #append)
|
|
56
|
+
# r = Rochefort.new(url)
|
|
57
|
+
# r.getMulti(
|
|
58
|
+
# namespace: "ns", # default nil (nil means the default namespace)
|
|
59
|
+
# offsets: [], # array of offsets
|
|
60
|
+
# read_timeout: 1, # default 1
|
|
61
|
+
# open_timeout: 1, # default 1
|
|
62
|
+
# )
|
|
63
|
+
# @return array of stored elements (array of strings)
|
|
30
64
|
def getMulti(opts)
|
|
31
|
-
data = RestClient::Request.execute(method: :get,
|
|
65
|
+
data = RestClient::Request.execute(method: :get,
|
|
66
|
+
url: "#{@urlGetMulti}?namespace=#{opts[:namespace]}",
|
|
67
|
+
payload: opts[:offsets].pack("q<*"),
|
|
68
|
+
read_timeout: opts[:read_timeout] || 1,
|
|
69
|
+
open_timeout: opts[:open_timeout] || 1).body
|
|
32
70
|
offset = 0
|
|
33
71
|
out = []
|
|
34
72
|
while offset != data.length
|
|
35
73
|
len = data.unpack('l<')[0]
|
|
36
|
-
out << data
|
|
74
|
+
out << data[offset + 4, len]
|
|
37
75
|
offset += 4 + len
|
|
38
76
|
end
|
|
39
77
|
return out
|
|
40
78
|
end
|
|
79
|
+
|
|
80
|
+
# scans a namespace, reading from a stream, so the namespace can be very big
|
|
81
|
+
# r = Rochefort.new(url)
|
|
82
|
+
# r.scan(namespace: ns) do |len, offset, value|
|
|
83
|
+
# puts value
|
|
84
|
+
# end
|
|
85
|
+
# @return calls the block for each item
|
|
86
|
+
def scan(opts,&input_block)
|
|
87
|
+
block = proc do |response|
|
|
88
|
+
buffer = ""
|
|
89
|
+
header_len = 12
|
|
90
|
+
need = header_len
|
|
91
|
+
|
|
92
|
+
waiting_for_header = true
|
|
93
|
+
|
|
94
|
+
len = 0
|
|
95
|
+
rochefort_offset = 0
|
|
96
|
+
|
|
97
|
+
response.read_body do |chunk|
|
|
98
|
+
buffer << chunk
|
|
99
|
+
while buffer.length >= need
|
|
100
|
+
|
|
101
|
+
if waiting_for_header
|
|
102
|
+
h = buffer.unpack('l<q<')
|
|
103
|
+
len = h[0]
|
|
104
|
+
rochefort_offset = h[1]
|
|
105
|
+
buffer = buffer[header_len, buffer.length - header_len]
|
|
106
|
+
need = len
|
|
107
|
+
waiting_for_header = false
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
if buffer.length >= need
|
|
111
|
+
input_block.call(len, rochefort_offset, buffer[0, len])
|
|
112
|
+
buffer = buffer[len, buffer.length - len]
|
|
113
|
+
need = header_len
|
|
114
|
+
waiting_for_header = true
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
RestClient::Request.execute(method: :get,
|
|
121
|
+
url: "#{@urlScan}?namespace=#{opts[:namespace]}",
|
|
122
|
+
read_timeout: opts[:read_timeout] || 1,
|
|
123
|
+
open_timeout: opts[:open_timeout] || 1,
|
|
124
|
+
block_response: block)
|
|
125
|
+
end
|
|
41
126
|
end
|