rochefort 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rochefort.rb +96 -11
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c15eb785404e7b278a6f3b33f71fa245fa705c4e
|
|
4
|
+
data.tar.gz: 2e5388be09ceb14991c5ba2d42c6634fa116fbfc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 802ec64a724befb9a4d60a1185d09a32451b4c3617037e6292c5e0272b006d9e1cb14da1a13058fa75227b6ba77863620eae63ada91378c7d0c08c4b5e381efc
|
|
7
|
+
data.tar.gz: 1e9f9be8803852c36c0bcb494ee2b3930fd028508b5bc022b33ef10e472316b2c40ad091b03be0ae4415f47194e3979873f31fd4cad5b3f974db88009996e54b
|
data/lib/rochefort.rb
CHANGED
|
@@ -2,40 +2,125 @@ require 'rest-client'
|
|
|
2
2
|
require 'uri'
|
|
3
3
|
require 'json'
|
|
4
4
|
|
|
5
|
+
# client for https://github.com/jackdoe/rochefort - disk speed append + offset service (poor man's kafka),
|
|
5
6
|
# example usage
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
7
|
+
# r = new Rochefort("http://localhost:8001")
|
|
8
|
+
# offset = r.append("example-namespace", "example-id","example-data")
|
|
9
|
+
# p r.get("example-namespace",offset)
|
|
12
10
|
class Rochefort
|
|
11
|
+
# @param url - the rochefort url (String)
|
|
13
12
|
def initialize(url)
|
|
14
13
|
@urlSet = URI::join(url,'append').to_s
|
|
15
14
|
@urlGet = URI::join(url,'get').to_s
|
|
15
|
+
@urlScan = URI::join(url,'scan').to_s
|
|
16
16
|
@urlGetMulti = URI::join(url,'getMulti').to_s
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
+
# append to rochefort, and returns the stored offset
|
|
20
|
+
# r = Rochefort.new(url)
|
|
21
|
+
# r.append(
|
|
22
|
+
# namespace: "ns", # default nil (nil means the default namespace)
|
|
23
|
+
# data: "some data", # the data you want to append
|
|
24
|
+
# read_timeout: 1, # default 1
|
|
25
|
+
# open_timeout: 1, # default 1
|
|
26
|
+
# )
|
|
27
|
+
# @return the offset at which the data was stored
|
|
19
28
|
def append(opts)
|
|
20
|
-
data = RestClient::Request.execute(method: :post,
|
|
29
|
+
data = RestClient::Request.execute(method: :post,
|
|
30
|
+
url: "#{@urlSet}?namespace=#{opts[:namespace]}&id=#{opts[:id]}",
|
|
31
|
+
payload: opts[:data],
|
|
32
|
+
read_timeout: opts[:read_timeout] || 1,
|
|
33
|
+
open_timeout: opts[:open_timeout] || 1)
|
|
21
34
|
out = JSON.parse(data.body)
|
|
22
35
|
return out["offset"]
|
|
23
36
|
end
|
|
24
37
|
|
|
38
|
+
# get data from rochefort
|
|
39
|
+
# r = Rochefort.new(url)
|
|
40
|
+
# r.get(
|
|
41
|
+
# namespace: "ns", # default nil (nil means the default namespace)
|
|
42
|
+
# offset: 0, # the offset returned from append()
|
|
43
|
+
# read_timeout: 1, # default 1
|
|
44
|
+
# open_timeout: 1, # default 1
|
|
45
|
+
# )
|
|
46
|
+
# @return the stored data (String)
|
|
25
47
|
def get(opts)
|
|
26
|
-
data = RestClient::Request.execute(method: :get,
|
|
48
|
+
data = RestClient::Request.execute(method: :get,
|
|
49
|
+
url: "#{@urlGet}?namespace=#{opts[:namespace]}&offset=#{opts[:offset]}",
|
|
50
|
+
read_timeout: opts[:read_timeout] || 1,
|
|
51
|
+
open_timeout: opts[:open_timeout] || 1).body
|
|
27
52
|
return data
|
|
28
|
-
end
|
|
53
|
+
end
|
|
29
54
|
|
|
55
|
+
# get multiple items from rochefort, (@see #get) (@see #append)
|
|
56
|
+
# r = Rochefort.new(url)
|
|
57
|
+
# r.getMulti(
|
|
58
|
+
# namespace: "ns", # default nil (nil means the default namespace)
|
|
59
|
+
# offsets: [], # array of offsets
|
|
60
|
+
# read_timeout: 1, # default 1
|
|
61
|
+
# open_timeout: 1, # default 1
|
|
62
|
+
# )
|
|
63
|
+
# @return array of stored elements (array of strings)
|
|
30
64
|
def getMulti(opts)
|
|
31
|
-
data = RestClient::Request.execute(method: :get,
|
|
65
|
+
data = RestClient::Request.execute(method: :get,
|
|
66
|
+
url: "#{@urlGetMulti}?namespace=#{opts[:namespace]}",
|
|
67
|
+
payload: opts[:offsets].pack("q<*"),
|
|
68
|
+
read_timeout: opts[:read_timeout] || 1,
|
|
69
|
+
open_timeout: opts[:open_timeout] || 1).body
|
|
32
70
|
offset = 0
|
|
33
71
|
out = []
|
|
34
72
|
while offset != data.length
|
|
35
73
|
len = data.unpack('l<')[0]
|
|
36
|
-
out << data
|
|
74
|
+
out << data[offset + 4, len]
|
|
37
75
|
offset += 4 + len
|
|
38
76
|
end
|
|
39
77
|
return out
|
|
40
78
|
end
|
|
79
|
+
|
|
80
|
+
# scans a namespace, reading from a stream, so the namespace can be very big
|
|
81
|
+
# r = Rochefort.new(url)
|
|
82
|
+
# r.scan(namespace: ns) do |len, offset, value|
|
|
83
|
+
# puts value
|
|
84
|
+
# end
|
|
85
|
+
# @return calls the block for each item
|
|
86
|
+
def scan(opts,&input_block)
|
|
87
|
+
block = proc do |response|
|
|
88
|
+
buffer = ""
|
|
89
|
+
header_len = 12
|
|
90
|
+
need = header_len
|
|
91
|
+
|
|
92
|
+
waiting_for_header = true
|
|
93
|
+
|
|
94
|
+
len = 0
|
|
95
|
+
rochefort_offset = 0
|
|
96
|
+
|
|
97
|
+
response.read_body do |chunk|
|
|
98
|
+
buffer << chunk
|
|
99
|
+
while buffer.length >= need
|
|
100
|
+
|
|
101
|
+
if waiting_for_header
|
|
102
|
+
h = buffer.unpack('l<q<')
|
|
103
|
+
len = h[0]
|
|
104
|
+
rochefort_offset = h[1]
|
|
105
|
+
buffer = buffer[header_len, buffer.length - header_len]
|
|
106
|
+
need = len
|
|
107
|
+
waiting_for_header = false
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
if buffer.length >= need
|
|
111
|
+
input_block.call(len, rochefort_offset, buffer[0, len])
|
|
112
|
+
buffer = buffer[len, buffer.length - len]
|
|
113
|
+
need = header_len
|
|
114
|
+
waiting_for_header = true
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
RestClient::Request.execute(method: :get,
|
|
121
|
+
url: "#{@urlScan}?namespace=#{opts[:namespace]}",
|
|
122
|
+
read_timeout: opts[:read_timeout] || 1,
|
|
123
|
+
open_timeout: opts[:open_timeout] || 1,
|
|
124
|
+
block_response: block)
|
|
125
|
+
end
|
|
41
126
|
end
|