thingfish 0.5.0.pre20160707192835
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.simplecov +7 -0
- data/History.rdoc +5 -0
- data/LICENSE +29 -0
- data/Manifest.txt +39 -0
- data/Procfile +4 -0
- data/README.rdoc +92 -0
- data/Rakefile +92 -0
- data/bin/tfprocessord +6 -0
- data/bin/thingfish +10 -0
- data/etc/thingfish.conf.example +26 -0
- data/lib/strelka/app/metadata.rb +38 -0
- data/lib/strelka/httprequest/metadata.rb +70 -0
- data/lib/thingfish.rb +43 -0
- data/lib/thingfish/behaviors.rb +263 -0
- data/lib/thingfish/datastore.rb +55 -0
- data/lib/thingfish/datastore/memory.rb +93 -0
- data/lib/thingfish/handler.rb +728 -0
- data/lib/thingfish/metastore.rb +55 -0
- data/lib/thingfish/metastore/memory.rb +201 -0
- data/lib/thingfish/mixins.rb +57 -0
- data/lib/thingfish/processor.rb +79 -0
- data/lib/thingfish/processor/mp3.rb +167 -0
- data/lib/thingfish/processordaemon.rb +16 -0
- data/lib/thingfish/spechelpers.rb +165 -0
- data/spec/data/APIC-1-image.mp3 +0 -0
- data/spec/data/APIC-2-images.mp3 +0 -0
- data/spec/data/PIC-1-image.mp3 +0 -0
- data/spec/data/PIC-2-images.mp3 +0 -0
- data/spec/helpers.rb +67 -0
- data/spec/spec.opts +4 -0
- data/spec/thingfish/datastore/memory_spec.rb +19 -0
- data/spec/thingfish/datastore_spec.rb +64 -0
- data/spec/thingfish/handler_spec.rb +838 -0
- data/spec/thingfish/metastore/memory_spec.rb +17 -0
- data/spec/thingfish/metastore_spec.rb +96 -0
- data/spec/thingfish/mixins_spec.rb +63 -0
- data/spec/thingfish/processor/mp3_spec.rb +50 -0
- data/spec/thingfish/processor_spec.rb +65 -0
- data/spec/thingfish_spec.rb +23 -0
- metadata +244 -0
data/lib/thingfish.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'loggability'
|
4
|
+
|
5
|
+
#
|
6
|
+
# Network-accessable datastore service
|
7
|
+
#
|
8
|
+
# == Version
|
9
|
+
#
|
10
|
+
# $Id: thingfish.rb,v ce172208b523 2013/11/20 02:21:12 ged $
|
11
|
+
#
|
12
|
+
# == Authors
|
13
|
+
#
|
14
|
+
# * Michael Granger <ged@FaerieMUD.org>
|
15
|
+
# * Mahlon E. Smith <mahlon@martini.nu>
|
16
|
+
#
|
17
|
+
module Thingfish
|
18
|
+
extend Loggability
|
19
|
+
|
20
|
+
|
21
|
+
# Loggability API -- log all Thingfish-related stuff to a separate logger
|
22
|
+
log_as :thingfish
|
23
|
+
|
24
|
+
|
25
|
+
# Package version
|
26
|
+
VERSION = '0.5.0'
|
27
|
+
|
28
|
+
# Version control revision
|
29
|
+
REVISION = %q$Revision: ce172208b523 $
|
30
|
+
|
31
|
+
|
32
|
+
### Get the library version. If +include_buildnum+ is true, the version string will
|
33
|
+
### include the VCS rev ID.
|
34
|
+
def self::version_string( include_buildnum=false )
|
35
|
+
vstring = "%s %s" % [ self.name, VERSION ]
|
36
|
+
vstring << " (build %s)" % [ REVISION[/: ([[:xdigit:]]+)/, 1] || '0' ] if include_buildnum
|
37
|
+
return vstring
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
end # module Thingfish
|
42
|
+
|
43
|
+
# vim: set nosta noet ts=4 sw=4:
|
@@ -0,0 +1,263 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
#encoding: utf-8
|
3
|
+
|
4
|
+
require 'rspec'
|
5
|
+
|
6
|
+
require 'thingfish/handler'
|
7
|
+
|
8
|
+
|
9
|
+
RSpec.shared_examples "a Thingfish metastore" do
|
10
|
+
|
11
|
+
let( :metastore ) do
|
12
|
+
Thingfish::Metastore.create( described_class )
|
13
|
+
end
|
14
|
+
|
15
|
+
|
16
|
+
it "can save and fetch data" do
|
17
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
18
|
+
expect( metastore.fetch(TEST_UUID) ).to eq( TEST_METADATA.first )
|
19
|
+
expect( metastore.fetch(TEST_UUID) ).to_not be( TEST_METADATA.first )
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
it "returns nil when fetching metadata for an object that doesn't exist" do
|
24
|
+
expect( metastore.fetch(TEST_UUID) ).to be_nil
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
it "doesn't care about the case of the UUID when saving and fetching data" do
|
29
|
+
metastore.save( TEST_UUID.downcase, TEST_METADATA.first.freeze )
|
30
|
+
expect( metastore.fetch(TEST_UUID) ).to eq( TEST_METADATA.first )
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
it "can fetch a single metadata value for a given oid" do
|
35
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
36
|
+
expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( TEST_METADATA.first['format'] )
|
37
|
+
expect( metastore.fetch_value(TEST_UUID, :extent) ).to eq( TEST_METADATA.first['extent'] )
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
it "can fetch a slice of data for a given oid" do
|
42
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
43
|
+
expect( metastore.fetch(TEST_UUID, :format, :extent) ).to eq({
|
44
|
+
'format' => TEST_METADATA.first['format'],
|
45
|
+
'extent' => TEST_METADATA.first['extent'],
|
46
|
+
})
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
it "returns nil when fetching a slice of data for an object that doesn't exist" do
|
51
|
+
expect( metastore.fetch_value(TEST_UUID, :format) ).to be_nil
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
it "doesn't care about the case of the UUID when fetching data" do
|
56
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
57
|
+
expect( metastore.fetch_value(TEST_UUID.downcase, :format) ).to eq( TEST_METADATA.first['format'] )
|
58
|
+
end
|
59
|
+
|
60
|
+
|
61
|
+
it "can update data" do
|
62
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
63
|
+
metastore.merge( TEST_UUID, format: 'image/jpeg' )
|
64
|
+
|
65
|
+
expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( 'image/jpeg' )
|
66
|
+
end
|
67
|
+
|
68
|
+
|
69
|
+
it "doesn't care about the case of the UUID when updating data" do
|
70
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
71
|
+
metastore.merge( TEST_UUID.downcase, format: 'image/jpeg' )
|
72
|
+
|
73
|
+
expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( 'image/jpeg' )
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
it "can remove metadata for a UUID" do
|
78
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
79
|
+
metastore.remove( TEST_UUID )
|
80
|
+
|
81
|
+
expect( metastore.fetch(TEST_UUID) ).to be_nil
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
it "can remove a single key/value pair from the metadata for a UUID" do
|
86
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
87
|
+
metastore.remove( TEST_UUID, :useragent )
|
88
|
+
|
89
|
+
expect( metastore.fetch_value(TEST_UUID, :useragent) ).to be_nil
|
90
|
+
end
|
91
|
+
|
92
|
+
|
93
|
+
it "can truncate metadata not in a list of OIDs for a UUID" do
|
94
|
+
keys = Thingfish::Handler::OPERATIONAL_METADATA_KEYS
|
95
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
96
|
+
metastore.remove_except( TEST_UUID, *keys )
|
97
|
+
|
98
|
+
metadata = metastore.fetch( TEST_UUID )
|
99
|
+
expect( metadata.size ).to eq( keys.size )
|
100
|
+
expect( metadata.keys ).to include( *keys.map(&:to_s) )
|
101
|
+
end
|
102
|
+
|
103
|
+
|
104
|
+
it "knows if it has data for a given OID" do
|
105
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
106
|
+
expect( metastore ).to include( TEST_UUID )
|
107
|
+
end
|
108
|
+
|
109
|
+
|
110
|
+
it "knows how many objects it contains" do
|
111
|
+
expect( metastore.size ).to eq( 0 )
|
112
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
113
|
+
expect( metastore.size ).to eq( 1 )
|
114
|
+
end
|
115
|
+
|
116
|
+
|
117
|
+
it "knows how to fetch UUIDs for related resources" do
|
118
|
+
rel_uuid1 = SecureRandom.uuid
|
119
|
+
rel_uuid2 = SecureRandom.uuid
|
120
|
+
unrel_uuid = SecureRandom.uuid
|
121
|
+
|
122
|
+
metastore.save( TEST_UUID, TEST_METADATA.first )
|
123
|
+
metastore.save( rel_uuid1, TEST_METADATA[1].merge('relation' => TEST_UUID.downcase) )
|
124
|
+
metastore.save( rel_uuid2, TEST_METADATA[2].merge('relation' => TEST_UUID.downcase) )
|
125
|
+
metastore.save( unrel_uuid, TEST_METADATA[3] )
|
126
|
+
|
127
|
+
uuids = metastore.fetch_related_oids( TEST_UUID )
|
128
|
+
|
129
|
+
expect( uuids ).to include( rel_uuid1, rel_uuid2 )
|
130
|
+
expect( uuids ).to_not include( unrel_uuid )
|
131
|
+
end
|
132
|
+
|
133
|
+
|
134
|
+
context "with some uploaded metadata" do
|
135
|
+
|
136
|
+
before( :each ) do
|
137
|
+
@uuids = []
|
138
|
+
TEST_METADATA.each do |file|
|
139
|
+
uuid = SecureRandom.uuid
|
140
|
+
@uuids << uuid
|
141
|
+
metastore.save( uuid, file )
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
|
146
|
+
it "can fetch an array of all of its OIDs" do
|
147
|
+
expect( metastore.oids ).to eq( @uuids )
|
148
|
+
end
|
149
|
+
|
150
|
+
it "can iterate over each of the store's oids" do
|
151
|
+
uuids = []
|
152
|
+
metastore.each_oid {|u| uuids << u }
|
153
|
+
|
154
|
+
expect( uuids ).to eq( @uuids )
|
155
|
+
end
|
156
|
+
|
157
|
+
it "can provide an enumerator over each of the store's oids" do
|
158
|
+
expect( metastore.each_oid.to_a ).to eq( @uuids )
|
159
|
+
end
|
160
|
+
|
161
|
+
it "can search for uuids" do
|
162
|
+
expect( metastore.search.to_a ).to eq( metastore.oids )
|
163
|
+
end
|
164
|
+
|
165
|
+
it "can apply criteria to searches" do
|
166
|
+
results = metastore.search( criteria: {format: 'audio/mp3'} )
|
167
|
+
expect( results.size ).to eq( 2 )
|
168
|
+
results.each do |uuid|
|
169
|
+
expect( metastore.fetch_value(uuid, 'format') ).to eq( 'audio/mp3' )
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
it "can limit the number of results returned from a search" do
|
174
|
+
expect( metastore.search( limit: 2 ).to_a ).to eq( metastore.oids[0,2] )
|
175
|
+
end
|
176
|
+
|
177
|
+
it "can order the results returned from a search" do
|
178
|
+
results = metastore.search( order: %w[title created] ).to_a
|
179
|
+
sorted_uuids = metastore.each_oid.
|
180
|
+
map {|oid| metastore.fetch(oid, :title, :created).merge(oid: oid) }.
|
181
|
+
sort_by {|tuple| tuple.values_at('title', 'created') }.
|
182
|
+
map {|tuple| tuple[:oid] }
|
183
|
+
|
184
|
+
expect( results ).to eq( sorted_uuids )
|
185
|
+
end
|
186
|
+
|
187
|
+
end
|
188
|
+
|
189
|
+
|
190
|
+
end
|
191
|
+
|
192
|
+
|
193
|
+
RSpec.shared_examples "a Thingfish datastore" do
|
194
|
+
|
195
|
+
let( :png_io ) { StringIO.new(TEST_PNG_DATA.dup) }
|
196
|
+
let( :text_io ) { StringIO.new(TEST_TEXT_DATA.dup) }
|
197
|
+
|
198
|
+
|
199
|
+
it "returns a UUID when saving" do
|
200
|
+
expect( store.save(png_io) ).to be_a_uuid()
|
201
|
+
end
|
202
|
+
|
203
|
+
|
204
|
+
it "restores the position of the IO after saving" do
|
205
|
+
png_io.pos = 11
|
206
|
+
store.save( png_io )
|
207
|
+
expect( png_io.pos ).to eq( 11 )
|
208
|
+
end
|
209
|
+
|
210
|
+
|
211
|
+
it "can replace existing data" do
|
212
|
+
new_uuid = store.save( text_io )
|
213
|
+
store.replace( new_uuid, png_io )
|
214
|
+
|
215
|
+
rval = store.fetch( new_uuid )
|
216
|
+
expect( rval ).to respond_to( :read )
|
217
|
+
expect( rval.read ).to eq( TEST_PNG_DATA )
|
218
|
+
end
|
219
|
+
|
220
|
+
|
221
|
+
it "doesn't care about the case of the uuid when replacing" do
|
222
|
+
new_uuid = store.save( text_io )
|
223
|
+
store.replace( new_uuid.upcase, png_io )
|
224
|
+
|
225
|
+
rval = store.fetch( new_uuid )
|
226
|
+
expect( rval ).to respond_to( :read )
|
227
|
+
expect( rval.read ).to eq( TEST_PNG_DATA )
|
228
|
+
end
|
229
|
+
|
230
|
+
|
231
|
+
it "can fetch saved data" do
|
232
|
+
oid = store.save( text_io )
|
233
|
+
rval = store.fetch( oid )
|
234
|
+
|
235
|
+
expect( rval ).to respond_to( :read )
|
236
|
+
expect( rval.external_encoding ).to eq( Encoding::ASCII_8BIT )
|
237
|
+
expect( rval.read ).to eq( TEST_TEXT_DATA )
|
238
|
+
end
|
239
|
+
|
240
|
+
|
241
|
+
it "doesn't care about the case of the uuid when fetching" do
|
242
|
+
oid = store.save( text_io )
|
243
|
+
rval = store.fetch( oid.upcase )
|
244
|
+
|
245
|
+
expect( rval ).to respond_to( :read )
|
246
|
+
expect( rval.read ).to eq( TEST_TEXT_DATA )
|
247
|
+
end
|
248
|
+
|
249
|
+
|
250
|
+
it "can remove data" do
|
251
|
+
oid = store.save( text_io )
|
252
|
+
store.remove( oid )
|
253
|
+
|
254
|
+
expect( store.fetch(oid) ).to be_nil
|
255
|
+
end
|
256
|
+
|
257
|
+
|
258
|
+
it "knows if it has data for a given OID" do
|
259
|
+
oid = store.save( text_io )
|
260
|
+
expect( store ).to include( oid )
|
261
|
+
end
|
262
|
+
|
263
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
#encoding: utf-8
|
3
|
+
|
4
|
+
require 'securerandom'
|
5
|
+
require 'pluggability'
|
6
|
+
require 'stringio'
|
7
|
+
require 'strelka'
|
8
|
+
|
9
|
+
require 'thingfish' unless defined?( Thingfish )
|
10
|
+
require 'thingfish/mixins'
|
11
|
+
|
12
|
+
# The base class for storage mechanisms used by Thingfish to store its data
|
13
|
+
# blobs.
|
14
|
+
class Thingfish::Datastore
|
15
|
+
extend Pluggability,
|
16
|
+
Strelka::AbstractClass
|
17
|
+
include Enumerable,
|
18
|
+
Thingfish::Normalization
|
19
|
+
|
20
|
+
|
21
|
+
# Pluggability API -- set the prefix for implementations of Datastore
|
22
|
+
plugin_prefixes 'thingfish/datastore'
|
23
|
+
|
24
|
+
# AbstractClass API -- register some virtual methods that must be implemented
|
25
|
+
# in subclasses
|
26
|
+
pure_virtual :save,
|
27
|
+
:replace,
|
28
|
+
:fetch,
|
29
|
+
:each,
|
30
|
+
:include?,
|
31
|
+
:each_oid,
|
32
|
+
:remove
|
33
|
+
|
34
|
+
|
35
|
+
# :TODO: Make a utility method that provides normalization for IO handling
|
36
|
+
# (restore .pos, etc.)
|
37
|
+
# def with_io( io ) ... end
|
38
|
+
|
39
|
+
### Return a representation of the object as a String suitable for debugging.
|
40
|
+
def inspect
|
41
|
+
return "#<%p:%#016x>" % [
|
42
|
+
self.class,
|
43
|
+
self.object_id * 2
|
44
|
+
]
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
### Provide transactional consistency to the provided block. Concrete datastores should
|
49
|
+
### override this if they can implement it. By default it's a no-op.
|
50
|
+
def transaction
|
51
|
+
yield
|
52
|
+
end
|
53
|
+
|
54
|
+
end # class Thingfish::Datastore
|
55
|
+
|
@@ -0,0 +1,93 @@
|
|
1
|
+
# -*- ruby -*-
|
2
|
+
#encoding: utf-8
|
3
|
+
|
4
|
+
require 'thingfish' unless defined?( Thingfish )
|
5
|
+
require 'thingfish/datastore' unless defined?( Thingfish::Datastore )
|
6
|
+
|
7
|
+
|
8
|
+
|
9
|
+
# An in-memory datastore for testing and tryout purposes.
|
10
|
+
class Thingfish::Datastore::Memory < Thingfish::Datastore
|
11
|
+
extend Loggability
|
12
|
+
|
13
|
+
# Loggability API -- log to the :thingfish logger
|
14
|
+
log_to :thingfish
|
15
|
+
|
16
|
+
|
17
|
+
### Create a new MemoryDatastore, using the given +storage+ object to store
|
18
|
+
### data in. The +storage+ should quack like a Hash.
|
19
|
+
def initialize( storage={} )
|
20
|
+
@storage = storage
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
### Save the +data+ read from the specified +io+ and return an ID that can be
|
25
|
+
### used to fetch it later.
|
26
|
+
def save( io )
|
27
|
+
oid = make_object_id()
|
28
|
+
offset = io.pos
|
29
|
+
data = io.read.dup
|
30
|
+
|
31
|
+
self.log.debug "Saving %d bytes of data under OID %s" % [ data.bytesize, oid ]
|
32
|
+
@storage[ oid ] = data
|
33
|
+
|
34
|
+
io.pos = offset
|
35
|
+
return oid
|
36
|
+
end
|
37
|
+
|
38
|
+
|
39
|
+
### Replace the existing object associated with +oid+ with the data read from the
|
40
|
+
### given +io+.
|
41
|
+
def replace( oid, io )
|
42
|
+
offset = io.pos
|
43
|
+
data = io.read.dup
|
44
|
+
oid = normalize_oid( oid )
|
45
|
+
|
46
|
+
self.log.debug "Replacing data under OID %s with %d bytes" % [ oid, data.bytesize ]
|
47
|
+
@storage[ oid ] = data
|
48
|
+
|
49
|
+
io.pos = offset
|
50
|
+
return true
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
### Fetch the data corresponding to the given +oid+ as an IOish object.
|
55
|
+
def fetch( oid )
|
56
|
+
oid = normalize_oid( oid )
|
57
|
+
self.log.debug "Fetching data for OID %s" % [ oid ]
|
58
|
+
data = @storage[ oid ] or return nil
|
59
|
+
return StringIO.new( data )
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
### Remove the data associated with +oid+ from the Datastore.
|
64
|
+
def remove( oid )
|
65
|
+
oid = normalize_oid( oid )
|
66
|
+
@storage.delete( oid )
|
67
|
+
end
|
68
|
+
|
69
|
+
|
70
|
+
### Return +true+ if the datastore has data associated with the specified +oid+.
|
71
|
+
def include?( oid )
|
72
|
+
oid = normalize_oid( oid )
|
73
|
+
return @storage.include?( oid )
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
### Iterator -- yield the UUID of each object in the datastore to the block, or
|
78
|
+
### return an Enumerator for each UUID if called without a block.
|
79
|
+
def each_oid( &block )
|
80
|
+
return @storage.each_key( &block )
|
81
|
+
end
|
82
|
+
|
83
|
+
|
84
|
+
### Iterator -- yield a pair:
|
85
|
+
### UUID => datablob
|
86
|
+
### of each object in the datastore to the block, or return an Enumerator
|
87
|
+
### for each UUID if called without a block.
|
88
|
+
def each( &block )
|
89
|
+
return @storage.each( &block )
|
90
|
+
end
|
91
|
+
|
92
|
+
end # class Thingfish::Datastore::Memory
|
93
|
+
|