thingfish 0.5.0.pre20160707192835

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +7 -0
  2. data/.simplecov +7 -0
  3. data/History.rdoc +5 -0
  4. data/LICENSE +29 -0
  5. data/Manifest.txt +39 -0
  6. data/Procfile +4 -0
  7. data/README.rdoc +92 -0
  8. data/Rakefile +92 -0
  9. data/bin/tfprocessord +6 -0
  10. data/bin/thingfish +10 -0
  11. data/etc/thingfish.conf.example +26 -0
  12. data/lib/strelka/app/metadata.rb +38 -0
  13. data/lib/strelka/httprequest/metadata.rb +70 -0
  14. data/lib/thingfish.rb +43 -0
  15. data/lib/thingfish/behaviors.rb +263 -0
  16. data/lib/thingfish/datastore.rb +55 -0
  17. data/lib/thingfish/datastore/memory.rb +93 -0
  18. data/lib/thingfish/handler.rb +728 -0
  19. data/lib/thingfish/metastore.rb +55 -0
  20. data/lib/thingfish/metastore/memory.rb +201 -0
  21. data/lib/thingfish/mixins.rb +57 -0
  22. data/lib/thingfish/processor.rb +79 -0
  23. data/lib/thingfish/processor/mp3.rb +167 -0
  24. data/lib/thingfish/processordaemon.rb +16 -0
  25. data/lib/thingfish/spechelpers.rb +165 -0
  26. data/spec/data/APIC-1-image.mp3 +0 -0
  27. data/spec/data/APIC-2-images.mp3 +0 -0
  28. data/spec/data/PIC-1-image.mp3 +0 -0
  29. data/spec/data/PIC-2-images.mp3 +0 -0
  30. data/spec/helpers.rb +67 -0
  31. data/spec/spec.opts +4 -0
  32. data/spec/thingfish/datastore/memory_spec.rb +19 -0
  33. data/spec/thingfish/datastore_spec.rb +64 -0
  34. data/spec/thingfish/handler_spec.rb +838 -0
  35. data/spec/thingfish/metastore/memory_spec.rb +17 -0
  36. data/spec/thingfish/metastore_spec.rb +96 -0
  37. data/spec/thingfish/mixins_spec.rb +63 -0
  38. data/spec/thingfish/processor/mp3_spec.rb +50 -0
  39. data/spec/thingfish/processor_spec.rb +65 -0
  40. data/spec/thingfish_spec.rb +23 -0
  41. metadata +244 -0
data/lib/thingfish.rb ADDED
@@ -0,0 +1,43 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'loggability'
4
+
5
+ #
6
+ # Network-accessable datastore service
7
+ #
8
+ # == Version
9
+ #
10
+ # $Id: thingfish.rb,v ce172208b523 2013/11/20 02:21:12 ged $
11
+ #
12
+ # == Authors
13
+ #
14
+ # * Michael Granger <ged@FaerieMUD.org>
15
+ # * Mahlon E. Smith <mahlon@martini.nu>
16
+ #
17
+ module Thingfish
18
+ extend Loggability
19
+
20
+
21
+ # Loggability API -- log all Thingfish-related stuff to a separate logger
22
+ log_as :thingfish
23
+
24
+
25
+ # Package version
26
+ VERSION = '0.5.0'
27
+
28
+ # Version control revision
29
+ REVISION = %q$Revision: ce172208b523 $
30
+
31
+
32
+ ### Get the library version. If +include_buildnum+ is true, the version string will
33
+ ### include the VCS rev ID.
34
+ def self::version_string( include_buildnum=false )
35
+ vstring = "%s %s" % [ self.name, VERSION ]
36
+ vstring << " (build %s)" % [ REVISION[/: ([[:xdigit:]]+)/, 1] || '0' ] if include_buildnum
37
+ return vstring
38
+ end
39
+
40
+
41
+ end # module Thingfish
42
+
43
+ # vim: set nosta noet ts=4 sw=4:
@@ -0,0 +1,263 @@
1
+ # -*- ruby -*-
2
+ #encoding: utf-8
3
+
4
+ require 'rspec'
5
+
6
+ require 'thingfish/handler'
7
+
8
+
9
+ RSpec.shared_examples "a Thingfish metastore" do
10
+
11
+ let( :metastore ) do
12
+ Thingfish::Metastore.create( described_class )
13
+ end
14
+
15
+
16
+ it "can save and fetch data" do
17
+ metastore.save( TEST_UUID, TEST_METADATA.first )
18
+ expect( metastore.fetch(TEST_UUID) ).to eq( TEST_METADATA.first )
19
+ expect( metastore.fetch(TEST_UUID) ).to_not be( TEST_METADATA.first )
20
+ end
21
+
22
+
23
+ it "returns nil when fetching metadata for an object that doesn't exist" do
24
+ expect( metastore.fetch(TEST_UUID) ).to be_nil
25
+ end
26
+
27
+
28
+ it "doesn't care about the case of the UUID when saving and fetching data" do
29
+ metastore.save( TEST_UUID.downcase, TEST_METADATA.first.freeze )
30
+ expect( metastore.fetch(TEST_UUID) ).to eq( TEST_METADATA.first )
31
+ end
32
+
33
+
34
+ it "can fetch a single metadata value for a given oid" do
35
+ metastore.save( TEST_UUID, TEST_METADATA.first )
36
+ expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( TEST_METADATA.first['format'] )
37
+ expect( metastore.fetch_value(TEST_UUID, :extent) ).to eq( TEST_METADATA.first['extent'] )
38
+ end
39
+
40
+
41
+ it "can fetch a slice of data for a given oid" do
42
+ metastore.save( TEST_UUID, TEST_METADATA.first )
43
+ expect( metastore.fetch(TEST_UUID, :format, :extent) ).to eq({
44
+ 'format' => TEST_METADATA.first['format'],
45
+ 'extent' => TEST_METADATA.first['extent'],
46
+ })
47
+ end
48
+
49
+
50
+ it "returns nil when fetching a slice of data for an object that doesn't exist" do
51
+ expect( metastore.fetch_value(TEST_UUID, :format) ).to be_nil
52
+ end
53
+
54
+
55
+ it "doesn't care about the case of the UUID when fetching data" do
56
+ metastore.save( TEST_UUID, TEST_METADATA.first )
57
+ expect( metastore.fetch_value(TEST_UUID.downcase, :format) ).to eq( TEST_METADATA.first['format'] )
58
+ end
59
+
60
+
61
+ it "can update data" do
62
+ metastore.save( TEST_UUID, TEST_METADATA.first )
63
+ metastore.merge( TEST_UUID, format: 'image/jpeg' )
64
+
65
+ expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( 'image/jpeg' )
66
+ end
67
+
68
+
69
+ it "doesn't care about the case of the UUID when updating data" do
70
+ metastore.save( TEST_UUID, TEST_METADATA.first )
71
+ metastore.merge( TEST_UUID.downcase, format: 'image/jpeg' )
72
+
73
+ expect( metastore.fetch_value(TEST_UUID, :format) ).to eq( 'image/jpeg' )
74
+ end
75
+
76
+
77
+ it "can remove metadata for a UUID" do
78
+ metastore.save( TEST_UUID, TEST_METADATA.first )
79
+ metastore.remove( TEST_UUID )
80
+
81
+ expect( metastore.fetch(TEST_UUID) ).to be_nil
82
+ end
83
+
84
+
85
+ it "can remove a single key/value pair from the metadata for a UUID" do
86
+ metastore.save( TEST_UUID, TEST_METADATA.first )
87
+ metastore.remove( TEST_UUID, :useragent )
88
+
89
+ expect( metastore.fetch_value(TEST_UUID, :useragent) ).to be_nil
90
+ end
91
+
92
+
93
+ it "can truncate metadata not in a list of OIDs for a UUID" do
94
+ keys = Thingfish::Handler::OPERATIONAL_METADATA_KEYS
95
+ metastore.save( TEST_UUID, TEST_METADATA.first )
96
+ metastore.remove_except( TEST_UUID, *keys )
97
+
98
+ metadata = metastore.fetch( TEST_UUID )
99
+ expect( metadata.size ).to eq( keys.size )
100
+ expect( metadata.keys ).to include( *keys.map(&:to_s) )
101
+ end
102
+
103
+
104
+ it "knows if it has data for a given OID" do
105
+ metastore.save( TEST_UUID, TEST_METADATA.first )
106
+ expect( metastore ).to include( TEST_UUID )
107
+ end
108
+
109
+
110
+ it "knows how many objects it contains" do
111
+ expect( metastore.size ).to eq( 0 )
112
+ metastore.save( TEST_UUID, TEST_METADATA.first )
113
+ expect( metastore.size ).to eq( 1 )
114
+ end
115
+
116
+
117
+ it "knows how to fetch UUIDs for related resources" do
118
+ rel_uuid1 = SecureRandom.uuid
119
+ rel_uuid2 = SecureRandom.uuid
120
+ unrel_uuid = SecureRandom.uuid
121
+
122
+ metastore.save( TEST_UUID, TEST_METADATA.first )
123
+ metastore.save( rel_uuid1, TEST_METADATA[1].merge('relation' => TEST_UUID.downcase) )
124
+ metastore.save( rel_uuid2, TEST_METADATA[2].merge('relation' => TEST_UUID.downcase) )
125
+ metastore.save( unrel_uuid, TEST_METADATA[3] )
126
+
127
+ uuids = metastore.fetch_related_oids( TEST_UUID )
128
+
129
+ expect( uuids ).to include( rel_uuid1, rel_uuid2 )
130
+ expect( uuids ).to_not include( unrel_uuid )
131
+ end
132
+
133
+
134
+ context "with some uploaded metadata" do
135
+
136
+ before( :each ) do
137
+ @uuids = []
138
+ TEST_METADATA.each do |file|
139
+ uuid = SecureRandom.uuid
140
+ @uuids << uuid
141
+ metastore.save( uuid, file )
142
+ end
143
+ end
144
+
145
+
146
+ it "can fetch an array of all of its OIDs" do
147
+ expect( metastore.oids ).to eq( @uuids )
148
+ end
149
+
150
+ it "can iterate over each of the store's oids" do
151
+ uuids = []
152
+ metastore.each_oid {|u| uuids << u }
153
+
154
+ expect( uuids ).to eq( @uuids )
155
+ end
156
+
157
+ it "can provide an enumerator over each of the store's oids" do
158
+ expect( metastore.each_oid.to_a ).to eq( @uuids )
159
+ end
160
+
161
+ it "can search for uuids" do
162
+ expect( metastore.search.to_a ).to eq( metastore.oids )
163
+ end
164
+
165
+ it "can apply criteria to searches" do
166
+ results = metastore.search( criteria: {format: 'audio/mp3'} )
167
+ expect( results.size ).to eq( 2 )
168
+ results.each do |uuid|
169
+ expect( metastore.fetch_value(uuid, 'format') ).to eq( 'audio/mp3' )
170
+ end
171
+ end
172
+
173
+ it "can limit the number of results returned from a search" do
174
+ expect( metastore.search( limit: 2 ).to_a ).to eq( metastore.oids[0,2] )
175
+ end
176
+
177
+ it "can order the results returned from a search" do
178
+ results = metastore.search( order: %w[title created] ).to_a
179
+ sorted_uuids = metastore.each_oid.
180
+ map {|oid| metastore.fetch(oid, :title, :created).merge(oid: oid) }.
181
+ sort_by {|tuple| tuple.values_at('title', 'created') }.
182
+ map {|tuple| tuple[:oid] }
183
+
184
+ expect( results ).to eq( sorted_uuids )
185
+ end
186
+
187
+ end
188
+
189
+
190
+ end
191
+
192
+
193
+ RSpec.shared_examples "a Thingfish datastore" do
194
+
195
+ let( :png_io ) { StringIO.new(TEST_PNG_DATA.dup) }
196
+ let( :text_io ) { StringIO.new(TEST_TEXT_DATA.dup) }
197
+
198
+
199
+ it "returns a UUID when saving" do
200
+ expect( store.save(png_io) ).to be_a_uuid()
201
+ end
202
+
203
+
204
+ it "restores the position of the IO after saving" do
205
+ png_io.pos = 11
206
+ store.save( png_io )
207
+ expect( png_io.pos ).to eq( 11 )
208
+ end
209
+
210
+
211
+ it "can replace existing data" do
212
+ new_uuid = store.save( text_io )
213
+ store.replace( new_uuid, png_io )
214
+
215
+ rval = store.fetch( new_uuid )
216
+ expect( rval ).to respond_to( :read )
217
+ expect( rval.read ).to eq( TEST_PNG_DATA )
218
+ end
219
+
220
+
221
+ it "doesn't care about the case of the uuid when replacing" do
222
+ new_uuid = store.save( text_io )
223
+ store.replace( new_uuid.upcase, png_io )
224
+
225
+ rval = store.fetch( new_uuid )
226
+ expect( rval ).to respond_to( :read )
227
+ expect( rval.read ).to eq( TEST_PNG_DATA )
228
+ end
229
+
230
+
231
+ it "can fetch saved data" do
232
+ oid = store.save( text_io )
233
+ rval = store.fetch( oid )
234
+
235
+ expect( rval ).to respond_to( :read )
236
+ expect( rval.external_encoding ).to eq( Encoding::ASCII_8BIT )
237
+ expect( rval.read ).to eq( TEST_TEXT_DATA )
238
+ end
239
+
240
+
241
+ it "doesn't care about the case of the uuid when fetching" do
242
+ oid = store.save( text_io )
243
+ rval = store.fetch( oid.upcase )
244
+
245
+ expect( rval ).to respond_to( :read )
246
+ expect( rval.read ).to eq( TEST_TEXT_DATA )
247
+ end
248
+
249
+
250
+ it "can remove data" do
251
+ oid = store.save( text_io )
252
+ store.remove( oid )
253
+
254
+ expect( store.fetch(oid) ).to be_nil
255
+ end
256
+
257
+
258
+ it "knows if it has data for a given OID" do
259
+ oid = store.save( text_io )
260
+ expect( store ).to include( oid )
261
+ end
262
+
263
+ end
@@ -0,0 +1,55 @@
1
+ # -*- ruby -*-
2
+ #encoding: utf-8
3
+
4
+ require 'securerandom'
5
+ require 'pluggability'
6
+ require 'stringio'
7
+ require 'strelka'
8
+
9
+ require 'thingfish' unless defined?( Thingfish )
10
+ require 'thingfish/mixins'
11
+
12
+ # The base class for storage mechanisms used by Thingfish to store its data
13
+ # blobs.
14
+ class Thingfish::Datastore
15
+ extend Pluggability,
16
+ Strelka::AbstractClass
17
+ include Enumerable,
18
+ Thingfish::Normalization
19
+
20
+
21
+ # Pluggability API -- set the prefix for implementations of Datastore
22
+ plugin_prefixes 'thingfish/datastore'
23
+
24
+ # AbstractClass API -- register some virtual methods that must be implemented
25
+ # in subclasses
26
+ pure_virtual :save,
27
+ :replace,
28
+ :fetch,
29
+ :each,
30
+ :include?,
31
+ :each_oid,
32
+ :remove
33
+
34
+
35
+ # :TODO: Make a utility method that provides normalization for IO handling
36
+ # (restore .pos, etc.)
37
+ # def with_io( io ) ... end
38
+
39
+ ### Return a representation of the object as a String suitable for debugging.
40
+ def inspect
41
+ return "#<%p:%#016x>" % [
42
+ self.class,
43
+ self.object_id * 2
44
+ ]
45
+ end
46
+
47
+
48
+ ### Provide transactional consistency to the provided block. Concrete datastores should
49
+ ### override this if they can implement it. By default it's a no-op.
50
+ def transaction
51
+ yield
52
+ end
53
+
54
+ end # class Thingfish::Datastore
55
+
@@ -0,0 +1,93 @@
1
+ # -*- ruby -*-
2
+ #encoding: utf-8
3
+
4
+ require 'thingfish' unless defined?( Thingfish )
5
+ require 'thingfish/datastore' unless defined?( Thingfish::Datastore )
6
+
7
+
8
+
9
+ # An in-memory datastore for testing and tryout purposes.
10
+ class Thingfish::Datastore::Memory < Thingfish::Datastore
11
+ extend Loggability
12
+
13
+ # Loggability API -- log to the :thingfish logger
14
+ log_to :thingfish
15
+
16
+
17
+ ### Create a new MemoryDatastore, using the given +storage+ object to store
18
+ ### data in. The +storage+ should quack like a Hash.
19
+ def initialize( storage={} )
20
+ @storage = storage
21
+ end
22
+
23
+
24
+ ### Save the +data+ read from the specified +io+ and return an ID that can be
25
+ ### used to fetch it later.
26
+ def save( io )
27
+ oid = make_object_id()
28
+ offset = io.pos
29
+ data = io.read.dup
30
+
31
+ self.log.debug "Saving %d bytes of data under OID %s" % [ data.bytesize, oid ]
32
+ @storage[ oid ] = data
33
+
34
+ io.pos = offset
35
+ return oid
36
+ end
37
+
38
+
39
+ ### Replace the existing object associated with +oid+ with the data read from the
40
+ ### given +io+.
41
+ def replace( oid, io )
42
+ offset = io.pos
43
+ data = io.read.dup
44
+ oid = normalize_oid( oid )
45
+
46
+ self.log.debug "Replacing data under OID %s with %d bytes" % [ oid, data.bytesize ]
47
+ @storage[ oid ] = data
48
+
49
+ io.pos = offset
50
+ return true
51
+ end
52
+
53
+
54
+ ### Fetch the data corresponding to the given +oid+ as an IOish object.
55
+ def fetch( oid )
56
+ oid = normalize_oid( oid )
57
+ self.log.debug "Fetching data for OID %s" % [ oid ]
58
+ data = @storage[ oid ] or return nil
59
+ return StringIO.new( data )
60
+ end
61
+
62
+
63
+ ### Remove the data associated with +oid+ from the Datastore.
64
+ def remove( oid )
65
+ oid = normalize_oid( oid )
66
+ @storage.delete( oid )
67
+ end
68
+
69
+
70
+ ### Return +true+ if the datastore has data associated with the specified +oid+.
71
+ def include?( oid )
72
+ oid = normalize_oid( oid )
73
+ return @storage.include?( oid )
74
+ end
75
+
76
+
77
+ ### Iterator -- yield the UUID of each object in the datastore to the block, or
78
+ ### return an Enumerator for each UUID if called without a block.
79
+ def each_oid( &block )
80
+ return @storage.each_key( &block )
81
+ end
82
+
83
+
84
+ ### Iterator -- yield a pair:
85
+ ### UUID => datablob
86
+ ### of each object in the datastore to the block, or return an Enumerator
87
+ ### for each UUID if called without a block.
88
+ def each( &block )
89
+ return @storage.each( &block )
90
+ end
91
+
92
+ end # class Thingfish::Datastore::Memory
93
+