cuboid 0.0.0 → 0.0.1alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +0 -0
  3. data/Gemfile +20 -5
  4. data/LICENSE.md +22 -0
  5. data/README.md +158 -19
  6. data/Rakefile +56 -3
  7. data/config/paths.yml +15 -0
  8. data/cuboid.gemspec +61 -23
  9. data/lib/cuboid.rb +96 -4
  10. data/lib/cuboid/application.rb +326 -0
  11. data/lib/cuboid/application/parts/data.rb +18 -0
  12. data/lib/cuboid/application/parts/report.rb +29 -0
  13. data/lib/cuboid/application/parts/state.rb +274 -0
  14. data/lib/cuboid/application/runtime.rb +25 -0
  15. data/lib/cuboid/banner.rb +13 -0
  16. data/lib/cuboid/data.rb +86 -0
  17. data/lib/cuboid/data/application.rb +52 -0
  18. data/lib/cuboid/error.rb +9 -0
  19. data/lib/cuboid/option_group.rb +129 -0
  20. data/lib/cuboid/option_groups.rb +8 -0
  21. data/lib/cuboid/option_groups/datastore.rb +23 -0
  22. data/lib/cuboid/option_groups/dispatcher.rb +38 -0
  23. data/lib/cuboid/option_groups/output.rb +14 -0
  24. data/lib/cuboid/option_groups/paths.rb +184 -0
  25. data/lib/cuboid/option_groups/report.rb +39 -0
  26. data/lib/cuboid/option_groups/rpc.rb +105 -0
  27. data/lib/cuboid/option_groups/scheduler.rb +27 -0
  28. data/lib/cuboid/option_groups/snapshot.rb +13 -0
  29. data/lib/cuboid/option_groups/system.rb +10 -0
  30. data/lib/cuboid/options.rb +254 -0
  31. data/lib/cuboid/processes.rb +13 -0
  32. data/lib/cuboid/processes/dispatchers.rb +140 -0
  33. data/lib/cuboid/processes/executables/base.rb +54 -0
  34. data/lib/cuboid/processes/executables/dispatcher.rb +5 -0
  35. data/lib/cuboid/processes/executables/instance.rb +12 -0
  36. data/lib/cuboid/processes/executables/rest_service.rb +13 -0
  37. data/lib/cuboid/processes/executables/scheduler.rb +5 -0
  38. data/lib/cuboid/processes/helpers.rb +4 -0
  39. data/lib/cuboid/processes/helpers/dispatchers.rb +23 -0
  40. data/lib/cuboid/processes/helpers/instances.rb +39 -0
  41. data/lib/cuboid/processes/helpers/processes.rb +23 -0
  42. data/lib/cuboid/processes/helpers/schedulers.rb +23 -0
  43. data/lib/cuboid/processes/instances.rb +203 -0
  44. data/lib/cuboid/processes/manager.rb +262 -0
  45. data/lib/cuboid/processes/schedulers.rb +128 -0
  46. data/lib/cuboid/report.rb +220 -0
  47. data/lib/cuboid/rest/server.rb +165 -0
  48. data/lib/cuboid/rest/server/instance_helpers.rb +99 -0
  49. data/lib/cuboid/rest/server/routes/dispatcher.rb +41 -0
  50. data/lib/cuboid/rest/server/routes/grid.rb +41 -0
  51. data/lib/cuboid/rest/server/routes/instances.rb +131 -0
  52. data/lib/cuboid/rest/server/routes/scheduler.rb +140 -0
  53. data/lib/cuboid/rpc/client.rb +3 -0
  54. data/lib/cuboid/rpc/client/base.rb +58 -0
  55. data/lib/cuboid/rpc/client/dispatcher.rb +58 -0
  56. data/lib/cuboid/rpc/client/instance.rb +100 -0
  57. data/lib/cuboid/rpc/client/instance/service.rb +37 -0
  58. data/lib/cuboid/rpc/client/scheduler.rb +46 -0
  59. data/lib/cuboid/rpc/serializer.rb +92 -0
  60. data/lib/cuboid/rpc/server/active_options.rb +38 -0
  61. data/lib/cuboid/rpc/server/application_wrapper.rb +138 -0
  62. data/lib/cuboid/rpc/server/base.rb +63 -0
  63. data/lib/cuboid/rpc/server/dispatcher.rb +317 -0
  64. data/lib/cuboid/rpc/server/dispatcher/node.rb +247 -0
  65. data/lib/cuboid/rpc/server/dispatcher/service.rb +145 -0
  66. data/lib/cuboid/rpc/server/instance.rb +338 -0
  67. data/lib/cuboid/rpc/server/output.rb +92 -0
  68. data/lib/cuboid/rpc/server/scheduler.rb +482 -0
  69. data/lib/cuboid/ruby.rb +4 -0
  70. data/lib/cuboid/ruby/array.rb +17 -0
  71. data/lib/cuboid/ruby/hash.rb +41 -0
  72. data/lib/cuboid/ruby/object.rb +32 -0
  73. data/lib/cuboid/snapshot.rb +186 -0
  74. data/lib/cuboid/state.rb +94 -0
  75. data/lib/cuboid/state/application.rb +309 -0
  76. data/lib/cuboid/state/options.rb +27 -0
  77. data/lib/cuboid/support.rb +11 -0
  78. data/lib/cuboid/support/buffer.rb +3 -0
  79. data/lib/cuboid/support/buffer/autoflush.rb +61 -0
  80. data/lib/cuboid/support/buffer/base.rb +91 -0
  81. data/lib/cuboid/support/cache.rb +7 -0
  82. data/lib/cuboid/support/cache/base.rb +226 -0
  83. data/lib/cuboid/support/cache/least_cost_replacement.rb +77 -0
  84. data/lib/cuboid/support/cache/least_recently_pushed.rb +21 -0
  85. data/lib/cuboid/support/cache/least_recently_used.rb +31 -0
  86. data/lib/cuboid/support/cache/preference.rb +31 -0
  87. data/lib/cuboid/support/cache/random_replacement.rb +20 -0
  88. data/lib/cuboid/support/crypto.rb +2 -0
  89. data/lib/cuboid/support/crypto/rsa_aes_cbc.rb +86 -0
  90. data/lib/cuboid/support/database.rb +5 -0
  91. data/lib/cuboid/support/database/base.rb +177 -0
  92. data/lib/cuboid/support/database/categorized_queue.rb +195 -0
  93. data/lib/cuboid/support/database/hash.rb +300 -0
  94. data/lib/cuboid/support/database/queue.rb +149 -0
  95. data/lib/cuboid/support/filter.rb +3 -0
  96. data/lib/cuboid/support/filter/base.rb +110 -0
  97. data/lib/cuboid/support/filter/set.rb +29 -0
  98. data/lib/cuboid/support/glob.rb +27 -0
  99. data/lib/cuboid/support/mixins.rb +8 -0
  100. data/lib/cuboid/support/mixins/observable.rb +99 -0
  101. data/lib/cuboid/support/mixins/parts.rb +20 -0
  102. data/lib/cuboid/support/mixins/profiler.rb +93 -0
  103. data/lib/cuboid/support/mixins/spec_instances.rb +65 -0
  104. data/lib/cuboid/support/mixins/terminal.rb +57 -0
  105. data/lib/cuboid/system.rb +119 -0
  106. data/lib/cuboid/system/platforms.rb +84 -0
  107. data/lib/cuboid/system/platforms/linux.rb +26 -0
  108. data/lib/cuboid/system/platforms/mixins/unix.rb +46 -0
  109. data/lib/cuboid/system/platforms/osx.rb +25 -0
  110. data/lib/cuboid/system/platforms/windows.rb +81 -0
  111. data/lib/cuboid/system/slots.rb +143 -0
  112. data/lib/cuboid/ui/output.rb +52 -0
  113. data/lib/cuboid/ui/output_interface.rb +43 -0
  114. data/lib/cuboid/ui/output_interface/abstract.rb +68 -0
  115. data/lib/cuboid/ui/output_interface/controls.rb +84 -0
  116. data/lib/cuboid/ui/output_interface/error_logging.rb +119 -0
  117. data/lib/cuboid/ui/output_interface/implemented.rb +58 -0
  118. data/lib/cuboid/ui/output_interface/personalization.rb +62 -0
  119. data/lib/cuboid/utilities.rb +155 -0
  120. data/lib/cuboid/version.rb +4 -3
  121. data/lib/version +1 -0
  122. data/logs/placeholder +0 -0
  123. data/spec/cuboid/application/parts/data_spec.rb +12 -0
  124. data/spec/cuboid/application/parts/report_spec.rb +6 -0
  125. data/spec/cuboid/application/parts/state_spec.rb +192 -0
  126. data/spec/cuboid/application/runtime_spec.rb +21 -0
  127. data/spec/cuboid/application_spec.rb +37 -0
  128. data/spec/cuboid/data/application_spec.rb +22 -0
  129. data/spec/cuboid/data_spec.rb +47 -0
  130. data/spec/cuboid/error_spec.rb +23 -0
  131. data/spec/cuboid/option_groups/datastore_spec.rb +54 -0
  132. data/spec/cuboid/option_groups/dispatcher_spec.rb +12 -0
  133. data/spec/cuboid/option_groups/output_spec.rb +11 -0
  134. data/spec/cuboid/option_groups/paths_spec.rb +184 -0
  135. data/spec/cuboid/option_groups/report_spec.rb +26 -0
  136. data/spec/cuboid/option_groups/rpc_spec.rb +53 -0
  137. data/spec/cuboid/option_groups/snapshot_spec.rb +26 -0
  138. data/spec/cuboid/option_groups/system.rb +12 -0
  139. data/spec/cuboid/options_spec.rb +218 -0
  140. data/spec/cuboid/report_spec.rb +221 -0
  141. data/spec/cuboid/rest/server_spec.rb +1205 -0
  142. data/spec/cuboid/rpc/client/base_spec.rb +151 -0
  143. data/spec/cuboid/rpc/client/dispatcher_spec.rb +13 -0
  144. data/spec/cuboid/rpc/client/instance_spec.rb +38 -0
  145. data/spec/cuboid/rpc/server/active_options_spec.rb +21 -0
  146. data/spec/cuboid/rpc/server/base_spec.rb +60 -0
  147. data/spec/cuboid/rpc/server/dispatcher/node_spec.rb +222 -0
  148. data/spec/cuboid/rpc/server/dispatcher/service_spec.rb +112 -0
  149. data/spec/cuboid/rpc/server/dispatcher_spec.rb +317 -0
  150. data/spec/cuboid/rpc/server/instance_spec.rb +307 -0
  151. data/spec/cuboid/rpc/server/output_spec.rb +32 -0
  152. data/spec/cuboid/rpc/server/scheduler_spec.rb +400 -0
  153. data/spec/cuboid/ruby/array_spec.rb +77 -0
  154. data/spec/cuboid/ruby/hash_spec.rb +63 -0
  155. data/spec/cuboid/ruby/object_spec.rb +22 -0
  156. data/spec/cuboid/snapshot_spec.rb +123 -0
  157. data/spec/cuboid/state/application_spec.rb +538 -0
  158. data/spec/cuboid/state/options_spec.rb +37 -0
  159. data/spec/cuboid/state_spec.rb +53 -0
  160. data/spec/cuboid/support/buffer/autoflush_spec.rb +78 -0
  161. data/spec/cuboid/support/buffer/base_spec.rb +193 -0
  162. data/spec/cuboid/support/cache/least_cost_replacement_spec.rb +61 -0
  163. data/spec/cuboid/support/cache/least_recently_pushed_spec.rb +90 -0
  164. data/spec/cuboid/support/cache/least_recently_used_spec.rb +80 -0
  165. data/spec/cuboid/support/cache/preference_spec.rb +37 -0
  166. data/spec/cuboid/support/cache/random_replacement_spec.rb +42 -0
  167. data/spec/cuboid/support/crypto/rsa_aes_cbc_spec.rb +28 -0
  168. data/spec/cuboid/support/database/categorized_queue_spec.rb +327 -0
  169. data/spec/cuboid/support/database/hash_spec.rb +204 -0
  170. data/spec/cuboid/support/database/scheduler_spec.rb +325 -0
  171. data/spec/cuboid/support/filter/set_spec.rb +19 -0
  172. data/spec/cuboid/support/glob_spec.rb +75 -0
  173. data/spec/cuboid/support/mixins/observable_spec.rb +95 -0
  174. data/spec/cuboid/system/platforms/linux_spec.rb +31 -0
  175. data/spec/cuboid/system/platforms/osx_spec.rb +32 -0
  176. data/spec/cuboid/system/platforms/windows_spec.rb +41 -0
  177. data/spec/cuboid/system/slots_spec.rb +202 -0
  178. data/spec/cuboid/system_spec.rb +105 -0
  179. data/spec/cuboid/utilities_spec.rb +131 -0
  180. data/spec/spec_helper.rb +46 -0
  181. data/spec/support/factories/placeholder +0 -0
  182. data/spec/support/factories/scan_report.rb +18 -0
  183. data/spec/support/fixtures/empty/placeholder +0 -0
  184. data/spec/support/fixtures/executables/node.rb +50 -0
  185. data/spec/support/fixtures/mock_app.rb +61 -0
  186. data/spec/support/fixtures/mock_app/test_service.rb +64 -0
  187. data/spec/support/fixtures/services/echo.rb +64 -0
  188. data/spec/support/helpers/framework.rb +3 -0
  189. data/spec/support/helpers/matchers.rb +5 -0
  190. data/spec/support/helpers/misc.rb +3 -0
  191. data/spec/support/helpers/paths.rb +15 -0
  192. data/spec/support/helpers/request_helpers.rb +38 -0
  193. data/spec/support/helpers/requires.rb +8 -0
  194. data/spec/support/helpers/resets.rb +52 -0
  195. data/spec/support/helpers/web_server.rb +15 -0
  196. data/spec/support/lib/factory.rb +107 -0
  197. data/spec/support/lib/web_server_client.rb +41 -0
  198. data/spec/support/lib/web_server_dispatcher.rb +25 -0
  199. data/spec/support/lib/web_server_manager.rb +118 -0
  200. data/spec/support/logs/placeholder +0 -0
  201. data/spec/support/pems/cacert.pem +37 -0
  202. data/spec/support/pems/client/cert.pem +37 -0
  203. data/spec/support/pems/client/foo-cert.pem +39 -0
  204. data/spec/support/pems/client/foo-key.pem +51 -0
  205. data/spec/support/pems/client/key.pem +51 -0
  206. data/spec/support/pems/server/cert.pem +37 -0
  207. data/spec/support/pems/server/key.pem +51 -0
  208. data/spec/support/reports/placeholder +0 -0
  209. data/spec/support/shared/application.rb +10 -0
  210. data/spec/support/shared/component.rb +31 -0
  211. data/spec/support/shared/component/options/base.rb +187 -0
  212. data/spec/support/shared/option_group.rb +98 -0
  213. data/spec/support/shared/support/cache.rb +419 -0
  214. data/spec/support/shared/support/filter.rb +143 -0
  215. data/spec/support/shared/system/platforms/base.rb +25 -0
  216. data/spec/support/shared/system/platforms/mixins/unix.rb +37 -0
  217. data/spec/support/snapshots/placeholder +0 -0
  218. metadata +566 -21
  219. data/.gitignore +0 -8
  220. data/bin/console +0 -15
  221. data/bin/setup +0 -8
@@ -0,0 +1,21 @@
1
+ module Cuboid
2
+ module Support::Cache
3
+
4
+ # Least Recently Pushed cache implementation.
5
+ #
6
+ # Discards the least recently pushed entries, in order to make room for newer ones.
7
+ #
8
+ # This is the cache with best performance across the board.
9
+ #
10
+ # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
11
+ class LeastRecentlyPushed < Base
12
+
13
+ private
14
+
15
+ def prune
16
+ @cache.delete( @cache.first.first )
17
+ end
18
+
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,31 @@
1
+ module Cuboid
2
+ module Support::Cache
3
+
4
+ # Least Recently Used cache implementation.
5
+ #
6
+ # Generally, the most desired mode under most circumstances.
7
+ # Discards the least recently used entries in order to make room for newer ones.
8
+ #
9
+ # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
10
+ class LeastRecentlyUsed < LeastRecentlyPushed
11
+
12
+ private
13
+
14
+ def get_with_internal_key( k )
15
+ if !@cache.include? k
16
+ @misses += 1
17
+ return
18
+ end
19
+
20
+ renew( k )
21
+
22
+ super k
23
+ end
24
+
25
+ def renew( internal_key )
26
+ @cache[internal_key] = @cache.delete( internal_key )
27
+ end
28
+
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,31 @@
1
+ module Cuboid
2
+ module Support::Cache
3
+
4
+ # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
5
+ class Preference < Base
6
+
7
+ def prefer( &block )
8
+ @preference = block
9
+ end
10
+
11
+ private
12
+
13
+ def store_with_internal_key( k, v )
14
+ prune if capped? && (size > max_size - 1)
15
+
16
+ _store( k, v )
17
+ end
18
+
19
+ def find_preference
20
+ @preference.call
21
+ end
22
+
23
+ def prune
24
+ preferred = find_preference
25
+ delete( preferred ) if preferred
26
+ end
27
+
28
+ end
29
+
30
+ end
31
+ end
@@ -0,0 +1,20 @@
1
+ module Cuboid
2
+ module Support::Cache
3
+
4
+ # Random Replacement cache implementation.
5
+ #
6
+ # Discards entries at random in order to make room for new ones.
7
+ #
8
+ # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
9
+ class RandomReplacement < Base
10
+
11
+ private
12
+
13
+ def prune
14
+ @cache.delete( @cache.keys.sample )
15
+ end
16
+
17
+ end
18
+
19
+ end
20
+ end
@@ -0,0 +1,2 @@
1
+ lib = Cuboid::Options.paths.support + 'crypto/'
2
+ require lib + 'rsa_aes_cbc'
@@ -0,0 +1,86 @@
1
+ require 'openssl'
2
+ require "base64"
3
+
4
+ module Cuboid
5
+ module Support::Crypto
6
+
7
+ # Simple hybrid crypto class using RSA for public key encryption and AES with CBC
8
+ # for bulk data encryption/decryption.
9
+ #
10
+ # RSA is used to encrypt the AES primitives which are used to encrypt the plaintext.
11
+ #
12
+ # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
13
+ class RSA_AES_CBC
14
+
15
+ # If only encryption is required the private key parameter can be omitted.
16
+ #
17
+ # @param [String] public_pem
18
+ # Location of the Public key in PEM format.
19
+ # @param [String] private_pem
20
+ # Location of the Private key in PEM format.
21
+ def initialize( public_pem, private_pem = nil )
22
+ @public_pem = public_pem
23
+ @private_pem = private_pem
24
+ end
25
+
26
+ # Encrypts data and returns a Base64 representation of the ciphertext
27
+ # and AES CBC primitives encrypted using the public key.
28
+ #
29
+ # @param [String] data
30
+ #
31
+ # @return [String]
32
+ # Base64 representation of the ciphertext and AES CBC primitives encrypted
33
+ # using the public key.
34
+ def encrypt( data )
35
+ rsa = OpenSSL::PKey::RSA.new( File.read( @public_pem ) )
36
+
37
+ # encrypt with 256 bit AES with CBC
38
+ aes = OpenSSL::Cipher::Cipher.new( 'aes-256-cbc' )
39
+ aes.encrypt
40
+
41
+ # use random key and IV
42
+ aes.key = key = aes.random_key
43
+ aes.iv = iv = aes.random_iv
44
+
45
+ # this will hold all primitives and ciphertext
46
+ primitives = {}
47
+
48
+ primitives['ciphertext'] = aes.update( data )
49
+ primitives['ciphertext'] << aes.final
50
+
51
+ primitives['key'] = rsa.public_encrypt( key )
52
+ primitives['iv'] = rsa.public_encrypt( iv )
53
+
54
+ # serialize everything and base64 encode it
55
+ Base64.encode64( primitives.to_yaml )
56
+ end
57
+
58
+ # Decrypts data.
59
+ #
60
+ # @param [String] data
61
+ #
62
+ # @return [String]
63
+ # Plaintext.
64
+ def decrypt( data )
65
+ rsa = OpenSSL::PKey::RSA.new( File.read( @private_pem ) )
66
+
67
+ # decrypt with 256 bit AES with CBC
68
+ aes = OpenSSL::Cipher::Cipher.new( 'aes-256-cbc' )
69
+ aes.decrypt
70
+
71
+ # unencode and unserialize to get the primitives and ciphertext
72
+ primitives = YAML::load( Base64.decode64( data ) )
73
+
74
+ aes.key = rsa.private_decrypt( primitives['key'] )
75
+ aes.iv = rsa.private_decrypt( primitives['iv'] )
76
+
77
+ plaintext = aes.update( primitives['ciphertext'] )
78
+ plaintext << aes.final
79
+
80
+ plaintext
81
+ end
82
+
83
+ end
84
+
85
+ end
86
+ end
@@ -0,0 +1,5 @@
1
+ lib = Cuboid::Options.paths.support + '/database/'
2
+ require lib + 'base'
3
+ require lib + 'queue'
4
+ require lib + 'categorized_queue'
5
+ require lib + 'hash'
@@ -0,0 +1,177 @@
1
+ module Cuboid
2
+ module Support::Database
3
+
4
+ # Base class for Database data structures
5
+ #
6
+ # Provides helper methods for data structures to be implemented related to
7
+ # objecting dumping, loading, unique filename generation, etc.
8
+ #
9
+ # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
10
+ #
11
+ # @abstract
12
+ class Base
13
+
14
+ DISK_SPACE_FILE = 'Database_disk_space'
15
+
16
+ class <<self
17
+
18
+ def reset
19
+ @@disk_space = 0
20
+ set_disk_space @@disk_space
21
+ end
22
+
23
+ def increment_disk_space( int )
24
+ set_disk_space @@disk_space + int
25
+ end
26
+
27
+ def decrement_disk_space( int )
28
+ set_disk_space @@disk_space - int
29
+ end
30
+
31
+ def disk_space
32
+ @@disk_space
33
+ end
34
+
35
+ def disk_directory
36
+ Options.paths.tmpdir
37
+ end
38
+
39
+ def disk_space_file
40
+ disk_space_file_for Process.pid
41
+ end
42
+
43
+ def disk_space_for( pid )
44
+ return 0 if !Dir.exists?( Options.paths.tmp_dir_for( pid ) )
45
+
46
+ IO.read( disk_space_file_for( pid ) ).to_i
47
+ end
48
+
49
+ def disk_space_file_for( pid )
50
+ "#{Options.paths.tmp_dir_for( pid )}/#{DISK_SPACE_FILE}"
51
+ end
52
+
53
+ private
54
+
55
+ def set_disk_space( int )
56
+ if !File.exist?( disk_directory )
57
+ # Could be caught in #at_exit callbacks, the tmpdir has already
58
+ # been deleted.
59
+ return
60
+ end
61
+
62
+ synchronize do
63
+ @@disk_space = int
64
+ IO.write( disk_space_file, @@disk_space.to_s )
65
+ end
66
+ end
67
+
68
+ def synchronize( &block )
69
+ (@@mutex ||= Mutex.new).synchronize( &block )
70
+ end
71
+
72
+ end
73
+ reset
74
+
75
+ # @param [Hash] options
76
+ # Any object that responds to 'dump' and 'load'.
77
+ def initialize( options = {} )
78
+ @options = options.dup
79
+
80
+ @options[:dumper] ||= Marshal
81
+ @options[:loader] ||= @options[:dumper]
82
+
83
+ @filename_counter = 0
84
+
85
+ at_exit do
86
+ clear
87
+ end
88
+ end
89
+
90
+ def serialize( obj )
91
+ compress(
92
+ @options[:dumper].respond_to?( :dump ) ?
93
+ @options[:dumper].dump( obj ) :
94
+ @options[:dumper].call( obj )
95
+ )
96
+ end
97
+
98
+ def unserialize( data )
99
+ data = decompress( data )
100
+
101
+ @options[:loader].respond_to?( :load ) ?
102
+ @options[:loader].load( data ) :
103
+ @options[:loader].call( data )
104
+ end
105
+
106
+ private
107
+
108
+ # Dumps the object to a unique file and returns its path.
109
+ #
110
+ # The path can be used as a reference to the original value
111
+ # by way of passing it to load().
112
+ #
113
+ # @param [Object] obj
114
+ #
115
+ # @return [String]
116
+ # Filepath
117
+ def dump( obj, &block )
118
+ File.open( get_unique_filename, 'wb' ) do |f|
119
+ serialized = serialize( obj )
120
+ self.class.increment_disk_space f.write( serialized )
121
+
122
+ block.call( serialized ) if block_given?
123
+
124
+ f.path
125
+ end
126
+ end
127
+
128
+ # Loads the object stored in filepath.
129
+ #
130
+ # @param [String] filepath
131
+ #
132
+ # @return [Object]
133
+ def load( filepath )
134
+ unserialize( IO.binread( filepath ) )
135
+ end
136
+
137
+ # Deletes a file.
138
+ #
139
+ # @param [String] filepath
140
+ def delete_file( filepath )
141
+ return if !File.exist?( filepath )
142
+
143
+ self.class.decrement_disk_space File.size( filepath )
144
+ File.delete( filepath )
145
+ end
146
+
147
+ # Loads the object in file and then removes it from the file-system.
148
+ #
149
+ # @param [String] filepath
150
+ #
151
+ # @return [Object]
152
+ def load_and_delete_file( filepath )
153
+ obj = load( filepath )
154
+ delete_file( filepath )
155
+ obj
156
+ end
157
+
158
+ def compress( string )
159
+ Zlib::Deflate.deflate string
160
+ end
161
+
162
+ def decompress( string )
163
+ Zlib::Inflate.inflate string
164
+ end
165
+
166
+ def get_unique_filename
167
+ # Should be unique enough...
168
+ ("#{self.class.disk_directory}/#{self.class.name}_#{object_id}_" <<
169
+ @filename_counter.to_s).gsub( '::', '_' )
170
+ ensure
171
+ @filename_counter += 1
172
+ end
173
+
174
+ end
175
+
176
+ end
177
+ end
@@ -0,0 +1,195 @@
1
+ module Cuboid
2
+ module Support::Database
3
+
4
+ # @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
5
+ class CategorizedQueue < Base
6
+
7
+ # Default {#max_buffer_size}.
8
+ DEFAULT_MAX_BUFFER_SIZE = 100
9
+
10
+ # @return [Integer]
11
+ # How many entries to keep in memory before starting to off-load to disk.
12
+ attr_accessor :max_buffer_size
13
+
14
+ attr_accessor :prefer
15
+
16
+ # @see Cuboid::Database::Base#initialize
17
+ def initialize( options = {}, &block )
18
+ super( options )
19
+
20
+ @prefer = block
21
+ @max_buffer_size = options[:max_buffer_size] || DEFAULT_MAX_BUFFER_SIZE
22
+
23
+ @categories ||= {}
24
+ @waiting = []
25
+ @mutex = Mutex.new
26
+
27
+ @buffer_size = 0
28
+ @disk_size = 0
29
+ end
30
+
31
+ # @note Defaults to {DEFAULT_MAX_BUFFER_SIZE}.
32
+ #
33
+ # @return [Integer]
34
+ # How many entries to keep in memory before starting to off-load to disk.
35
+ def max_buffer_size
36
+ @max_buffer_size
37
+ end
38
+
39
+ def categories
40
+ @categories.keys
41
+ end
42
+
43
+ def data_for( category )
44
+ @categories[category.to_s] ||= {
45
+ disk: [],
46
+ buffer: []
47
+ }
48
+ end
49
+
50
+ def insert_to_disk( category, path )
51
+ data_for( category )[:disk] << path
52
+ @disk_size += 1
53
+ end
54
+
55
+ # @param [Object] obj
56
+ # Object to add to the queue.
57
+ # Must respond to #category.
58
+ def <<( obj )
59
+ fail ArgumentError, 'Missing #prefer block.' if !@prefer
60
+
61
+ if !obj.respond_to?( :category )
62
+ fail ArgumentError, "#{obj.class} does not respond to #category."
63
+ end
64
+
65
+ synchronize do
66
+ data = data_for( obj.category )
67
+
68
+ if data[:buffer].size < max_buffer_size
69
+ @buffer_size += 1
70
+ data[:buffer] << obj
71
+ else
72
+ @disk_size += 1
73
+ data[:disk] << dump( obj )
74
+ end
75
+
76
+ begin
77
+ t = @waiting.shift
78
+ t.wakeup if t
79
+ rescue ThreadError
80
+ retry
81
+ end
82
+ end
83
+ end
84
+ alias :push :<<
85
+ alias :enq :<<
86
+
87
+ # @return [Object]
88
+ # Removes an object from the queue and returns it.
89
+ def pop( non_block = false )
90
+ fail ArgumentError, 'Missing #prefer block.' if !@prefer
91
+
92
+ synchronize do
93
+ loop do
94
+ if internal_empty?
95
+ raise ThreadError, 'queue empty' if non_block
96
+ @waiting.push Thread.current
97
+ @mutex.sleep
98
+ else
99
+ # Get preferred category, hopefully there'll be some data
100
+ # for it.
101
+ category = @prefer.call( @categories.keys )
102
+
103
+ # Get all other available categories just in case the
104
+ # preferred one is empty.
105
+ categories = @categories.keys
106
+ categories.delete category
107
+
108
+ data = nil
109
+ # Check if our category has data and pick another if not.
110
+ loop do
111
+ data = data_for( category )
112
+ if data[:buffer].any? || data[:disk].any?
113
+ break
114
+ end
115
+
116
+ category = categories.pop
117
+ end
118
+
119
+ if data[:buffer].any?
120
+ @buffer_size -= 1
121
+ return data[:buffer].shift
122
+ end
123
+
124
+ @disk_size -= 1
125
+ return load_and_delete_file( data[:disk].shift )
126
+ end
127
+ end
128
+ end
129
+ end
130
+ alias :deq :pop
131
+ alias :shift :pop
132
+
133
+ # @return [Integer]
134
+ # Size of the queue, the number of objects it currently holds.
135
+ def size
136
+ buffer_size + disk_size
137
+ end
138
+ alias :length :size
139
+
140
+ def free_buffer_size
141
+ max_buffer_size - buffer_size
142
+ end
143
+
144
+ def buffer_size
145
+ @buffer_size
146
+ end
147
+
148
+ def disk_size
149
+ @disk_size
150
+ end
151
+
152
+ # @return [Bool]
153
+ # `true` if the queue if empty, `false` otherwise.
154
+ def empty?
155
+ synchronize do
156
+ internal_empty?
157
+ end
158
+ end
159
+
160
+ # Removes all objects from the queue.
161
+ def clear
162
+ synchronize do
163
+ @categories.values.each do |data|
164
+ data[:buffer].clear
165
+
166
+ while !data[:disk].empty?
167
+ path = data[:disk].pop
168
+ next if !path
169
+ delete_file path
170
+ end
171
+ end
172
+
173
+ @buffer_size = 0
174
+ @disk_size = 0
175
+ end
176
+ end
177
+
178
+ def num_waiting
179
+ @waiting.size
180
+ end
181
+
182
+ private
183
+
184
+ def internal_empty?
185
+ @buffer_size == 0 && @disk_size == 0
186
+ end
187
+
188
+ def synchronize( &block )
189
+ @mutex.synchronize( &block )
190
+ end
191
+
192
+ end
193
+
194
+ end
195
+ end