cuboid 0.0.0 → 0.0.1alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +0 -0
- data/Gemfile +20 -5
- data/LICENSE.md +22 -0
- data/README.md +158 -19
- data/Rakefile +56 -3
- data/config/paths.yml +15 -0
- data/cuboid.gemspec +61 -23
- data/lib/cuboid.rb +96 -4
- data/lib/cuboid/application.rb +326 -0
- data/lib/cuboid/application/parts/data.rb +18 -0
- data/lib/cuboid/application/parts/report.rb +29 -0
- data/lib/cuboid/application/parts/state.rb +274 -0
- data/lib/cuboid/application/runtime.rb +25 -0
- data/lib/cuboid/banner.rb +13 -0
- data/lib/cuboid/data.rb +86 -0
- data/lib/cuboid/data/application.rb +52 -0
- data/lib/cuboid/error.rb +9 -0
- data/lib/cuboid/option_group.rb +129 -0
- data/lib/cuboid/option_groups.rb +8 -0
- data/lib/cuboid/option_groups/datastore.rb +23 -0
- data/lib/cuboid/option_groups/dispatcher.rb +38 -0
- data/lib/cuboid/option_groups/output.rb +14 -0
- data/lib/cuboid/option_groups/paths.rb +184 -0
- data/lib/cuboid/option_groups/report.rb +39 -0
- data/lib/cuboid/option_groups/rpc.rb +105 -0
- data/lib/cuboid/option_groups/scheduler.rb +27 -0
- data/lib/cuboid/option_groups/snapshot.rb +13 -0
- data/lib/cuboid/option_groups/system.rb +10 -0
- data/lib/cuboid/options.rb +254 -0
- data/lib/cuboid/processes.rb +13 -0
- data/lib/cuboid/processes/dispatchers.rb +140 -0
- data/lib/cuboid/processes/executables/base.rb +54 -0
- data/lib/cuboid/processes/executables/dispatcher.rb +5 -0
- data/lib/cuboid/processes/executables/instance.rb +12 -0
- data/lib/cuboid/processes/executables/rest_service.rb +13 -0
- data/lib/cuboid/processes/executables/scheduler.rb +5 -0
- data/lib/cuboid/processes/helpers.rb +4 -0
- data/lib/cuboid/processes/helpers/dispatchers.rb +23 -0
- data/lib/cuboid/processes/helpers/instances.rb +39 -0
- data/lib/cuboid/processes/helpers/processes.rb +23 -0
- data/lib/cuboid/processes/helpers/schedulers.rb +23 -0
- data/lib/cuboid/processes/instances.rb +203 -0
- data/lib/cuboid/processes/manager.rb +262 -0
- data/lib/cuboid/processes/schedulers.rb +128 -0
- data/lib/cuboid/report.rb +220 -0
- data/lib/cuboid/rest/server.rb +165 -0
- data/lib/cuboid/rest/server/instance_helpers.rb +99 -0
- data/lib/cuboid/rest/server/routes/dispatcher.rb +41 -0
- data/lib/cuboid/rest/server/routes/grid.rb +41 -0
- data/lib/cuboid/rest/server/routes/instances.rb +131 -0
- data/lib/cuboid/rest/server/routes/scheduler.rb +140 -0
- data/lib/cuboid/rpc/client.rb +3 -0
- data/lib/cuboid/rpc/client/base.rb +58 -0
- data/lib/cuboid/rpc/client/dispatcher.rb +58 -0
- data/lib/cuboid/rpc/client/instance.rb +100 -0
- data/lib/cuboid/rpc/client/instance/service.rb +37 -0
- data/lib/cuboid/rpc/client/scheduler.rb +46 -0
- data/lib/cuboid/rpc/serializer.rb +92 -0
- data/lib/cuboid/rpc/server/active_options.rb +38 -0
- data/lib/cuboid/rpc/server/application_wrapper.rb +138 -0
- data/lib/cuboid/rpc/server/base.rb +63 -0
- data/lib/cuboid/rpc/server/dispatcher.rb +317 -0
- data/lib/cuboid/rpc/server/dispatcher/node.rb +247 -0
- data/lib/cuboid/rpc/server/dispatcher/service.rb +145 -0
- data/lib/cuboid/rpc/server/instance.rb +338 -0
- data/lib/cuboid/rpc/server/output.rb +92 -0
- data/lib/cuboid/rpc/server/scheduler.rb +482 -0
- data/lib/cuboid/ruby.rb +4 -0
- data/lib/cuboid/ruby/array.rb +17 -0
- data/lib/cuboid/ruby/hash.rb +41 -0
- data/lib/cuboid/ruby/object.rb +32 -0
- data/lib/cuboid/snapshot.rb +186 -0
- data/lib/cuboid/state.rb +94 -0
- data/lib/cuboid/state/application.rb +309 -0
- data/lib/cuboid/state/options.rb +27 -0
- data/lib/cuboid/support.rb +11 -0
- data/lib/cuboid/support/buffer.rb +3 -0
- data/lib/cuboid/support/buffer/autoflush.rb +61 -0
- data/lib/cuboid/support/buffer/base.rb +91 -0
- data/lib/cuboid/support/cache.rb +7 -0
- data/lib/cuboid/support/cache/base.rb +226 -0
- data/lib/cuboid/support/cache/least_cost_replacement.rb +77 -0
- data/lib/cuboid/support/cache/least_recently_pushed.rb +21 -0
- data/lib/cuboid/support/cache/least_recently_used.rb +31 -0
- data/lib/cuboid/support/cache/preference.rb +31 -0
- data/lib/cuboid/support/cache/random_replacement.rb +20 -0
- data/lib/cuboid/support/crypto.rb +2 -0
- data/lib/cuboid/support/crypto/rsa_aes_cbc.rb +86 -0
- data/lib/cuboid/support/database.rb +5 -0
- data/lib/cuboid/support/database/base.rb +177 -0
- data/lib/cuboid/support/database/categorized_queue.rb +195 -0
- data/lib/cuboid/support/database/hash.rb +300 -0
- data/lib/cuboid/support/database/queue.rb +149 -0
- data/lib/cuboid/support/filter.rb +3 -0
- data/lib/cuboid/support/filter/base.rb +110 -0
- data/lib/cuboid/support/filter/set.rb +29 -0
- data/lib/cuboid/support/glob.rb +27 -0
- data/lib/cuboid/support/mixins.rb +8 -0
- data/lib/cuboid/support/mixins/observable.rb +99 -0
- data/lib/cuboid/support/mixins/parts.rb +20 -0
- data/lib/cuboid/support/mixins/profiler.rb +93 -0
- data/lib/cuboid/support/mixins/spec_instances.rb +65 -0
- data/lib/cuboid/support/mixins/terminal.rb +57 -0
- data/lib/cuboid/system.rb +119 -0
- data/lib/cuboid/system/platforms.rb +84 -0
- data/lib/cuboid/system/platforms/linux.rb +26 -0
- data/lib/cuboid/system/platforms/mixins/unix.rb +46 -0
- data/lib/cuboid/system/platforms/osx.rb +25 -0
- data/lib/cuboid/system/platforms/windows.rb +81 -0
- data/lib/cuboid/system/slots.rb +143 -0
- data/lib/cuboid/ui/output.rb +52 -0
- data/lib/cuboid/ui/output_interface.rb +43 -0
- data/lib/cuboid/ui/output_interface/abstract.rb +68 -0
- data/lib/cuboid/ui/output_interface/controls.rb +84 -0
- data/lib/cuboid/ui/output_interface/error_logging.rb +119 -0
- data/lib/cuboid/ui/output_interface/implemented.rb +58 -0
- data/lib/cuboid/ui/output_interface/personalization.rb +62 -0
- data/lib/cuboid/utilities.rb +155 -0
- data/lib/cuboid/version.rb +4 -3
- data/lib/version +1 -0
- data/logs/placeholder +0 -0
- data/spec/cuboid/application/parts/data_spec.rb +12 -0
- data/spec/cuboid/application/parts/report_spec.rb +6 -0
- data/spec/cuboid/application/parts/state_spec.rb +192 -0
- data/spec/cuboid/application/runtime_spec.rb +21 -0
- data/spec/cuboid/application_spec.rb +37 -0
- data/spec/cuboid/data/application_spec.rb +22 -0
- data/spec/cuboid/data_spec.rb +47 -0
- data/spec/cuboid/error_spec.rb +23 -0
- data/spec/cuboid/option_groups/datastore_spec.rb +54 -0
- data/spec/cuboid/option_groups/dispatcher_spec.rb +12 -0
- data/spec/cuboid/option_groups/output_spec.rb +11 -0
- data/spec/cuboid/option_groups/paths_spec.rb +184 -0
- data/spec/cuboid/option_groups/report_spec.rb +26 -0
- data/spec/cuboid/option_groups/rpc_spec.rb +53 -0
- data/spec/cuboid/option_groups/snapshot_spec.rb +26 -0
- data/spec/cuboid/option_groups/system.rb +12 -0
- data/spec/cuboid/options_spec.rb +218 -0
- data/spec/cuboid/report_spec.rb +221 -0
- data/spec/cuboid/rest/server_spec.rb +1205 -0
- data/spec/cuboid/rpc/client/base_spec.rb +151 -0
- data/spec/cuboid/rpc/client/dispatcher_spec.rb +13 -0
- data/spec/cuboid/rpc/client/instance_spec.rb +38 -0
- data/spec/cuboid/rpc/server/active_options_spec.rb +21 -0
- data/spec/cuboid/rpc/server/base_spec.rb +60 -0
- data/spec/cuboid/rpc/server/dispatcher/node_spec.rb +222 -0
- data/spec/cuboid/rpc/server/dispatcher/service_spec.rb +112 -0
- data/spec/cuboid/rpc/server/dispatcher_spec.rb +317 -0
- data/spec/cuboid/rpc/server/instance_spec.rb +307 -0
- data/spec/cuboid/rpc/server/output_spec.rb +32 -0
- data/spec/cuboid/rpc/server/scheduler_spec.rb +400 -0
- data/spec/cuboid/ruby/array_spec.rb +77 -0
- data/spec/cuboid/ruby/hash_spec.rb +63 -0
- data/spec/cuboid/ruby/object_spec.rb +22 -0
- data/spec/cuboid/snapshot_spec.rb +123 -0
- data/spec/cuboid/state/application_spec.rb +538 -0
- data/spec/cuboid/state/options_spec.rb +37 -0
- data/spec/cuboid/state_spec.rb +53 -0
- data/spec/cuboid/support/buffer/autoflush_spec.rb +78 -0
- data/spec/cuboid/support/buffer/base_spec.rb +193 -0
- data/spec/cuboid/support/cache/least_cost_replacement_spec.rb +61 -0
- data/spec/cuboid/support/cache/least_recently_pushed_spec.rb +90 -0
- data/spec/cuboid/support/cache/least_recently_used_spec.rb +80 -0
- data/spec/cuboid/support/cache/preference_spec.rb +37 -0
- data/spec/cuboid/support/cache/random_replacement_spec.rb +42 -0
- data/spec/cuboid/support/crypto/rsa_aes_cbc_spec.rb +28 -0
- data/spec/cuboid/support/database/categorized_queue_spec.rb +327 -0
- data/spec/cuboid/support/database/hash_spec.rb +204 -0
- data/spec/cuboid/support/database/scheduler_spec.rb +325 -0
- data/spec/cuboid/support/filter/set_spec.rb +19 -0
- data/spec/cuboid/support/glob_spec.rb +75 -0
- data/spec/cuboid/support/mixins/observable_spec.rb +95 -0
- data/spec/cuboid/system/platforms/linux_spec.rb +31 -0
- data/spec/cuboid/system/platforms/osx_spec.rb +32 -0
- data/spec/cuboid/system/platforms/windows_spec.rb +41 -0
- data/spec/cuboid/system/slots_spec.rb +202 -0
- data/spec/cuboid/system_spec.rb +105 -0
- data/spec/cuboid/utilities_spec.rb +131 -0
- data/spec/spec_helper.rb +46 -0
- data/spec/support/factories/placeholder +0 -0
- data/spec/support/factories/scan_report.rb +18 -0
- data/spec/support/fixtures/empty/placeholder +0 -0
- data/spec/support/fixtures/executables/node.rb +50 -0
- data/spec/support/fixtures/mock_app.rb +61 -0
- data/spec/support/fixtures/mock_app/test_service.rb +64 -0
- data/spec/support/fixtures/services/echo.rb +64 -0
- data/spec/support/helpers/framework.rb +3 -0
- data/spec/support/helpers/matchers.rb +5 -0
- data/spec/support/helpers/misc.rb +3 -0
- data/spec/support/helpers/paths.rb +15 -0
- data/spec/support/helpers/request_helpers.rb +38 -0
- data/spec/support/helpers/requires.rb +8 -0
- data/spec/support/helpers/resets.rb +52 -0
- data/spec/support/helpers/web_server.rb +15 -0
- data/spec/support/lib/factory.rb +107 -0
- data/spec/support/lib/web_server_client.rb +41 -0
- data/spec/support/lib/web_server_dispatcher.rb +25 -0
- data/spec/support/lib/web_server_manager.rb +118 -0
- data/spec/support/logs/placeholder +0 -0
- data/spec/support/pems/cacert.pem +37 -0
- data/spec/support/pems/client/cert.pem +37 -0
- data/spec/support/pems/client/foo-cert.pem +39 -0
- data/spec/support/pems/client/foo-key.pem +51 -0
- data/spec/support/pems/client/key.pem +51 -0
- data/spec/support/pems/server/cert.pem +37 -0
- data/spec/support/pems/server/key.pem +51 -0
- data/spec/support/reports/placeholder +0 -0
- data/spec/support/shared/application.rb +10 -0
- data/spec/support/shared/component.rb +31 -0
- data/spec/support/shared/component/options/base.rb +187 -0
- data/spec/support/shared/option_group.rb +98 -0
- data/spec/support/shared/support/cache.rb +419 -0
- data/spec/support/shared/support/filter.rb +143 -0
- data/spec/support/shared/system/platforms/base.rb +25 -0
- data/spec/support/shared/system/platforms/mixins/unix.rb +37 -0
- data/spec/support/snapshots/placeholder +0 -0
- metadata +566 -21
- data/.gitignore +0 -8
- data/bin/console +0 -15
- data/bin/setup +0 -8
@@ -0,0 +1,21 @@
|
|
1
|
+
module Cuboid
|
2
|
+
module Support::Cache
|
3
|
+
|
4
|
+
# Least Recently Pushed cache implementation.
|
5
|
+
#
|
6
|
+
# Discards the least recently pushed entries, in order to make room for newer ones.
|
7
|
+
#
|
8
|
+
# This is the cache with best performance across the board.
|
9
|
+
#
|
10
|
+
# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
|
11
|
+
class LeastRecentlyPushed < Base
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def prune
|
16
|
+
@cache.delete( @cache.first.first )
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Cuboid
|
2
|
+
module Support::Cache
|
3
|
+
|
4
|
+
# Least Recently Used cache implementation.
|
5
|
+
#
|
6
|
+
# Generally, the most desired mode under most circumstances.
|
7
|
+
# Discards the least recently used entries in order to make room for newer ones.
|
8
|
+
#
|
9
|
+
# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
|
10
|
+
class LeastRecentlyUsed < LeastRecentlyPushed
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def get_with_internal_key( k )
|
15
|
+
if !@cache.include? k
|
16
|
+
@misses += 1
|
17
|
+
return
|
18
|
+
end
|
19
|
+
|
20
|
+
renew( k )
|
21
|
+
|
22
|
+
super k
|
23
|
+
end
|
24
|
+
|
25
|
+
def renew( internal_key )
|
26
|
+
@cache[internal_key] = @cache.delete( internal_key )
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Cuboid
|
2
|
+
module Support::Cache
|
3
|
+
|
4
|
+
# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
|
5
|
+
class Preference < Base
|
6
|
+
|
7
|
+
def prefer( &block )
|
8
|
+
@preference = block
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def store_with_internal_key( k, v )
|
14
|
+
prune if capped? && (size > max_size - 1)
|
15
|
+
|
16
|
+
_store( k, v )
|
17
|
+
end
|
18
|
+
|
19
|
+
def find_preference
|
20
|
+
@preference.call
|
21
|
+
end
|
22
|
+
|
23
|
+
def prune
|
24
|
+
preferred = find_preference
|
25
|
+
delete( preferred ) if preferred
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Cuboid
|
2
|
+
module Support::Cache
|
3
|
+
|
4
|
+
# Random Replacement cache implementation.
|
5
|
+
#
|
6
|
+
# Discards entries at random in order to make room for new ones.
|
7
|
+
#
|
8
|
+
# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
|
9
|
+
class RandomReplacement < Base
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def prune
|
14
|
+
@cache.delete( @cache.keys.sample )
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'openssl'
|
2
|
+
require "base64"
|
3
|
+
|
4
|
+
module Cuboid
|
5
|
+
module Support::Crypto
|
6
|
+
|
7
|
+
# Simple hybrid crypto class using RSA for public key encryption and AES with CBC
|
8
|
+
# for bulk data encryption/decryption.
|
9
|
+
#
|
10
|
+
# RSA is used to encrypt the AES primitives which are used to encrypt the plaintext.
|
11
|
+
#
|
12
|
+
# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
|
13
|
+
class RSA_AES_CBC
|
14
|
+
|
15
|
+
# If only encryption is required the private key parameter can be omitted.
|
16
|
+
#
|
17
|
+
# @param [String] public_pem
|
18
|
+
# Location of the Public key in PEM format.
|
19
|
+
# @param [String] private_pem
|
20
|
+
# Location of the Private key in PEM format.
|
21
|
+
def initialize( public_pem, private_pem = nil )
|
22
|
+
@public_pem = public_pem
|
23
|
+
@private_pem = private_pem
|
24
|
+
end
|
25
|
+
|
26
|
+
# Encrypts data and returns a Base64 representation of the ciphertext
|
27
|
+
# and AES CBC primitives encrypted using the public key.
|
28
|
+
#
|
29
|
+
# @param [String] data
|
30
|
+
#
|
31
|
+
# @return [String]
|
32
|
+
# Base64 representation of the ciphertext and AES CBC primitives encrypted
|
33
|
+
# using the public key.
|
34
|
+
def encrypt( data )
|
35
|
+
rsa = OpenSSL::PKey::RSA.new( File.read( @public_pem ) )
|
36
|
+
|
37
|
+
# encrypt with 256 bit AES with CBC
|
38
|
+
aes = OpenSSL::Cipher::Cipher.new( 'aes-256-cbc' )
|
39
|
+
aes.encrypt
|
40
|
+
|
41
|
+
# use random key and IV
|
42
|
+
aes.key = key = aes.random_key
|
43
|
+
aes.iv = iv = aes.random_iv
|
44
|
+
|
45
|
+
# this will hold all primitives and ciphertext
|
46
|
+
primitives = {}
|
47
|
+
|
48
|
+
primitives['ciphertext'] = aes.update( data )
|
49
|
+
primitives['ciphertext'] << aes.final
|
50
|
+
|
51
|
+
primitives['key'] = rsa.public_encrypt( key )
|
52
|
+
primitives['iv'] = rsa.public_encrypt( iv )
|
53
|
+
|
54
|
+
# serialize everything and base64 encode it
|
55
|
+
Base64.encode64( primitives.to_yaml )
|
56
|
+
end
|
57
|
+
|
58
|
+
# Decrypts data.
|
59
|
+
#
|
60
|
+
# @param [String] data
|
61
|
+
#
|
62
|
+
# @return [String]
|
63
|
+
# Plaintext.
|
64
|
+
def decrypt( data )
|
65
|
+
rsa = OpenSSL::PKey::RSA.new( File.read( @private_pem ) )
|
66
|
+
|
67
|
+
# decrypt with 256 bit AES with CBC
|
68
|
+
aes = OpenSSL::Cipher::Cipher.new( 'aes-256-cbc' )
|
69
|
+
aes.decrypt
|
70
|
+
|
71
|
+
# unencode and unserialize to get the primitives and ciphertext
|
72
|
+
primitives = YAML::load( Base64.decode64( data ) )
|
73
|
+
|
74
|
+
aes.key = rsa.private_decrypt( primitives['key'] )
|
75
|
+
aes.iv = rsa.private_decrypt( primitives['iv'] )
|
76
|
+
|
77
|
+
plaintext = aes.update( primitives['ciphertext'] )
|
78
|
+
plaintext << aes.final
|
79
|
+
|
80
|
+
plaintext
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,177 @@
|
|
1
|
+
module Cuboid
|
2
|
+
module Support::Database
|
3
|
+
|
4
|
+
# Base class for Database data structures
|
5
|
+
#
|
6
|
+
# Provides helper methods for data structures to be implemented related to
|
7
|
+
# objecting dumping, loading, unique filename generation, etc.
|
8
|
+
#
|
9
|
+
# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
|
10
|
+
#
|
11
|
+
# @abstract
|
12
|
+
class Base
|
13
|
+
|
14
|
+
DISK_SPACE_FILE = 'Database_disk_space'
|
15
|
+
|
16
|
+
class <<self
|
17
|
+
|
18
|
+
def reset
|
19
|
+
@@disk_space = 0
|
20
|
+
set_disk_space @@disk_space
|
21
|
+
end
|
22
|
+
|
23
|
+
def increment_disk_space( int )
|
24
|
+
set_disk_space @@disk_space + int
|
25
|
+
end
|
26
|
+
|
27
|
+
def decrement_disk_space( int )
|
28
|
+
set_disk_space @@disk_space - int
|
29
|
+
end
|
30
|
+
|
31
|
+
def disk_space
|
32
|
+
@@disk_space
|
33
|
+
end
|
34
|
+
|
35
|
+
def disk_directory
|
36
|
+
Options.paths.tmpdir
|
37
|
+
end
|
38
|
+
|
39
|
+
def disk_space_file
|
40
|
+
disk_space_file_for Process.pid
|
41
|
+
end
|
42
|
+
|
43
|
+
def disk_space_for( pid )
|
44
|
+
return 0 if !Dir.exists?( Options.paths.tmp_dir_for( pid ) )
|
45
|
+
|
46
|
+
IO.read( disk_space_file_for( pid ) ).to_i
|
47
|
+
end
|
48
|
+
|
49
|
+
def disk_space_file_for( pid )
|
50
|
+
"#{Options.paths.tmp_dir_for( pid )}/#{DISK_SPACE_FILE}"
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
def set_disk_space( int )
|
56
|
+
if !File.exist?( disk_directory )
|
57
|
+
# Could be caught in #at_exit callbacks, the tmpdir has already
|
58
|
+
# been deleted.
|
59
|
+
return
|
60
|
+
end
|
61
|
+
|
62
|
+
synchronize do
|
63
|
+
@@disk_space = int
|
64
|
+
IO.write( disk_space_file, @@disk_space.to_s )
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def synchronize( &block )
|
69
|
+
(@@mutex ||= Mutex.new).synchronize( &block )
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
reset
|
74
|
+
|
75
|
+
# @param [Hash] options
|
76
|
+
# Any object that responds to 'dump' and 'load'.
|
77
|
+
def initialize( options = {} )
|
78
|
+
@options = options.dup
|
79
|
+
|
80
|
+
@options[:dumper] ||= Marshal
|
81
|
+
@options[:loader] ||= @options[:dumper]
|
82
|
+
|
83
|
+
@filename_counter = 0
|
84
|
+
|
85
|
+
at_exit do
|
86
|
+
clear
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def serialize( obj )
|
91
|
+
compress(
|
92
|
+
@options[:dumper].respond_to?( :dump ) ?
|
93
|
+
@options[:dumper].dump( obj ) :
|
94
|
+
@options[:dumper].call( obj )
|
95
|
+
)
|
96
|
+
end
|
97
|
+
|
98
|
+
def unserialize( data )
|
99
|
+
data = decompress( data )
|
100
|
+
|
101
|
+
@options[:loader].respond_to?( :load ) ?
|
102
|
+
@options[:loader].load( data ) :
|
103
|
+
@options[:loader].call( data )
|
104
|
+
end
|
105
|
+
|
106
|
+
private
|
107
|
+
|
108
|
+
# Dumps the object to a unique file and returns its path.
|
109
|
+
#
|
110
|
+
# The path can be used as a reference to the original value
|
111
|
+
# by way of passing it to load().
|
112
|
+
#
|
113
|
+
# @param [Object] obj
|
114
|
+
#
|
115
|
+
# @return [String]
|
116
|
+
# Filepath
|
117
|
+
def dump( obj, &block )
|
118
|
+
File.open( get_unique_filename, 'wb' ) do |f|
|
119
|
+
serialized = serialize( obj )
|
120
|
+
self.class.increment_disk_space f.write( serialized )
|
121
|
+
|
122
|
+
block.call( serialized ) if block_given?
|
123
|
+
|
124
|
+
f.path
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
# Loads the object stored in filepath.
|
129
|
+
#
|
130
|
+
# @param [String] filepath
|
131
|
+
#
|
132
|
+
# @return [Object]
|
133
|
+
def load( filepath )
|
134
|
+
unserialize( IO.binread( filepath ) )
|
135
|
+
end
|
136
|
+
|
137
|
+
# Deletes a file.
|
138
|
+
#
|
139
|
+
# @param [String] filepath
|
140
|
+
def delete_file( filepath )
|
141
|
+
return if !File.exist?( filepath )
|
142
|
+
|
143
|
+
self.class.decrement_disk_space File.size( filepath )
|
144
|
+
File.delete( filepath )
|
145
|
+
end
|
146
|
+
|
147
|
+
# Loads the object in file and then removes it from the file-system.
|
148
|
+
#
|
149
|
+
# @param [String] filepath
|
150
|
+
#
|
151
|
+
# @return [Object]
|
152
|
+
def load_and_delete_file( filepath )
|
153
|
+
obj = load( filepath )
|
154
|
+
delete_file( filepath )
|
155
|
+
obj
|
156
|
+
end
|
157
|
+
|
158
|
+
def compress( string )
|
159
|
+
Zlib::Deflate.deflate string
|
160
|
+
end
|
161
|
+
|
162
|
+
def decompress( string )
|
163
|
+
Zlib::Inflate.inflate string
|
164
|
+
end
|
165
|
+
|
166
|
+
def get_unique_filename
|
167
|
+
# Should be unique enough...
|
168
|
+
("#{self.class.disk_directory}/#{self.class.name}_#{object_id}_" <<
|
169
|
+
@filename_counter.to_s).gsub( '::', '_' )
|
170
|
+
ensure
|
171
|
+
@filename_counter += 1
|
172
|
+
end
|
173
|
+
|
174
|
+
end
|
175
|
+
|
176
|
+
end
|
177
|
+
end
|
@@ -0,0 +1,195 @@
|
|
1
|
+
module Cuboid
|
2
|
+
module Support::Database
|
3
|
+
|
4
|
+
# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
|
5
|
+
class CategorizedQueue < Base
|
6
|
+
|
7
|
+
# Default {#max_buffer_size}.
|
8
|
+
DEFAULT_MAX_BUFFER_SIZE = 100
|
9
|
+
|
10
|
+
# @return [Integer]
|
11
|
+
# How many entries to keep in memory before starting to off-load to disk.
|
12
|
+
attr_accessor :max_buffer_size
|
13
|
+
|
14
|
+
attr_accessor :prefer
|
15
|
+
|
16
|
+
# @see Cuboid::Database::Base#initialize
|
17
|
+
def initialize( options = {}, &block )
|
18
|
+
super( options )
|
19
|
+
|
20
|
+
@prefer = block
|
21
|
+
@max_buffer_size = options[:max_buffer_size] || DEFAULT_MAX_BUFFER_SIZE
|
22
|
+
|
23
|
+
@categories ||= {}
|
24
|
+
@waiting = []
|
25
|
+
@mutex = Mutex.new
|
26
|
+
|
27
|
+
@buffer_size = 0
|
28
|
+
@disk_size = 0
|
29
|
+
end
|
30
|
+
|
31
|
+
# @note Defaults to {DEFAULT_MAX_BUFFER_SIZE}.
|
32
|
+
#
|
33
|
+
# @return [Integer]
|
34
|
+
# How many entries to keep in memory before starting to off-load to disk.
|
35
|
+
def max_buffer_size
|
36
|
+
@max_buffer_size
|
37
|
+
end
|
38
|
+
|
39
|
+
def categories
|
40
|
+
@categories.keys
|
41
|
+
end
|
42
|
+
|
43
|
+
def data_for( category )
|
44
|
+
@categories[category.to_s] ||= {
|
45
|
+
disk: [],
|
46
|
+
buffer: []
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
def insert_to_disk( category, path )
|
51
|
+
data_for( category )[:disk] << path
|
52
|
+
@disk_size += 1
|
53
|
+
end
|
54
|
+
|
55
|
+
# @param [Object] obj
|
56
|
+
# Object to add to the queue.
|
57
|
+
# Must respond to #category.
|
58
|
+
def <<( obj )
|
59
|
+
fail ArgumentError, 'Missing #prefer block.' if !@prefer
|
60
|
+
|
61
|
+
if !obj.respond_to?( :category )
|
62
|
+
fail ArgumentError, "#{obj.class} does not respond to #category."
|
63
|
+
end
|
64
|
+
|
65
|
+
synchronize do
|
66
|
+
data = data_for( obj.category )
|
67
|
+
|
68
|
+
if data[:buffer].size < max_buffer_size
|
69
|
+
@buffer_size += 1
|
70
|
+
data[:buffer] << obj
|
71
|
+
else
|
72
|
+
@disk_size += 1
|
73
|
+
data[:disk] << dump( obj )
|
74
|
+
end
|
75
|
+
|
76
|
+
begin
|
77
|
+
t = @waiting.shift
|
78
|
+
t.wakeup if t
|
79
|
+
rescue ThreadError
|
80
|
+
retry
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
alias :push :<<
|
85
|
+
alias :enq :<<
|
86
|
+
|
87
|
+
# @return [Object]
|
88
|
+
# Removes an object from the queue and returns it.
|
89
|
+
def pop( non_block = false )
|
90
|
+
fail ArgumentError, 'Missing #prefer block.' if !@prefer
|
91
|
+
|
92
|
+
synchronize do
|
93
|
+
loop do
|
94
|
+
if internal_empty?
|
95
|
+
raise ThreadError, 'queue empty' if non_block
|
96
|
+
@waiting.push Thread.current
|
97
|
+
@mutex.sleep
|
98
|
+
else
|
99
|
+
# Get preferred category, hopefully there'll be some data
|
100
|
+
# for it.
|
101
|
+
category = @prefer.call( @categories.keys )
|
102
|
+
|
103
|
+
# Get all other available categories just in case the
|
104
|
+
# preferred one is empty.
|
105
|
+
categories = @categories.keys
|
106
|
+
categories.delete category
|
107
|
+
|
108
|
+
data = nil
|
109
|
+
# Check if our category has data and pick another if not.
|
110
|
+
loop do
|
111
|
+
data = data_for( category )
|
112
|
+
if data[:buffer].any? || data[:disk].any?
|
113
|
+
break
|
114
|
+
end
|
115
|
+
|
116
|
+
category = categories.pop
|
117
|
+
end
|
118
|
+
|
119
|
+
if data[:buffer].any?
|
120
|
+
@buffer_size -= 1
|
121
|
+
return data[:buffer].shift
|
122
|
+
end
|
123
|
+
|
124
|
+
@disk_size -= 1
|
125
|
+
return load_and_delete_file( data[:disk].shift )
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
alias :deq :pop
|
131
|
+
alias :shift :pop
|
132
|
+
|
133
|
+
# @return [Integer]
|
134
|
+
# Size of the queue, the number of objects it currently holds.
|
135
|
+
def size
|
136
|
+
buffer_size + disk_size
|
137
|
+
end
|
138
|
+
alias :length :size
|
139
|
+
|
140
|
+
def free_buffer_size
|
141
|
+
max_buffer_size - buffer_size
|
142
|
+
end
|
143
|
+
|
144
|
+
def buffer_size
|
145
|
+
@buffer_size
|
146
|
+
end
|
147
|
+
|
148
|
+
def disk_size
|
149
|
+
@disk_size
|
150
|
+
end
|
151
|
+
|
152
|
+
# @return [Bool]
|
153
|
+
# `true` if the queue if empty, `false` otherwise.
|
154
|
+
def empty?
|
155
|
+
synchronize do
|
156
|
+
internal_empty?
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
# Removes all objects from the queue.
|
161
|
+
def clear
|
162
|
+
synchronize do
|
163
|
+
@categories.values.each do |data|
|
164
|
+
data[:buffer].clear
|
165
|
+
|
166
|
+
while !data[:disk].empty?
|
167
|
+
path = data[:disk].pop
|
168
|
+
next if !path
|
169
|
+
delete_file path
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
@buffer_size = 0
|
174
|
+
@disk_size = 0
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
def num_waiting
|
179
|
+
@waiting.size
|
180
|
+
end
|
181
|
+
|
182
|
+
private
|
183
|
+
|
184
|
+
def internal_empty?
|
185
|
+
@buffer_size == 0 && @disk_size == 0
|
186
|
+
end
|
187
|
+
|
188
|
+
def synchronize( &block )
|
189
|
+
@mutex.synchronize( &block )
|
190
|
+
end
|
191
|
+
|
192
|
+
end
|
193
|
+
|
194
|
+
end
|
195
|
+
end
|