offroad 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +674 -674
- data/README.rdoc +29 -29
- data/Rakefile +75 -75
- data/TODO +42 -42
- data/lib/app/models/offroad/group_state.rb +85 -85
- data/lib/app/models/offroad/mirror_info.rb +53 -53
- data/lib/app/models/offroad/model_state.rb +36 -36
- data/lib/app/models/offroad/received_record_state.rb +115 -115
- data/lib/app/models/offroad/sendable_record_state.rb +91 -91
- data/lib/app/models/offroad/system_state.rb +33 -33
- data/lib/cargo_streamer.rb +222 -222
- data/lib/controller_extensions.rb +74 -74
- data/lib/exceptions.rb +16 -16
- data/lib/migrate/20100512164608_create_offroad_tables.rb +72 -72
- data/lib/mirror_data.rb +376 -376
- data/lib/model_extensions.rb +378 -377
- data/lib/module_funcs.rb +94 -94
- data/lib/offroad.rb +41 -41
- data/lib/version.rb +3 -3
- data/lib/view_helper.rb +7 -7
- data/templates/offline.rb +36 -36
- data/templates/offline_database.yml +7 -7
- data/templates/offroad.yml +6 -6
- data/test/app_root/app/controllers/application_controller.rb +2 -2
- data/test/app_root/app/controllers/group_controller.rb +28 -28
- data/test/app_root/app/models/global_record.rb +10 -10
- data/test/app_root/app/models/group.rb +12 -12
- data/test/app_root/app/models/group_owned_record.rb +68 -68
- data/test/app_root/app/models/guest.rb +7 -7
- data/test/app_root/app/models/subrecord.rb +12 -12
- data/test/app_root/app/models/unmirrored_record.rb +4 -4
- data/test/app_root/app/views/group/download_down_mirror.html.erb +3 -3
- data/test/app_root/app/views/group/download_initial_down_mirror.html.erb +3 -3
- data/test/app_root/app/views/group/download_up_mirror.html.erb +5 -5
- data/test/app_root/app/views/layouts/mirror.html.erb +8 -8
- data/test/app_root/config/boot.rb +115 -115
- data/test/app_root/config/database-pg.yml +8 -8
- data/test/app_root/config/database.yml +5 -5
- data/test/app_root/config/environment.rb +24 -24
- data/test/app_root/config/environments/test.rb +17 -17
- data/test/app_root/config/offroad.yml +6 -6
- data/test/app_root/config/routes.rb +4 -4
- data/test/app_root/db/migrate/20100529235049_create_tables.rb +64 -64
- data/test/app_root/lib/common_hobo.rb +15 -15
- data/test/app_root/vendor/plugins/offroad/init.rb +2 -2
- data/test/functional/mirror_operations_test.rb +148 -148
- data/test/test_helper.rb +453 -453
- data/test/unit/app_state_tracking_test.rb +275 -275
- data/test/unit/cargo_streamer_test.rb +332 -332
- data/test/unit/global_data_test.rb +102 -102
- data/test/unit/group_controller_test.rb +152 -152
- data/test/unit/group_data_test.rb +442 -435
- data/test/unit/group_single_test.rb +136 -136
- data/test/unit/hobo_permissions_test.rb +57 -57
- data/test/unit/mirror_data_test.rb +1283 -1283
- data/test/unit/mirror_info_test.rb +31 -31
- data/test/unit/module_funcs_test.rb +37 -37
- data/test/unit/pathological_model_test.rb +62 -62
- data/test/unit/test_framework_test.rb +86 -86
- data/test/unit/unmirrored_data_test.rb +14 -14
- metadata +6 -8
@@ -1,74 +1,74 @@
|
|
1
|
-
module Offroad
|
2
|
-
module ControllerExtensions
|
3
|
-
def offroad_group_controller
|
4
|
-
include InstanceMethods
|
5
|
-
end
|
6
|
-
|
7
|
-
module InstanceMethods
|
8
|
-
def render_up_mirror_file(group, filename, render_args = {})
|
9
|
-
ensure_group_offline(group)
|
10
|
-
raise PluginError.new("Cannot generate up-mirror file when app in online mode") if Offroad::app_online?
|
11
|
-
mirror_data = MirrorData.new(
|
12
|
-
group,
|
13
|
-
:skip_validation => render_args.delete(:skip_validation)
|
14
|
-
)
|
15
|
-
render_appending_mirror_data(group, filename, render_args) do |output|
|
16
|
-
mirror_data.write_upwards_data(output)
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def render_down_mirror_file(group, filename, render_args = {})
|
21
|
-
ensure_group_offline(group)
|
22
|
-
raise PluginError.new("Cannot generate down-mirror file when app in offline mode") if Offroad::app_offline?
|
23
|
-
mirror_data = MirrorData.new(
|
24
|
-
group,
|
25
|
-
:initial_mode => render_args.delete(:initial_mode),
|
26
|
-
:skip_validation => render_args.delete(:skip_validation)
|
27
|
-
)
|
28
|
-
render_appending_mirror_data(group, filename, render_args) do |output|
|
29
|
-
mirror_data.write_downwards_data(output)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
def load_up_mirror_file(group, data, options = {})
|
34
|
-
ensure_group_offline(group)
|
35
|
-
raise PluginError.new("Cannot accept up mirror file when app is in offline mode") if Offroad::app_offline?
|
36
|
-
m = MirrorData.new(
|
37
|
-
group,
|
38
|
-
:skip_validation => options[:skip_validation]
|
39
|
-
)
|
40
|
-
m.load_upwards_data(data)
|
41
|
-
end
|
42
|
-
|
43
|
-
def load_down_mirror_file(group, data, options = {})
|
44
|
-
ensure_group_offline(group) if group
|
45
|
-
raise PluginError.new("Cannot accept down mirror file when app is in online mode") if Offroad::app_online?
|
46
|
-
m = MirrorData.new(
|
47
|
-
group,
|
48
|
-
:initial_mode => options[:initial_mode],
|
49
|
-
:skip_validation => options[:skip_validation]
|
50
|
-
)
|
51
|
-
m.load_downwards_data(data)
|
52
|
-
end
|
53
|
-
|
54
|
-
private
|
55
|
-
|
56
|
-
def ensure_group_offline(group)
|
57
|
-
raise PluginError.new("Cannot perform mirror operations on online group") unless group.group_offline?
|
58
|
-
end
|
59
|
-
|
60
|
-
def render_appending_mirror_data(group, filename, render_args)
|
61
|
-
# Encourage browser to download this to disk instead of displaying it
|
62
|
-
headers['Content-Disposition'] = "attachment; filename=\"#{filename}\""
|
63
|
-
viewable_content = render_to_string render_args
|
64
|
-
|
65
|
-
render_proc = Proc.new do |response, output|
|
66
|
-
output.write(viewable_content)
|
67
|
-
yield output
|
68
|
-
end
|
69
|
-
|
70
|
-
render :text => render_proc
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
1
|
+
module Offroad
|
2
|
+
module ControllerExtensions
|
3
|
+
def offroad_group_controller
|
4
|
+
include InstanceMethods
|
5
|
+
end
|
6
|
+
|
7
|
+
module InstanceMethods
|
8
|
+
def render_up_mirror_file(group, filename, render_args = {})
|
9
|
+
ensure_group_offline(group)
|
10
|
+
raise PluginError.new("Cannot generate up-mirror file when app in online mode") if Offroad::app_online?
|
11
|
+
mirror_data = MirrorData.new(
|
12
|
+
group,
|
13
|
+
:skip_validation => render_args.delete(:skip_validation)
|
14
|
+
)
|
15
|
+
render_appending_mirror_data(group, filename, render_args) do |output|
|
16
|
+
mirror_data.write_upwards_data(output)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def render_down_mirror_file(group, filename, render_args = {})
|
21
|
+
ensure_group_offline(group)
|
22
|
+
raise PluginError.new("Cannot generate down-mirror file when app in offline mode") if Offroad::app_offline?
|
23
|
+
mirror_data = MirrorData.new(
|
24
|
+
group,
|
25
|
+
:initial_mode => render_args.delete(:initial_mode),
|
26
|
+
:skip_validation => render_args.delete(:skip_validation)
|
27
|
+
)
|
28
|
+
render_appending_mirror_data(group, filename, render_args) do |output|
|
29
|
+
mirror_data.write_downwards_data(output)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def load_up_mirror_file(group, data, options = {})
|
34
|
+
ensure_group_offline(group)
|
35
|
+
raise PluginError.new("Cannot accept up mirror file when app is in offline mode") if Offroad::app_offline?
|
36
|
+
m = MirrorData.new(
|
37
|
+
group,
|
38
|
+
:skip_validation => options[:skip_validation]
|
39
|
+
)
|
40
|
+
m.load_upwards_data(data)
|
41
|
+
end
|
42
|
+
|
43
|
+
def load_down_mirror_file(group, data, options = {})
|
44
|
+
ensure_group_offline(group) if group
|
45
|
+
raise PluginError.new("Cannot accept down mirror file when app is in online mode") if Offroad::app_online?
|
46
|
+
m = MirrorData.new(
|
47
|
+
group,
|
48
|
+
:initial_mode => options[:initial_mode],
|
49
|
+
:skip_validation => options[:skip_validation]
|
50
|
+
)
|
51
|
+
m.load_downwards_data(data)
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def ensure_group_offline(group)
|
57
|
+
raise PluginError.new("Cannot perform mirror operations on online group") unless group.group_offline?
|
58
|
+
end
|
59
|
+
|
60
|
+
def render_appending_mirror_data(group, filename, render_args)
|
61
|
+
# Encourage browser to download this to disk instead of displaying it
|
62
|
+
headers['Content-Disposition'] = "attachment; filename=\"#{filename}\""
|
63
|
+
viewable_content = render_to_string render_args
|
64
|
+
|
65
|
+
render_proc = Proc.new do |response, output|
|
66
|
+
output.write(viewable_content)
|
67
|
+
yield output
|
68
|
+
end
|
69
|
+
|
70
|
+
render :text => render_proc
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
data/lib/exceptions.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
|
-
module Offroad
|
2
|
-
class DataError < RuntimeError
|
3
|
-
end
|
4
|
-
|
5
|
-
class OldDataError < DataError
|
6
|
-
end
|
7
|
-
|
8
|
-
class ModelError < RuntimeError
|
9
|
-
end
|
10
|
-
|
11
|
-
class PluginError < RuntimeError
|
12
|
-
end
|
13
|
-
|
14
|
-
class AppModeUnknownError < RuntimeError
|
15
|
-
end
|
16
|
-
end
|
1
|
+
module Offroad
|
2
|
+
class DataError < RuntimeError
|
3
|
+
end
|
4
|
+
|
5
|
+
class OldDataError < DataError
|
6
|
+
end
|
7
|
+
|
8
|
+
class ModelError < RuntimeError
|
9
|
+
end
|
10
|
+
|
11
|
+
class PluginError < RuntimeError
|
12
|
+
end
|
13
|
+
|
14
|
+
class AppModeUnknownError < RuntimeError
|
15
|
+
end
|
16
|
+
end
|
@@ -1,72 +1,72 @@
|
|
1
|
-
class CreateOffroadTables < ActiveRecord::Migration
|
2
|
-
def self.up
|
3
|
-
create_table :offroad_system_state do |t|
|
4
|
-
t.column :current_mirror_version, :integer
|
5
|
-
end
|
6
|
-
|
7
|
-
create_table :offroad_group_states do |t|
|
8
|
-
t.column :app_group_id, :integer, :null => false
|
9
|
-
|
10
|
-
# This is used to allow group_owned records to be destroyed when their parent is.
|
11
|
-
# Without this, groups with :dependent => :destroy would not be allowed on the online app.
|
12
|
-
# This is NOT used to propogate group deletion through mirror files.
|
13
|
-
t.column :group_being_destroyed, :boolean, :default => false, :null => false
|
14
|
-
|
15
|
-
# If a group is locked (which can only occur offline) then its records cannot be altered anymore.
|
16
|
-
# If a locked group is sent up to the online system, then after loading it the online system puts that group back
|
17
|
-
# online.
|
18
|
-
t.column :group_locked, :boolean, :default => false, :null => false
|
19
|
-
|
20
|
-
# On both the online and offline systems, these are the latest data versions remote side is known to have
|
21
|
-
t.column :confirmed_group_data_version, :integer, :null => false
|
22
|
-
t.column :confirmed_global_data_version, :integer, :null => false
|
23
|
-
|
24
|
-
t.column :last_installer_downloaded_at, :datetime
|
25
|
-
t.column :last_installation_at, :datetime
|
26
|
-
t.column :last_down_mirror_created_at, :datetime
|
27
|
-
t.column :last_down_mirror_loaded_at, :datetime
|
28
|
-
t.column :last_up_mirror_created_at, :datetime
|
29
|
-
t.column :last_up_mirror_loaded_at, :datetime
|
30
|
-
t.column :launcher_version, :integer
|
31
|
-
t.column :app_version, :integer
|
32
|
-
t.column :operating_system, :string, :default => "Unknown", :null => false
|
33
|
-
end
|
34
|
-
add_index :offroad_group_states, :app_group_id, :unique => true
|
35
|
-
# This lets us quickly find min(global_mirror_version) for clearing old deleted global record SRSes
|
36
|
-
add_index :offroad_group_states, :confirmed_global_data_version
|
37
|
-
|
38
|
-
create_table :offroad_model_states do |t|
|
39
|
-
t.column :app_model_name, :string, :null => false
|
40
|
-
end
|
41
|
-
add_index :offroad_model_states, :app_model_name, :unique => true
|
42
|
-
|
43
|
-
create_table :offroad_sendable_record_states do |t|
|
44
|
-
t.column :model_state_id, :integer, :null => false
|
45
|
-
t.column :local_record_id, :integer, :null => false
|
46
|
-
t.column :mirror_version, :integer, :default => 0, :null => false
|
47
|
-
t.column :deleted, :boolean, :default => false, :null => false
|
48
|
-
end
|
49
|
-
# This index is for locating the SRS for any given local app record
|
50
|
-
add_index :offroad_sendable_record_states, [:local_record_id, :model_state_id], :unique => true
|
51
|
-
# This index is for generating mirror files: for a given model need to find everything above a given mirror_version
|
52
|
-
add_index :offroad_sendable_record_states, [:model_state_id, :deleted, :mirror_version]
|
53
|
-
|
54
|
-
create_table :offroad_received_record_states do |t|
|
55
|
-
t.column :model_state_id, :integer, :null => false
|
56
|
-
t.column :group_state_id, :integer, :default => 0, :null => false # If 0, is a global record
|
57
|
-
t.column :local_record_id, :integer, :null => false
|
58
|
-
t.column :remote_record_id, :integer, :null => false
|
59
|
-
end
|
60
|
-
add_index :offroad_received_record_states, [:model_state_id, :group_state_id, :remote_record_id], :unique => true
|
61
|
-
# TODO: Perhaps index below can be removed; it enforces data integrity, but isn't actually used for lookups
|
62
|
-
add_index :offroad_received_record_states, [:model_state_id, :local_record_id], :unique => true
|
63
|
-
end
|
64
|
-
|
65
|
-
def self.down
|
66
|
-
drop_table :offroad_system_state
|
67
|
-
drop_table :offroad_group_states
|
68
|
-
drop_table :offroad_model_states
|
69
|
-
drop_table :offroad_sendable_record_states
|
70
|
-
drop_table :offroad_received_record_states
|
71
|
-
end
|
72
|
-
end
|
1
|
+
class CreateOffroadTables < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
create_table :offroad_system_state do |t|
|
4
|
+
t.column :current_mirror_version, :integer
|
5
|
+
end
|
6
|
+
|
7
|
+
create_table :offroad_group_states do |t|
|
8
|
+
t.column :app_group_id, :integer, :null => false
|
9
|
+
|
10
|
+
# This is used to allow group_owned records to be destroyed when their parent is.
|
11
|
+
# Without this, groups with :dependent => :destroy would not be allowed on the online app.
|
12
|
+
# This is NOT used to propogate group deletion through mirror files.
|
13
|
+
t.column :group_being_destroyed, :boolean, :default => false, :null => false
|
14
|
+
|
15
|
+
# If a group is locked (which can only occur offline) then its records cannot be altered anymore.
|
16
|
+
# If a locked group is sent up to the online system, then after loading it the online system puts that group back
|
17
|
+
# online.
|
18
|
+
t.column :group_locked, :boolean, :default => false, :null => false
|
19
|
+
|
20
|
+
# On both the online and offline systems, these are the latest data versions remote side is known to have
|
21
|
+
t.column :confirmed_group_data_version, :integer, :null => false
|
22
|
+
t.column :confirmed_global_data_version, :integer, :null => false
|
23
|
+
|
24
|
+
t.column :last_installer_downloaded_at, :datetime
|
25
|
+
t.column :last_installation_at, :datetime
|
26
|
+
t.column :last_down_mirror_created_at, :datetime
|
27
|
+
t.column :last_down_mirror_loaded_at, :datetime
|
28
|
+
t.column :last_up_mirror_created_at, :datetime
|
29
|
+
t.column :last_up_mirror_loaded_at, :datetime
|
30
|
+
t.column :launcher_version, :integer
|
31
|
+
t.column :app_version, :integer
|
32
|
+
t.column :operating_system, :string, :default => "Unknown", :null => false
|
33
|
+
end
|
34
|
+
add_index :offroad_group_states, :app_group_id, :unique => true
|
35
|
+
# This lets us quickly find min(global_mirror_version) for clearing old deleted global record SRSes
|
36
|
+
add_index :offroad_group_states, :confirmed_global_data_version
|
37
|
+
|
38
|
+
create_table :offroad_model_states do |t|
|
39
|
+
t.column :app_model_name, :string, :null => false
|
40
|
+
end
|
41
|
+
add_index :offroad_model_states, :app_model_name, :unique => true
|
42
|
+
|
43
|
+
create_table :offroad_sendable_record_states do |t|
|
44
|
+
t.column :model_state_id, :integer, :null => false
|
45
|
+
t.column :local_record_id, :integer, :null => false
|
46
|
+
t.column :mirror_version, :integer, :default => 0, :null => false
|
47
|
+
t.column :deleted, :boolean, :default => false, :null => false
|
48
|
+
end
|
49
|
+
# This index is for locating the SRS for any given local app record
|
50
|
+
add_index :offroad_sendable_record_states, [:local_record_id, :model_state_id], :unique => true
|
51
|
+
# This index is for generating mirror files: for a given model need to find everything above a given mirror_version
|
52
|
+
add_index :offroad_sendable_record_states, [:model_state_id, :deleted, :mirror_version]
|
53
|
+
|
54
|
+
create_table :offroad_received_record_states do |t|
|
55
|
+
t.column :model_state_id, :integer, :null => false
|
56
|
+
t.column :group_state_id, :integer, :default => 0, :null => false # If 0, is a global record
|
57
|
+
t.column :local_record_id, :integer, :null => false
|
58
|
+
t.column :remote_record_id, :integer, :null => false
|
59
|
+
end
|
60
|
+
add_index :offroad_received_record_states, [:model_state_id, :group_state_id, :remote_record_id], :unique => true
|
61
|
+
# TODO: Perhaps index below can be removed; it enforces data integrity, but isn't actually used for lookups
|
62
|
+
add_index :offroad_received_record_states, [:model_state_id, :local_record_id], :unique => true
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.down
|
66
|
+
drop_table :offroad_system_state
|
67
|
+
drop_table :offroad_group_states
|
68
|
+
drop_table :offroad_model_states
|
69
|
+
drop_table :offroad_sendable_record_states
|
70
|
+
drop_table :offroad_received_record_states
|
71
|
+
end
|
72
|
+
end
|
data/lib/mirror_data.rb
CHANGED
@@ -1,376 +1,376 @@
|
|
1
|
-
module Offroad
|
2
|
-
private
|
3
|
-
|
4
|
-
class MirrorData
|
5
|
-
attr_reader :group, :mode
|
6
|
-
|
7
|
-
def initialize(group, options = {})
|
8
|
-
@group = group
|
9
|
-
@initial_mode = options.delete(:initial_mode) || false
|
10
|
-
@skip_validation = options.delete(:skip_validation) || false
|
11
|
-
|
12
|
-
raise PluginError.new("Invalid option keys") unless options.size == 0
|
13
|
-
|
14
|
-
unless Offroad::app_offline? && @initial_mode
|
15
|
-
raise PluginError.new("Need group") unless @group.is_a?(Offroad::group_base_model) && !@group.new_record?
|
16
|
-
raise DataError.new("Group must be in offline mode") unless @group.group_offline?
|
17
|
-
end
|
18
|
-
|
19
|
-
@imported_models_to_validate = []
|
20
|
-
end
|
21
|
-
|
22
|
-
def write_upwards_data(tgt = nil)
|
23
|
-
raise PluginError.new("Can only write upwards data in offline mode") unless Offroad.app_offline?
|
24
|
-
raise PluginError.new("No such thing as initial upwards data") if @initial_mode
|
25
|
-
write_data(tgt) do |cs|
|
26
|
-
add_group_specific_cargo(cs)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
def write_downwards_data(tgt = nil)
|
31
|
-
raise PluginError.new("Can only write downwards data in online mode") unless Offroad.app_online?
|
32
|
-
write_data(tgt) do |cs|
|
33
|
-
add_global_cargo(cs)
|
34
|
-
if @initial_mode
|
35
|
-
add_group_specific_cargo(cs)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
def load_upwards_data(src)
|
41
|
-
raise PluginError.new("Can only load upwards data in online mode") unless Offroad.app_online?
|
42
|
-
raise PluginError.new("No such thing as initial upwards data") if @initial_mode
|
43
|
-
|
44
|
-
read_data_from("offline", src) do |cs, mirror_info, cargo_group_state|
|
45
|
-
unless cargo_group_state.confirmed_group_data_version > @group.group_state.confirmed_group_data_version
|
46
|
-
raise OldDataError.new("File contains old up-mirror data")
|
47
|
-
end
|
48
|
-
import_group_specific_cargo(cs)
|
49
|
-
@group.group_offline = false if cargo_group_state.group_locked?
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
def load_downwards_data(src)
|
54
|
-
raise PluginError.new("Can only load downwards data in offline mode") unless Offroad.app_offline?
|
55
|
-
|
56
|
-
read_data_from("online", src) do |cs, mirror_info, cargo_group_state|
|
57
|
-
raise DataError.new("Unexpected initial file value") unless mirror_info.initial_file == @initial_mode
|
58
|
-
|
59
|
-
group_cargo_name = MirrorData::data_cargo_name_for_model(Offroad::group_base_model)
|
60
|
-
if mirror_info.initial_file
|
61
|
-
raise DataError.new("No group data in initial down mirror file") unless cs.has_cargo_named?(group_cargo_name)
|
62
|
-
# This is an initial mirror file, so we want it to determine the entirety of the database's new state
|
63
|
-
# However, existing data is safe if there's a mid-import error; read_data_from places us in a transaction
|
64
|
-
delete_all_existing_database_records!
|
65
|
-
|
66
|
-
import_global_cargo(cs) # Global cargo must be done first because group data might belong_to global data
|
67
|
-
import_group_specific_cargo(cs)
|
68
|
-
else
|
69
|
-
# Regular, non-initial down mirror file
|
70
|
-
unless cargo_group_state.confirmed_global_data_version > @group.group_state.confirmed_global_data_version
|
71
|
-
raise OldDataError.new("File contains old down-mirror data")
|
72
|
-
end
|
73
|
-
import_global_cargo(cs)
|
74
|
-
end
|
75
|
-
|
76
|
-
# Load information into our group state that the online app is in a better position to know about
|
77
|
-
@group = Offroad::offline_group if @initial_mode
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
private
|
82
|
-
|
83
|
-
def self.data_cargo_name_for_model(model)
|
84
|
-
"data_#{model.name}"
|
85
|
-
end
|
86
|
-
|
87
|
-
def self.deletion_cargo_name_for_model(model)
|
88
|
-
"deletion_#{model.name}"
|
89
|
-
end
|
90
|
-
|
91
|
-
def delete_all_existing_database_records!
|
92
|
-
tables = ActiveRecord::Base.connection.tables
|
93
|
-
if ActiveRecord::Base.connection.adapter_name.downcase.include?("sqlite")
|
94
|
-
# Emptying sqlite_sequence resets SQLite's autoincrement counters.
|
95
|
-
# SQLite's autoincrement is nice in that it automatically picks largest ever id + 1.
|
96
|
-
# This means that after clearing sqlite_sequence and then populating database with manually-id'd rows,
|
97
|
-
# new records will be inserted with unique id's, no problem.
|
98
|
-
tables << "sqlite_sequence"
|
99
|
-
end
|
100
|
-
|
101
|
-
tables.each do |table|
|
102
|
-
next if table.start_with?("virtual_") # Used in testing # FIXME Should pick something less likely to collide with app name
|
103
|
-
next if table == "schema_migrations"
|
104
|
-
ActiveRecord::Base.connection.execute "DELETE FROM #{table}"
|
105
|
-
end
|
106
|
-
|
107
|
-
if ActiveRecord::Base.connection.adapter_name.downcase.include?("postgres")
|
108
|
-
# Reset all sequences so that autoincremented ids start from 1 again
|
109
|
-
seqnames = ActiveRecord::Base.connection.select_values "SELECT c.relname FROM pg_class c WHERE c.relkind = 'S'"
|
110
|
-
seqnames.each do |s|
|
111
|
-
ActiveRecord::Base.connection.execute "SELECT setval('#{s}', 1, false)"
|
112
|
-
end
|
113
|
-
end
|
114
|
-
end
|
115
|
-
|
116
|
-
def write_data(tgt)
|
117
|
-
cs = nil
|
118
|
-
temp_sio = nil
|
119
|
-
case tgt
|
120
|
-
when CargoStreamer
|
121
|
-
cs = tgt
|
122
|
-
when nil
|
123
|
-
temp_sio = StringIO.new("", "w")
|
124
|
-
cs = CargoStreamer.new(temp_sio, "w")
|
125
|
-
else
|
126
|
-
cs = CargoStreamer.new(tgt, "w")
|
127
|
-
end
|
128
|
-
|
129
|
-
# TODO: Figure out if this transaction ensures we get a consistent read state
|
130
|
-
Offroad::group_base_model.connection.transaction do
|
131
|
-
Offroad::group_base_model.cache do
|
132
|
-
begin
|
133
|
-
mirror_info = MirrorInfo.new_from_group(@group, @initial_mode)
|
134
|
-
cs.write_cargo_section("mirror_info", [mirror_info], :human_readable => true)
|
135
|
-
|
136
|
-
group_state = @group.group_state
|
137
|
-
if Offroad::app_online?
|
138
|
-
# Let the offline app know what global data version it's being updated to
|
139
|
-
group_state.confirmed_global_data_version = SystemState::current_mirror_version
|
140
|
-
else
|
141
|
-
# Let the online app know what group data version the online mirror of this group is being updated to
|
142
|
-
group_state.confirmed_group_data_version = SystemState::current_mirror_version
|
143
|
-
end
|
144
|
-
cs.write_cargo_section("group_state", [group_state], :human_readable => true)
|
145
|
-
|
146
|
-
yield cs
|
147
|
-
|
148
|
-
SystemState::increment_mirror_version
|
149
|
-
rescue Offroad::CargoStreamerError
|
150
|
-
raise Offroad::DataError.new("Encountered data validation error while writing to cargo file")
|
151
|
-
end
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
155
|
-
return temp_sio.string if temp_sio
|
156
|
-
end
|
157
|
-
|
158
|
-
def read_data_from(expected_source_app_mode, src)
|
159
|
-
cs = case src
|
160
|
-
when CargoStreamer then src
|
161
|
-
when String then CargoStreamer.new(StringIO.new(src, "r"), "r")
|
162
|
-
else CargoStreamer.new(src, "r")
|
163
|
-
end
|
164
|
-
|
165
|
-
raise DataError.new("Invalid mirror file, no info section found") unless cs.has_cargo_named?("mirror_info")
|
166
|
-
mirror_info = cs.first_cargo_element("mirror_info")
|
167
|
-
raise DataError.new("Invalid info section type") unless mirror_info.is_a?(MirrorInfo)
|
168
|
-
unless mirror_info.app_mode.downcase == expected_source_app_mode.downcase
|
169
|
-
raise DataError.new "Mirror file was generated by app in wrong mode; was expecting #{expected_source_app_mode}"
|
170
|
-
end
|
171
|
-
|
172
|
-
raise DataError.new("Invalid mirror file, no group state found") unless cs.has_cargo_named?("group_state")
|
173
|
-
group_state = cs.first_cargo_element("group_state")
|
174
|
-
raise DataError.new("Invalid group state type") unless group_state.is_a?(GroupState)
|
175
|
-
group_state.readonly!
|
176
|
-
|
177
|
-
# FIXME: Is this transaction call helping at all?
|
178
|
-
Offroad::group_base_model.connection.transaction do
|
179
|
-
Offroad::group_base_model.cache do
|
180
|
-
yield cs, mirror_info, group_state
|
181
|
-
validate_imported_models(cs) unless @skip_validation
|
182
|
-
|
183
|
-
# Load information into our group state that the remote app is in a better position to know about
|
184
|
-
@group.group_state.update_from_remote_group_state!(group_state) if @group && @group.group_offline?
|
185
|
-
end
|
186
|
-
end
|
187
|
-
|
188
|
-
SystemState::increment_mirror_version if @initial_mode
|
189
|
-
end
|
190
|
-
|
191
|
-
def add_group_specific_cargo(cs)
|
192
|
-
Offroad::group_owned_models.each do |name, model|
|
193
|
-
add_model_cargo(cs, model)
|
194
|
-
end
|
195
|
-
Offroad::group_single_models.each do |name, model|
|
196
|
-
add_model_cargo(cs, model)
|
197
|
-
end
|
198
|
-
add_model_cargo(cs, Offroad::group_base_model)
|
199
|
-
end
|
200
|
-
|
201
|
-
def add_global_cargo(cs)
|
202
|
-
Offroad::global_data_models.each do |name, model|
|
203
|
-
add_model_cargo(cs, model)
|
204
|
-
end
|
205
|
-
end
|
206
|
-
|
207
|
-
def add_model_cargo(cs, model)
|
208
|
-
if @initial_mode
|
209
|
-
add_initial_model_cargo(cs, model)
|
210
|
-
else
|
211
|
-
add_non_initial_model_cargo(cs, model)
|
212
|
-
end
|
213
|
-
end
|
214
|
-
|
215
|
-
def add_initial_model_cargo(cs, model)
|
216
|
-
# Include the data for relevant records in this model
|
217
|
-
data_source = model
|
218
|
-
data_source = data_source.owned_by_offroad_group(@group) if model.offroad_group_data? && @group
|
219
|
-
data_source.find_in_batches(:batch_size => 100) do |batch|
|
220
|
-
cs.write_cargo_section(
|
221
|
-
MirrorData::data_cargo_name_for_model(model),
|
222
|
-
batch,
|
223
|
-
:skip_validation => @skip_validation
|
224
|
-
)
|
225
|
-
|
226
|
-
if model.offroad_group_data?
|
227
|
-
# In initial mode the remote app will create records with the same id's as the corresponding records here
|
228
|
-
# So we'll create RRSes indicating that we've already "received" the data we're about to send
|
229
|
-
# Later when the remote app sends new information on those records, we'll know which ones it means
|
230
|
-
rrs_source = Offroad::ReceivedRecordState.for_model_and_group_if_apropos(model, @group)
|
231
|
-
existing_rrs = rrs_source.all(:conditions => {:remote_record_id => batch.map(&:id)}).index_by(&:remote_record_id)
|
232
|
-
new_rrs = batch.reject{|r| existing_rrs.has_key?(r.id)}.map{|r| rrs_source.for_record(r).new(:remote_record_id => r.id)}
|
233
|
-
if new_rrs.size > 0
|
234
|
-
Offroad::ReceivedRecordState.import(new_rrs, :validate => false, :timestamps => false)
|
235
|
-
end
|
236
|
-
end
|
237
|
-
end
|
238
|
-
end
|
239
|
-
|
240
|
-
def add_non_initial_model_cargo(cs, model)
|
241
|
-
# Include the data for relevant records in this model that are newer than the remote side's known latest version
|
242
|
-
gs = @group.group_state
|
243
|
-
remote_version = nil
|
244
|
-
if model.offroad_group_data?
|
245
|
-
remote_version = gs.confirmed_group_data_version
|
246
|
-
else
|
247
|
-
remote_version = gs.confirmed_global_data_version
|
248
|
-
end
|
249
|
-
srs_source = SendableRecordState.for_model(model).with_version_greater_than(remote_version)
|
250
|
-
srs_source.for_non_deleted_records.find_in_batches(:batch_size => 100) do |srs_batch|
|
251
|
-
# TODO Might be able to optimize this to one query using a join on app model and SRS tables
|
252
|
-
record_ids = srs_batch.map { |srs| srs.local_record_id }
|
253
|
-
data_batch = model.find(:all, :conditions => {:id => record_ids})
|
254
|
-
raise PluginError.new("Invalid SRS ids") if data_batch.size != srs_batch.size
|
255
|
-
cs.write_cargo_section(
|
256
|
-
MirrorData::data_cargo_name_for_model(model),
|
257
|
-
data_batch,
|
258
|
-
:skip_validation => @skip_validation
|
259
|
-
)
|
260
|
-
end
|
261
|
-
|
262
|
-
# Also need to include information about records that have been destroyed
|
263
|
-
srs_source.for_deleted_records.find_in_batches(:batch_size => 100) do |deletion_batch|
|
264
|
-
cs.write_cargo_section(MirrorData::deletion_cargo_name_for_model(model), deletion_batch)
|
265
|
-
end
|
266
|
-
end
|
267
|
-
|
268
|
-
def import_group_specific_cargo(cs)
|
269
|
-
import_model_cargo(cs, Offroad::group_base_model)
|
270
|
-
Offroad::group_owned_models.each do |name, model|
|
271
|
-
import_model_cargo(cs, model)
|
272
|
-
end
|
273
|
-
Offroad::group_single_models.each do |name, model|
|
274
|
-
import_model_cargo(cs, model)
|
275
|
-
end
|
276
|
-
end
|
277
|
-
|
278
|
-
def import_global_cargo(cs)
|
279
|
-
Offroad::global_data_models.each do |name, model|
|
280
|
-
import_model_cargo(cs, model)
|
281
|
-
end
|
282
|
-
end
|
283
|
-
|
284
|
-
def import_model_cargo(cs, model)
|
285
|
-
@imported_models_to_validate.push model
|
286
|
-
|
287
|
-
if @initial_mode && model.offroad_group_data?
|
288
|
-
import_initial_model_cargo(cs, model)
|
289
|
-
else
|
290
|
-
import_non_initial_model_cargo(cs, model)
|
291
|
-
end
|
292
|
-
end
|
293
|
-
|
294
|
-
def import_initial_model_cargo(cs, model)
|
295
|
-
cs.each_cargo_section(MirrorData::data_cargo_name_for_model(model)) do |batch|
|
296
|
-
# Notice we are using the same primary key values as the online system, not allocating new ones
|
297
|
-
model.import batch, :validate => false, :timestamps => false
|
298
|
-
if model.offroad_group_base? && batch.size > 0
|
299
|
-
GroupState.for_group(model.first).create!
|
300
|
-
end
|
301
|
-
SendableRecordState.setup_imported(model, batch)
|
302
|
-
if model.instance_methods.include?("after_offroad_upload")
|
303
|
-
batch.each { |rec| rec.after_offroad_upload }
|
304
|
-
end
|
305
|
-
end
|
306
|
-
if ActiveRecord::Base.connection.adapter_name.downcase.include?("postgres")
|
307
|
-
# Need to adjust the sequences so that records inserted from this point on don't collide with existing ids
|
308
|
-
cols = ActiveRecord::Base.connection.select_rows "select table_name, column_name, column_default from information_schema.columns WHERE column_default like 'nextval%'"
|
309
|
-
cols.each do |table_name, column_name, column_default|
|
310
|
-
if column_default =~ /nextval\('(.+)'(?:::.+)?\)/
|
311
|
-
seqname = $1
|
312
|
-
ActiveRecord::Base.connection.execute "SELECT setval('#{seqname}', (SELECT MAX(\"#{column_name}\") FROM \"#{table_name}\"))"
|
313
|
-
end
|
314
|
-
end
|
315
|
-
end
|
316
|
-
end
|
317
|
-
|
318
|
-
def import_non_initial_model_cargo(cs, model)
|
319
|
-
rrs_source = ReceivedRecordState.for_model_and_group_if_apropos(model, @group)
|
320
|
-
|
321
|
-
# Update/create records
|
322
|
-
cs.each_cargo_section(MirrorData::data_cargo_name_for_model(model)) do |batch|
|
323
|
-
# Update foreign key associations to use local ids instead of remote ids
|
324
|
-
model.reflect_on_all_associations(:belongs_to).each do |a|
|
325
|
-
ReceivedRecordState.redirect_to_local_ids(batch, a.primary_key_name, a.klass, @group)
|
326
|
-
end
|
327
|
-
|
328
|
-
# Delete existing records in the database; that way we can just do INSERTs, don't have to worry about UPDATEs
|
329
|
-
# TODO: Is this necessary? Perhaps ar-extensions can deal with a mix of new and updated records...
|
330
|
-
model.delete rrs_source.all(:conditions => {:remote_record_id => batch.map(&:id)} ).map(&:local_record_id)
|
331
|
-
|
332
|
-
# Update the primary keys to use local ids, then insert the records
|
333
|
-
ReceivedRecordState.redirect_to_local_ids(batch, model.primary_key, model, @group)
|
334
|
-
model.import batch, :validate => false, :timestamps => false
|
335
|
-
|
336
|
-
if model.instance_methods.include?("after_offroad_upload")
|
337
|
-
batch.each { |rec| rec.after_offroad_upload }
|
338
|
-
end
|
339
|
-
end
|
340
|
-
|
341
|
-
# Delete records here which were destroyed there (except for group_base records, that would cause trouble)
|
342
|
-
return if model == Offroad::group_base_model
|
343
|
-
cs.each_cargo_section(MirrorData::deletion_cargo_name_for_model(model)) do |batch|
|
344
|
-
# If there's a callback, we need to load the local records before deleting them
|
345
|
-
local_recs = []
|
346
|
-
if model.instance_methods.include?("after_offroad_destroy")
|
347
|
-
local_recs = model.all(:conditions => {:id => batch.map(&:local_record_id)})
|
348
|
-
end
|
349
|
-
|
350
|
-
# Each deletion batch is made up of SendableRecordStates from the remote system
|
351
|
-
dying_rrs_batch = rrs_source.all(:conditions => {:remote_record_id => batch.map(&:local_record_id)})
|
352
|
-
model.delete dying_rrs_batch.map(&:local_record_id)
|
353
|
-
ReceivedRecordState.delete dying_rrs_batch.map(&:id)
|
354
|
-
local_recs.each { |rec| rec.after_offroad_destroy }
|
355
|
-
end
|
356
|
-
end
|
357
|
-
|
358
|
-
def validate_imported_models(cs)
|
359
|
-
Offroad::group_base_model.connection.clear_query_cache
|
360
|
-
while @imported_models_to_validate.size > 0
|
361
|
-
model = @imported_models_to_validate.pop
|
362
|
-
rrs_source = Offroad::ReceivedRecordState.for_model_and_group_if_apropos(model, @group)
|
363
|
-
|
364
|
-
cs.each_cargo_section(MirrorData::data_cargo_name_for_model(model)) do |cargo_batch|
|
365
|
-
if @initial_mode
|
366
|
-
local_batch = model.all(:conditions => {:id => cargo_batch.map(&:id)})
|
367
|
-
else
|
368
|
-
local_rrs_batch = rrs_source.all(:conditions => {:remote_record_id => cargo_batch.map(&:id)})
|
369
|
-
local_batch = model.all(:conditions => {:id => local_rrs_batch.map(&:local_record_id)})
|
370
|
-
end
|
371
|
-
raise Offroad::DataError.new("Invalid record found in mirror data") unless local_batch.all?(&:valid?)
|
372
|
-
end
|
373
|
-
end
|
374
|
-
end
|
375
|
-
end
|
376
|
-
end
|
1
|
+
module Offroad
|
2
|
+
private
|
3
|
+
|
4
|
+
class MirrorData
|
5
|
+
attr_reader :group, :mode
|
6
|
+
|
7
|
+
def initialize(group, options = {})
|
8
|
+
@group = group
|
9
|
+
@initial_mode = options.delete(:initial_mode) || false
|
10
|
+
@skip_validation = options.delete(:skip_validation) || false
|
11
|
+
|
12
|
+
raise PluginError.new("Invalid option keys") unless options.size == 0
|
13
|
+
|
14
|
+
unless Offroad::app_offline? && @initial_mode
|
15
|
+
raise PluginError.new("Need group") unless @group.is_a?(Offroad::group_base_model) && !@group.new_record?
|
16
|
+
raise DataError.new("Group must be in offline mode") unless @group.group_offline?
|
17
|
+
end
|
18
|
+
|
19
|
+
@imported_models_to_validate = []
|
20
|
+
end
|
21
|
+
|
22
|
+
def write_upwards_data(tgt = nil)
|
23
|
+
raise PluginError.new("Can only write upwards data in offline mode") unless Offroad.app_offline?
|
24
|
+
raise PluginError.new("No such thing as initial upwards data") if @initial_mode
|
25
|
+
write_data(tgt) do |cs|
|
26
|
+
add_group_specific_cargo(cs)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def write_downwards_data(tgt = nil)
|
31
|
+
raise PluginError.new("Can only write downwards data in online mode") unless Offroad.app_online?
|
32
|
+
write_data(tgt) do |cs|
|
33
|
+
add_global_cargo(cs)
|
34
|
+
if @initial_mode
|
35
|
+
add_group_specific_cargo(cs)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def load_upwards_data(src)
|
41
|
+
raise PluginError.new("Can only load upwards data in online mode") unless Offroad.app_online?
|
42
|
+
raise PluginError.new("No such thing as initial upwards data") if @initial_mode
|
43
|
+
|
44
|
+
read_data_from("offline", src) do |cs, mirror_info, cargo_group_state|
|
45
|
+
unless cargo_group_state.confirmed_group_data_version > @group.group_state.confirmed_group_data_version
|
46
|
+
raise OldDataError.new("File contains old up-mirror data")
|
47
|
+
end
|
48
|
+
import_group_specific_cargo(cs)
|
49
|
+
@group.group_offline = false if cargo_group_state.group_locked?
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def load_downwards_data(src)
|
54
|
+
raise PluginError.new("Can only load downwards data in offline mode") unless Offroad.app_offline?
|
55
|
+
|
56
|
+
read_data_from("online", src) do |cs, mirror_info, cargo_group_state|
|
57
|
+
raise DataError.new("Unexpected initial file value") unless mirror_info.initial_file == @initial_mode
|
58
|
+
|
59
|
+
group_cargo_name = MirrorData::data_cargo_name_for_model(Offroad::group_base_model)
|
60
|
+
if mirror_info.initial_file
|
61
|
+
raise DataError.new("No group data in initial down mirror file") unless cs.has_cargo_named?(group_cargo_name)
|
62
|
+
# This is an initial mirror file, so we want it to determine the entirety of the database's new state
|
63
|
+
# However, existing data is safe if there's a mid-import error; read_data_from places us in a transaction
|
64
|
+
delete_all_existing_database_records!
|
65
|
+
|
66
|
+
import_global_cargo(cs) # Global cargo must be done first because group data might belong_to global data
|
67
|
+
import_group_specific_cargo(cs)
|
68
|
+
else
|
69
|
+
# Regular, non-initial down mirror file
|
70
|
+
unless cargo_group_state.confirmed_global_data_version > @group.group_state.confirmed_global_data_version
|
71
|
+
raise OldDataError.new("File contains old down-mirror data")
|
72
|
+
end
|
73
|
+
import_global_cargo(cs)
|
74
|
+
end
|
75
|
+
|
76
|
+
# Load information into our group state that the online app is in a better position to know about
|
77
|
+
@group = Offroad::offline_group if @initial_mode
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def self.data_cargo_name_for_model(model)
|
84
|
+
"data_#{model.name}"
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.deletion_cargo_name_for_model(model)
|
88
|
+
"deletion_#{model.name}"
|
89
|
+
end
|
90
|
+
|
91
|
+
def delete_all_existing_database_records!
|
92
|
+
tables = ActiveRecord::Base.connection.tables
|
93
|
+
if ActiveRecord::Base.connection.adapter_name.downcase.include?("sqlite")
|
94
|
+
# Emptying sqlite_sequence resets SQLite's autoincrement counters.
|
95
|
+
# SQLite's autoincrement is nice in that it automatically picks largest ever id + 1.
|
96
|
+
# This means that after clearing sqlite_sequence and then populating database with manually-id'd rows,
|
97
|
+
# new records will be inserted with unique id's, no problem.
|
98
|
+
tables << "sqlite_sequence"
|
99
|
+
end
|
100
|
+
|
101
|
+
tables.each do |table|
|
102
|
+
next if table.start_with?("virtual_") # Used in testing # FIXME Should pick something less likely to collide with app name
|
103
|
+
next if table == "schema_migrations"
|
104
|
+
ActiveRecord::Base.connection.execute "DELETE FROM #{table}"
|
105
|
+
end
|
106
|
+
|
107
|
+
if ActiveRecord::Base.connection.adapter_name.downcase.include?("postgres")
|
108
|
+
# Reset all sequences so that autoincremented ids start from 1 again
|
109
|
+
seqnames = ActiveRecord::Base.connection.select_values "SELECT c.relname FROM pg_class c WHERE c.relkind = 'S'"
|
110
|
+
seqnames.each do |s|
|
111
|
+
ActiveRecord::Base.connection.execute "SELECT setval('#{s}', 1, false)"
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def write_data(tgt)
|
117
|
+
cs = nil
|
118
|
+
temp_sio = nil
|
119
|
+
case tgt
|
120
|
+
when CargoStreamer
|
121
|
+
cs = tgt
|
122
|
+
when nil
|
123
|
+
temp_sio = StringIO.new("", "w")
|
124
|
+
cs = CargoStreamer.new(temp_sio, "w")
|
125
|
+
else
|
126
|
+
cs = CargoStreamer.new(tgt, "w")
|
127
|
+
end
|
128
|
+
|
129
|
+
# TODO: Figure out if this transaction ensures we get a consistent read state
|
130
|
+
Offroad::group_base_model.connection.transaction do
|
131
|
+
Offroad::group_base_model.cache do
|
132
|
+
begin
|
133
|
+
mirror_info = MirrorInfo.new_from_group(@group, @initial_mode)
|
134
|
+
cs.write_cargo_section("mirror_info", [mirror_info], :human_readable => true)
|
135
|
+
|
136
|
+
group_state = @group.group_state
|
137
|
+
if Offroad::app_online?
|
138
|
+
# Let the offline app know what global data version it's being updated to
|
139
|
+
group_state.confirmed_global_data_version = SystemState::current_mirror_version
|
140
|
+
else
|
141
|
+
# Let the online app know what group data version the online mirror of this group is being updated to
|
142
|
+
group_state.confirmed_group_data_version = SystemState::current_mirror_version
|
143
|
+
end
|
144
|
+
cs.write_cargo_section("group_state", [group_state], :human_readable => true)
|
145
|
+
|
146
|
+
yield cs
|
147
|
+
|
148
|
+
SystemState::increment_mirror_version
|
149
|
+
rescue Offroad::CargoStreamerError
|
150
|
+
raise Offroad::DataError.new("Encountered data validation error while writing to cargo file")
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
return temp_sio.string if temp_sio
|
156
|
+
end
|
157
|
+
|
158
|
+
def read_data_from(expected_source_app_mode, src)
|
159
|
+
cs = case src
|
160
|
+
when CargoStreamer then src
|
161
|
+
when String then CargoStreamer.new(StringIO.new(src, "r"), "r")
|
162
|
+
else CargoStreamer.new(src, "r")
|
163
|
+
end
|
164
|
+
|
165
|
+
raise DataError.new("Invalid mirror file, no info section found") unless cs.has_cargo_named?("mirror_info")
|
166
|
+
mirror_info = cs.first_cargo_element("mirror_info")
|
167
|
+
raise DataError.new("Invalid info section type") unless mirror_info.is_a?(MirrorInfo)
|
168
|
+
unless mirror_info.app_mode.downcase == expected_source_app_mode.downcase
|
169
|
+
raise DataError.new "Mirror file was generated by app in wrong mode; was expecting #{expected_source_app_mode}"
|
170
|
+
end
|
171
|
+
|
172
|
+
raise DataError.new("Invalid mirror file, no group state found") unless cs.has_cargo_named?("group_state")
|
173
|
+
group_state = cs.first_cargo_element("group_state")
|
174
|
+
raise DataError.new("Invalid group state type") unless group_state.is_a?(GroupState)
|
175
|
+
group_state.readonly!
|
176
|
+
|
177
|
+
# FIXME: Is this transaction call helping at all?
|
178
|
+
Offroad::group_base_model.connection.transaction do
|
179
|
+
Offroad::group_base_model.cache do
|
180
|
+
yield cs, mirror_info, group_state
|
181
|
+
validate_imported_models(cs) unless @skip_validation
|
182
|
+
|
183
|
+
# Load information into our group state that the remote app is in a better position to know about
|
184
|
+
@group.group_state.update_from_remote_group_state!(group_state) if @group && @group.group_offline?
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
SystemState::increment_mirror_version if @initial_mode
|
189
|
+
end
|
190
|
+
|
191
|
+
def add_group_specific_cargo(cs)
|
192
|
+
Offroad::group_owned_models.each do |name, model|
|
193
|
+
add_model_cargo(cs, model)
|
194
|
+
end
|
195
|
+
Offroad::group_single_models.each do |name, model|
|
196
|
+
add_model_cargo(cs, model)
|
197
|
+
end
|
198
|
+
add_model_cargo(cs, Offroad::group_base_model)
|
199
|
+
end
|
200
|
+
|
201
|
+
def add_global_cargo(cs)
|
202
|
+
Offroad::global_data_models.each do |name, model|
|
203
|
+
add_model_cargo(cs, model)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
def add_model_cargo(cs, model)
|
208
|
+
if @initial_mode
|
209
|
+
add_initial_model_cargo(cs, model)
|
210
|
+
else
|
211
|
+
add_non_initial_model_cargo(cs, model)
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
def add_initial_model_cargo(cs, model)
|
216
|
+
# Include the data for relevant records in this model
|
217
|
+
data_source = model
|
218
|
+
data_source = data_source.owned_by_offroad_group(@group) if model.offroad_group_data? && @group
|
219
|
+
data_source.find_in_batches(:batch_size => 100) do |batch|
|
220
|
+
cs.write_cargo_section(
|
221
|
+
MirrorData::data_cargo_name_for_model(model),
|
222
|
+
batch,
|
223
|
+
:skip_validation => @skip_validation
|
224
|
+
)
|
225
|
+
|
226
|
+
if model.offroad_group_data?
|
227
|
+
# In initial mode the remote app will create records with the same id's as the corresponding records here
|
228
|
+
# So we'll create RRSes indicating that we've already "received" the data we're about to send
|
229
|
+
# Later when the remote app sends new information on those records, we'll know which ones it means
|
230
|
+
rrs_source = Offroad::ReceivedRecordState.for_model_and_group_if_apropos(model, @group)
|
231
|
+
existing_rrs = rrs_source.all(:conditions => {:remote_record_id => batch.map(&:id)}).index_by(&:remote_record_id)
|
232
|
+
new_rrs = batch.reject{|r| existing_rrs.has_key?(r.id)}.map{|r| rrs_source.for_record(r).new(:remote_record_id => r.id)}
|
233
|
+
if new_rrs.size > 0
|
234
|
+
Offroad::ReceivedRecordState.import(new_rrs, :validate => false, :timestamps => false)
|
235
|
+
end
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
def add_non_initial_model_cargo(cs, model)
|
241
|
+
# Include the data for relevant records in this model that are newer than the remote side's known latest version
|
242
|
+
gs = @group.group_state
|
243
|
+
remote_version = nil
|
244
|
+
if model.offroad_group_data?
|
245
|
+
remote_version = gs.confirmed_group_data_version
|
246
|
+
else
|
247
|
+
remote_version = gs.confirmed_global_data_version
|
248
|
+
end
|
249
|
+
srs_source = SendableRecordState.for_model(model).with_version_greater_than(remote_version)
|
250
|
+
srs_source.for_non_deleted_records.find_in_batches(:batch_size => 100) do |srs_batch|
|
251
|
+
# TODO Might be able to optimize this to one query using a join on app model and SRS tables
|
252
|
+
record_ids = srs_batch.map { |srs| srs.local_record_id }
|
253
|
+
data_batch = model.find(:all, :conditions => {:id => record_ids})
|
254
|
+
raise PluginError.new("Invalid SRS ids") if data_batch.size != srs_batch.size
|
255
|
+
cs.write_cargo_section(
|
256
|
+
MirrorData::data_cargo_name_for_model(model),
|
257
|
+
data_batch,
|
258
|
+
:skip_validation => @skip_validation
|
259
|
+
)
|
260
|
+
end
|
261
|
+
|
262
|
+
# Also need to include information about records that have been destroyed
|
263
|
+
srs_source.for_deleted_records.find_in_batches(:batch_size => 100) do |deletion_batch|
|
264
|
+
cs.write_cargo_section(MirrorData::deletion_cargo_name_for_model(model), deletion_batch)
|
265
|
+
end
|
266
|
+
end
|
267
|
+
|
268
|
+
def import_group_specific_cargo(cs)
|
269
|
+
import_model_cargo(cs, Offroad::group_base_model)
|
270
|
+
Offroad::group_owned_models.each do |name, model|
|
271
|
+
import_model_cargo(cs, model)
|
272
|
+
end
|
273
|
+
Offroad::group_single_models.each do |name, model|
|
274
|
+
import_model_cargo(cs, model)
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
def import_global_cargo(cs)
|
279
|
+
Offroad::global_data_models.each do |name, model|
|
280
|
+
import_model_cargo(cs, model)
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
def import_model_cargo(cs, model)
|
285
|
+
@imported_models_to_validate.push model
|
286
|
+
|
287
|
+
if @initial_mode && model.offroad_group_data?
|
288
|
+
import_initial_model_cargo(cs, model)
|
289
|
+
else
|
290
|
+
import_non_initial_model_cargo(cs, model)
|
291
|
+
end
|
292
|
+
end
|
293
|
+
|
294
|
+
def import_initial_model_cargo(cs, model)
|
295
|
+
cs.each_cargo_section(MirrorData::data_cargo_name_for_model(model)) do |batch|
|
296
|
+
# Notice we are using the same primary key values as the online system, not allocating new ones
|
297
|
+
model.import batch, :validate => false, :timestamps => false
|
298
|
+
if model.offroad_group_base? && batch.size > 0
|
299
|
+
GroupState.for_group(model.first).create!
|
300
|
+
end
|
301
|
+
SendableRecordState.setup_imported(model, batch)
|
302
|
+
if model.instance_methods.include?("after_offroad_upload")
|
303
|
+
batch.each { |rec| rec.after_offroad_upload }
|
304
|
+
end
|
305
|
+
end
|
306
|
+
if ActiveRecord::Base.connection.adapter_name.downcase.include?("postgres")
|
307
|
+
# Need to adjust the sequences so that records inserted from this point on don't collide with existing ids
|
308
|
+
cols = ActiveRecord::Base.connection.select_rows "select table_name, column_name, column_default from information_schema.columns WHERE column_default like 'nextval%'"
|
309
|
+
cols.each do |table_name, column_name, column_default|
|
310
|
+
if column_default =~ /nextval\('(.+)'(?:::.+)?\)/
|
311
|
+
seqname = $1
|
312
|
+
ActiveRecord::Base.connection.execute "SELECT setval('#{seqname}', (SELECT MAX(\"#{column_name}\") FROM \"#{table_name}\"))"
|
313
|
+
end
|
314
|
+
end
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
318
|
+
def import_non_initial_model_cargo(cs, model)
|
319
|
+
rrs_source = ReceivedRecordState.for_model_and_group_if_apropos(model, @group)
|
320
|
+
|
321
|
+
# Update/create records
|
322
|
+
cs.each_cargo_section(MirrorData::data_cargo_name_for_model(model)) do |batch|
|
323
|
+
# Update foreign key associations to use local ids instead of remote ids
|
324
|
+
model.reflect_on_all_associations(:belongs_to).each do |a|
|
325
|
+
ReceivedRecordState.redirect_to_local_ids(batch, a.primary_key_name, a.klass, @group)
|
326
|
+
end
|
327
|
+
|
328
|
+
# Delete existing records in the database; that way we can just do INSERTs, don't have to worry about UPDATEs
|
329
|
+
# TODO: Is this necessary? Perhaps ar-extensions can deal with a mix of new and updated records...
|
330
|
+
model.delete rrs_source.all(:conditions => {:remote_record_id => batch.map(&:id)} ).map(&:local_record_id)
|
331
|
+
|
332
|
+
# Update the primary keys to use local ids, then insert the records
|
333
|
+
ReceivedRecordState.redirect_to_local_ids(batch, model.primary_key, model, @group)
|
334
|
+
model.import batch, :validate => false, :timestamps => false
|
335
|
+
|
336
|
+
if model.instance_methods.include?("after_offroad_upload")
|
337
|
+
batch.each { |rec| rec.after_offroad_upload }
|
338
|
+
end
|
339
|
+
end
|
340
|
+
|
341
|
+
# Delete records here which were destroyed there (except for group_base records, that would cause trouble)
|
342
|
+
return if model == Offroad::group_base_model
|
343
|
+
cs.each_cargo_section(MirrorData::deletion_cargo_name_for_model(model)) do |batch|
|
344
|
+
# If there's a callback, we need to load the local records before deleting them
|
345
|
+
local_recs = []
|
346
|
+
if model.instance_methods.include?("after_offroad_destroy")
|
347
|
+
local_recs = model.all(:conditions => {:id => batch.map(&:local_record_id)})
|
348
|
+
end
|
349
|
+
|
350
|
+
# Each deletion batch is made up of SendableRecordStates from the remote system
|
351
|
+
dying_rrs_batch = rrs_source.all(:conditions => {:remote_record_id => batch.map(&:local_record_id)})
|
352
|
+
model.delete dying_rrs_batch.map(&:local_record_id)
|
353
|
+
ReceivedRecordState.delete dying_rrs_batch.map(&:id)
|
354
|
+
local_recs.each { |rec| rec.after_offroad_destroy }
|
355
|
+
end
|
356
|
+
end
|
357
|
+
|
358
|
+
def validate_imported_models(cs)
|
359
|
+
Offroad::group_base_model.connection.clear_query_cache
|
360
|
+
while @imported_models_to_validate.size > 0
|
361
|
+
model = @imported_models_to_validate.pop
|
362
|
+
rrs_source = Offroad::ReceivedRecordState.for_model_and_group_if_apropos(model, @group)
|
363
|
+
|
364
|
+
cs.each_cargo_section(MirrorData::data_cargo_name_for_model(model)) do |cargo_batch|
|
365
|
+
if @initial_mode
|
366
|
+
local_batch = model.all(:conditions => {:id => cargo_batch.map(&:id)})
|
367
|
+
else
|
368
|
+
local_rrs_batch = rrs_source.all(:conditions => {:remote_record_id => cargo_batch.map(&:id)})
|
369
|
+
local_batch = model.all(:conditions => {:id => local_rrs_batch.map(&:local_record_id)})
|
370
|
+
end
|
371
|
+
raise Offroad::DataError.new("Invalid record found in mirror data") unless local_batch.all?(&:valid?)
|
372
|
+
end
|
373
|
+
end
|
374
|
+
end
|
375
|
+
end
|
376
|
+
end
|