marty 0.5.15 → 0.5.16

Sign up to get free protection for your applications and to get access to all the features.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +27 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +23 -0
  5. data/Gemfile +23 -0
  6. data/INDEPENDENCE_ISSUES.md +23 -0
  7. data/app/assets/images/marty/.gitkeep +0 -0
  8. data/app/components/marty/report_form.rb +9 -4
  9. data/gemini_deprecations.md +6 -0
  10. data/lib/marty/data_change.rb +99 -0
  11. data/lib/marty/data_conversion.rb +11 -3
  12. data/lib/marty/data_exporter.rb +9 -0
  13. data/lib/marty/version.rb +1 -1
  14. data/marty.gemspec +35 -0
  15. data/script/rails +8 -0
  16. data/spec/controllers/application_controller_spec.rb +52 -0
  17. data/spec/controllers/job_controller_spec.rb +226 -0
  18. data/spec/controllers/rpc_controller_spec.rb +379 -0
  19. data/spec/controllers/rpc_import_spec.rb +45 -0
  20. data/spec/dummy/README.rdoc +261 -0
  21. data/spec/dummy/Rakefile +7 -0
  22. data/spec/dummy/app/assets/javascripts/application.js +15 -0
  23. data/spec/dummy/app/assets/stylesheets/application.css +13 -0
  24. data/spec/dummy/app/controllers/application_controller.rb +3 -0
  25. data/spec/dummy/app/controllers/components_controller.rb +7 -0
  26. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  27. data/spec/dummy/app/mailers/.gitkeep +0 -0
  28. data/spec/dummy/app/models/.gitkeep +0 -0
  29. data/spec/dummy/app/models/gemini/amortization_type.rb +5 -0
  30. data/spec/dummy/app/models/gemini/bud_category.rb +7 -0
  31. data/spec/dummy/app/models/gemini/entity.rb +2 -0
  32. data/spec/dummy/app/models/gemini/extras/data_import.rb +5 -0
  33. data/spec/dummy/app/models/gemini/extras/settlement_import.rb +28 -0
  34. data/spec/dummy/app/models/gemini/fannie_bup.rb +29 -0
  35. data/spec/dummy/app/models/gemini/grouping.rb +8 -0
  36. data/spec/dummy/app/models/gemini/grouping_head_version.rb +14 -0
  37. data/spec/dummy/app/models/gemini/head.rb +7 -0
  38. data/spec/dummy/app/models/gemini/head_version.rb +14 -0
  39. data/spec/dummy/app/models/gemini/helper.rb +44 -0
  40. data/spec/dummy/app/models/gemini/loan_program.rb +11 -0
  41. data/spec/dummy/app/models/gemini/mortgage_type.rb +5 -0
  42. data/spec/dummy/app/models/gemini/simple.rb +6 -0
  43. data/spec/dummy/app/models/gemini/streamline_type.rb +16 -0
  44. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  45. data/spec/dummy/config.ru +4 -0
  46. data/spec/dummy/config/application.rb +82 -0
  47. data/spec/dummy/config/boot.rb +10 -0
  48. data/spec/dummy/config/database.yml.example +10 -0
  49. data/spec/dummy/config/environment.rb +5 -0
  50. data/spec/dummy/config/environments/development.rb +35 -0
  51. data/spec/dummy/config/environments/production.rb +69 -0
  52. data/spec/dummy/config/environments/test.rb +39 -0
  53. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  54. data/spec/dummy/config/initializers/delayed_job.rb +5 -0
  55. data/spec/dummy/config/initializers/inflections.rb +15 -0
  56. data/spec/dummy/config/initializers/mime_types.rb +5 -0
  57. data/spec/dummy/config/initializers/secret_token.rb +7 -0
  58. data/spec/dummy/config/initializers/session_store.rb +8 -0
  59. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  60. data/spec/dummy/config/locales/en.yml +5 -0
  61. data/spec/dummy/config/routes.rb +12 -0
  62. data/spec/dummy/db/migrate/20140801000000_create_groupings.rb +11 -0
  63. data/spec/dummy/db/migrate/20150406171536_create_categories.rb +27 -0
  64. data/spec/dummy/db/migrate/20150408200916_create_loan_programs.rb +26 -0
  65. data/spec/dummy/db/migrate/20150408201429_create_types.rb +21 -0
  66. data/spec/dummy/db/migrate/20150420000001_create_heads.rb +14 -0
  67. data/spec/dummy/db/migrate/20150420000002_create_head_versions.rb +15 -0
  68. data/spec/dummy/db/migrate/20150420000003_create_grouping_head_versions.rb +12 -0
  69. data/spec/dummy/db/migrate/20151023000001_create_simple.rb +12 -0
  70. data/spec/dummy/db/seeds.rb +8 -0
  71. data/spec/dummy/delorean/blame_report.dl +171 -0
  72. data/spec/dummy/delorean/data_report.dl +105 -0
  73. data/spec/dummy/delorean/fields.dl +52 -0
  74. data/spec/dummy/delorean/styles.dl +134 -0
  75. data/spec/dummy/lib/assets/.gitkeep +0 -0
  76. data/spec/dummy/lib/class_list.rb +3 -0
  77. data/spec/dummy/log/.gitkeep +0 -0
  78. data/spec/dummy/public/404.html +26 -0
  79. data/spec/dummy/public/422.html +26 -0
  80. data/spec/dummy/public/500.html +25 -0
  81. data/spec/dummy/public/favicon.ico +0 -0
  82. data/spec/dummy/public/icons/READ.txt +3 -0
  83. data/spec/dummy/public/icons/application_cascade.png +0 -0
  84. data/spec/dummy/public/icons/application_delete.png +0 -0
  85. data/spec/dummy/public/icons/application_put.png +0 -0
  86. data/spec/dummy/public/icons/application_view_detail.png +0 -0
  87. data/spec/dummy/public/icons/arrow_in.png +0 -0
  88. data/spec/dummy/public/icons/arrow_refresh.png +0 -0
  89. data/spec/dummy/public/icons/database_save.png +0 -0
  90. data/spec/dummy/public/icons/door_in.png +0 -0
  91. data/spec/dummy/public/icons/door_out.png +0 -0
  92. data/spec/dummy/public/icons/group.png +0 -0
  93. data/spec/dummy/public/icons/page_lightning.png +0 -0
  94. data/spec/dummy/public/icons/printer.png +0 -0
  95. data/spec/dummy/public/icons/report_disk.png +0 -0
  96. data/spec/dummy/public/icons/report_go.png +0 -0
  97. data/spec/dummy/public/icons/report_magnify.png +0 -0
  98. data/spec/dummy/public/icons/script.png +0 -0
  99. data/spec/dummy/public/icons/script_add.png +0 -0
  100. data/spec/dummy/public/icons/script_go.png +0 -0
  101. data/spec/dummy/public/icons/script_key.png +0 -0
  102. data/spec/dummy/public/icons/table_go.png +0 -0
  103. data/spec/dummy/public/icons/time.png +0 -0
  104. data/spec/dummy/public/icons/time_add.png +0 -0
  105. data/spec/dummy/public/icons/time_go.png +0 -0
  106. data/spec/dummy/public/icons/timeline_marker.png +0 -0
  107. data/spec/dummy/public/icons/user_add.png +0 -0
  108. data/spec/dummy/public/icons/user_delete.png +0 -0
  109. data/spec/dummy/public/icons/user_edit.png +0 -0
  110. data/spec/dummy/public/icons/wrench.png +0 -0
  111. data/spec/dummy/script/delayed_job +6 -0
  112. data/spec/dummy/script/rails +6 -0
  113. data/spec/features/javascripts/job_dashboard_live_search.js.coffee +8 -0
  114. data/spec/features/javascripts/login.js.coffee +8 -0
  115. data/spec/features/jobs_dashboard_netzke_spec.rb +24 -0
  116. data/spec/features/jobs_dashboard_spec.rb +49 -0
  117. data/spec/fixtures/scripts/load_tests/script1.dl +2 -0
  118. data/spec/fixtures/scripts/load_tests/script2.dl +2 -0
  119. data/spec/job_helper.rb +102 -0
  120. data/spec/lib/data_exporter_spec.rb +71 -0
  121. data/spec/lib/data_importer_spec.rb +461 -0
  122. data/spec/lib/xl_spec.rb +198 -0
  123. data/spec/lib/xl_styles_spec.rb +115 -0
  124. data/spec/models/api_auth_spec.rb +187 -0
  125. data/spec/models/posting_spec.rb +107 -0
  126. data/spec/models/promise_spec.rb +65 -0
  127. data/spec/models/script_spec.rb +187 -0
  128. data/spec/models/user_spec.rb +68 -0
  129. data/spec/requests/routes_spec.rb +12 -0
  130. data/spec/spec_helper.rb +61 -0
  131. data/spec/support/clean_db_helpers.rb +18 -0
  132. data/spec/support/delayed_job_helpers.rb +12 -0
  133. data/spec/support/user_helpers.rb +12 -0
  134. metadata +139 -89
  135. data/app/components/marty/auth_app.rb~ +0 -51
  136. data/app/components/marty/auth_app/javascripts/auth_app.js~ +0 -91
  137. data/app/components/marty/cm_form_panel.rb~ +0 -5
  138. data/app/components/marty/cm_grid_panel.rb~ +0 -35
  139. data/app/components/marty/data_import_view.rb~ +0 -142
  140. data/app/components/marty/extras/layout.rb~ +0 -46
  141. data/app/components/marty/live_search_grid_panel.rb~ +0 -49
  142. data/app/components/marty/main_auth_app.rb~ +0 -238
  143. data/app/components/marty/mcfly_grid_panel.rb~ +0 -80
  144. data/app/components/marty/new_posting_form.rb~ +0 -46
  145. data/app/components/marty/new_posting_window.rb~ +0 -21
  146. data/app/components/marty/pivot_grid.rb +0 -52
  147. data/app/components/marty/pivot_grid/endpoints.rb +0 -45
  148. data/app/components/marty/pivot_grid/javascripts/extensions.js +0 -150
  149. data/app/components/marty/pivot_grid/javascripts/pivot_grid.js +0 -86
  150. data/app/components/marty/pivot_grid/services.rb +0 -44
  151. data/app/components/marty/posting_grid.rb~ +0 -140
  152. data/app/components/marty/promise_view.rb~ +0 -157
  153. data/app/components/marty/promise_view/stylesheets/promise_view.css~ +0 -15
  154. data/app/components/marty/report_form.rb~ +0 -217
  155. data/app/components/marty/report_select.rb~ +0 -133
  156. data/app/components/marty/reporting.rb~ +0 -39
  157. data/app/components/marty/script_detail.rb~ +0 -430
  158. data/app/components/marty/script_form.rb~ +0 -233
  159. data/app/components/marty/script_form/javascripts/Ext.ux.form.field.CodeMirror.js~ +0 -909
  160. data/app/components/marty/script_grid.rb~ +0 -99
  161. data/app/components/marty/script_tester.rb~ +0 -213
  162. data/app/components/marty/scripting.rb~ +0 -124
  163. data/app/components/marty/select_report.rb~ +0 -143
  164. data/app/components/marty/simple_app.rb~ +0 -101
  165. data/app/components/marty/tag_grid.rb~ +0 -89
  166. data/app/components/marty/tree_panel.rb~ +0 -256
  167. data/app/components/marty/tree_panel/javascripts/tree_panel.js~ +0 -317
  168. data/app/components/marty/user_pivot.rb +0 -128
  169. data/app/components/marty/user_view.rb~ +0 -188
  170. data/app/controllers/marty/application_controller.rb~ +0 -133
  171. data/app/controllers/marty/components_controller.rb~ +0 -37
  172. data/app/controllers/marty/job_controller.rb~ +0 -28
  173. data/app/controllers/marty/rpc_controller.rb~ +0 -61
  174. data/app/helpers/marty/script_set.rb~ +0 -59
  175. data/app/models/marty/api_auth.rb~ +0 -48
  176. data/app/models/marty/data_change.rb~ +0 -141
  177. data/app/models/marty/enum.rb~ +0 -16
  178. data/app/models/marty/import_type.rb~ +0 -48
  179. data/app/models/marty/poop.rb~ +0 -169
  180. data/app/models/marty/posting.rb~ +0 -86
  181. data/app/models/marty/posting_type.rb~ +0 -21
  182. data/app/models/marty/promise.rb~ +0 -196
  183. data/app/models/marty/role.rb~ +0 -10
  184. data/app/models/marty/script.rb~ +0 -62
  185. data/app/models/marty/tag.rb~ +0 -91
  186. data/app/models/marty/user.rb~ +0 -148
  187. data/app/models/marty/user_role.rb~ +0 -13
  188. data/app/views/layouts/marty/application.html.erb~ +0 -11
  189. data/config/routes.rb~ +0 -10
  190. data/db/migrate/019_create_marty_postings.rb~ +0 -19
  191. data/db/migrate/095_create_marty_tags.rb~ +0 -19
  192. data/lib/marty.rb~ +0 -13
  193. data/lib/marty/content_handler.rb~ +0 -93
  194. data/lib/marty/data_exporter.rb~ +0 -137
  195. data/lib/marty/data_importer.rb~ +0 -114
  196. data/lib/marty/data_row_processor.rb~ +0 -206
  197. data/lib/marty/drop_folder_hook.rb~ +0 -17
  198. data/lib/marty/folder_hook.rb~ +0 -9
  199. data/lib/marty/lazy_column_loader.rb~ +0 -47
  200. data/lib/marty/mcfly_query.rb~ +0 -188
  201. data/lib/marty/migrations.rb~ +0 -65
  202. data/lib/marty/monkey.rb~ +0 -160
  203. data/lib/marty/permissions.rb~ +0 -69
  204. data/lib/marty/promise.rb~ +0 -41
  205. data/lib/marty/promise_job.rb~ +0 -121
  206. data/lib/marty/promise_proxy.rb~ +0 -69
  207. data/lib/marty/util.rb~ +0 -80
  208. data/lib/marty/version.rb~ +0 -3
  209. data/lib/marty/xl.rb~ +0 -526
  210. data/lib/pyxll/README.txt~ +0 -16
  211. data/lib/pyxll/gemini.py~ +0 -110
  212. data/lib/pyxll/pyxll.cfg~ +0 -12
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
4
+ require 'delayed/command'
5
+
6
+ Delayed::Command.new(ARGV).daemonize
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ # This command will automatically be run when you run "rails" with Rails 3 gems installed from the root of your application.
3
+
4
+ APP_PATH = File.expand_path('../../config/application', __FILE__)
5
+ require File.expand_path('../../config/boot', __FILE__)
6
+ require 'rails/commands'
@@ -0,0 +1,8 @@
1
+ describe 'Marty::PromiseView', ->
2
+ it 'sees two jobs then filters down to one when searched', (done) ->
3
+ wait ->
4
+ expect(grid().getStore().getCount()).to.eql 2
5
+ fill textfield('live_search_text'), with: 'marty'
6
+ wait 1000, ->
7
+ expect(grid().getStore().getCount()).to.eql 1
8
+ done()
@@ -0,0 +1,8 @@
1
+ describe 'Marty::AuthApp', ->
2
+ it 'logs the marty user in', (done) ->
3
+ Netzke.page.martyAuthApp.authSpecMode = true
4
+ click button 'Sign in'
5
+ fill textfield('login'), with: 'marty'
6
+ fill textfield('password'), with: 'marty'
7
+ click button 'OK'
8
+ done()
@@ -0,0 +1,24 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'Jobs Dashboard - Netzke::Testing', type: :feature, js: true,
4
+ capybara: true, netzke_testing: true do
5
+ it 'filters out usernames other than the one specified in the search box' do
6
+ other_user = Marty::User.create(login: 'other',
7
+ firstname: 'other',
8
+ lastname: 'other',
9
+ active: true)
10
+ Marty::Promise.create title: "Test Job 1",
11
+ user: Marty::User.find_by(login: 'marty'),
12
+ cformat: 'csv',
13
+ start_dt: Time.now
14
+ Marty::Promise.create title: "Test Job 2",
15
+ user: other_user,
16
+ cformat: 'csv',
17
+ start_dt: Time.now
18
+
19
+ run_mocha_spec 'login', component: 'marty__auth_app'
20
+
21
+ run_mocha_spec 'job_dashboard_live_search',
22
+ component: 'Marty::PromiseView'
23
+ end
24
+ end
@@ -0,0 +1,49 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'Jobs Dashboard', type: :feature, js: true, capybara: true do
4
+ it 'filters out usernames other than the one specified in the search box' do
5
+ other_user = Marty::User.create(login: 'other',
6
+ firstname: 'other',
7
+ lastname: 'other',
8
+ active: true)
9
+ Marty::Promise.create title: "Test Job 1",
10
+ user: Marty::User.find_by(login: 'marty'),
11
+ cformat: 'csv',
12
+ start_dt: Time.now
13
+ Marty::Promise.create title: "Test Job 2",
14
+ user: other_user,
15
+ cformat: 'csv',
16
+ start_dt: Time.now
17
+
18
+ visit "/"
19
+ find(ext_button_id('Sign in')).click
20
+ fill_in 'Login', with: 'marty'
21
+ fill_in 'Password', with: 'marty'
22
+ find(:xpath, '//a[contains(., "OK")]').click
23
+ expect(page).to have_content('marty marty')
24
+
25
+ find(ext_button_id('Applications')).click
26
+ find(ext_menuitem_id('Jobs Dashboard')).click
27
+ page_title = I18n.t("jobs.promise_view")
28
+ expect(page).to have_content(page_title)
29
+ expect(tree_row_count(page_title)).to eq(2)
30
+
31
+ fill_in "live_search_text", with: 'marty'
32
+ sleep 1
33
+ expect(tree_row_count(page_title)).to eq(1)
34
+ end
35
+
36
+ def ext_button_id title
37
+ id = page.evaluate_script("Ext.ComponentQuery.query(\"button{isVisible(true)}[text='#{title}']\")[0].id")
38
+ "##{id}"
39
+ end
40
+
41
+ def ext_menuitem_id title
42
+ id = page.evaluate_script("Ext.ComponentQuery.query(\"menuitem[text='#{title}']\")[0].id")
43
+ "##{id}"
44
+ end
45
+
46
+ def tree_row_count name
47
+ page.evaluate_script("Ext.ComponentQuery.query('treepanel[title=\"#{name}\"]')[0].getStore().getCount()")
48
+ end
49
+ end
@@ -0,0 +1,2 @@
1
+ NodeC:
2
+ attr = 789
@@ -0,0 +1,2 @@
1
+ NodeD:
2
+ attr = 234
@@ -0,0 +1,102 @@
1
+ # monkey patch needed for Zip::InputStream.open_buffer
2
+ class StringIO
3
+ def path
4
+ end
5
+ end
6
+
7
+ NAME_A = "PromiseA2"
8
+ SCRIPT_A = <<EOS
9
+ Y:
10
+ q =? nil
11
+ s =? 0.875
12
+ a = Y(p_title="aaa", q=q, s=s, p_timeout=3) | ['b']
13
+ b = Y(p_title="bbb", q=q, s=s, p_timeout=3) | ['e']
14
+ e = if q == s then ERR("xxx") else q-s
15
+ z = Gemini::Helper.sleep(s) && s
16
+ d = [Y(p_title="z/a %d" % i, q=i, s=s, p_timeout=30) | ['z','a']
17
+ for i in [1, 2, 3]]
18
+ f = Y(s=s) | ["d"]
19
+ EOS
20
+
21
+ NAME_B = "PromiseB"
22
+ SCRIPT_B = <<EOS
23
+ Y:
24
+ result = [{"a": i, "b": i*i} for i in [1,2,3]]
25
+ format = "csv"
26
+ title = "#{NAME_B}"
27
+ Z:
28
+ result = [Y() | ["result", "format", "title"] for i in [1,2,3]]
29
+ format = "zip"
30
+ title = "Root#{NAME_B}"
31
+ EOS
32
+
33
+ NAME_C = "PromiseC"
34
+ SCRIPT_C = <<EOS
35
+ Y:
36
+ node =?
37
+ x = 123
38
+ res = node() | "x"
39
+ Z:
40
+ result = Y(node=Y) | "res"
41
+ title = "#{NAME_C}"
42
+ EOS
43
+
44
+ NAME_D = "PromiseD"
45
+ SCRIPT_D = <<EOS
46
+ Y:
47
+ arg =?
48
+
49
+ Z:
50
+ x = ERR("xxx")
51
+ lazy = Z() | "x"
52
+ result = Y(arg=lazy, p_title="#{NAME_D}") | "arg"
53
+ title = "#{NAME_D}"
54
+ EOS
55
+
56
+ NAME_E = "PromiseE"
57
+ SCRIPT_E = <<EOS
58
+ X:
59
+ x = 'x'*10
60
+ Z:
61
+ result = [X() | "x" for i in [1,2,3,4,5,6]]
62
+ EOS
63
+
64
+ NAME_F = "PromiseF"
65
+ SCRIPT_F = <<EOS
66
+ import #{NAME_E}
67
+ Z:
68
+ result = #{NAME_E}::X() | "x"
69
+ EOS
70
+
71
+ NAME_G = "PromiseG"
72
+ SCRIPT_G = <<EOS
73
+ U:
74
+ result = [123]
75
+ R:
76
+ result = U(p_title="#{NAME_G}2") | "result"
77
+ A:
78
+ result = R().result
79
+ V:
80
+ result = A(p_title="#{NAME_G}") | "result"
81
+ EOS
82
+
83
+ NAME_H = "PromiseH"
84
+ SCRIPT_H = <<EOS
85
+ Y:
86
+ q =? nil
87
+ a = Gemini::Helper.sleep(5) && q*q
88
+ d = [Y(q=i) | ['a'] for i in [1, 2]]
89
+ EOS
90
+
91
+ def promise_bodies
92
+ {
93
+ NAME_A => SCRIPT_A,
94
+ NAME_B => SCRIPT_B,
95
+ NAME_C => SCRIPT_C,
96
+ NAME_D => SCRIPT_D,
97
+ NAME_E => SCRIPT_E,
98
+ NAME_F => SCRIPT_F,
99
+ NAME_G => SCRIPT_G,
100
+ NAME_H => SCRIPT_H,
101
+ }
102
+ end
@@ -0,0 +1,71 @@
1
+ require "spec_helper"
2
+ require "class_list"
3
+
4
+ module Marty
5
+
6
+ groupings =<<EOF
7
+ name
8
+ g1
9
+ g2
10
+ g3
11
+ EOF
12
+
13
+ heads =<<EOF
14
+ name condition_text
15
+ h1 foo
16
+ h2 bar
17
+ h3 baz
18
+ EOF
19
+
20
+ head_versions =<<EOF
21
+ head__name version result_text
22
+ h1 base x=1
23
+ h1 600 x=2
24
+ h2 base y=z
25
+ h3 base z=4
26
+ EOF
27
+
28
+ grouping_head_versions =<<EOF
29
+ grouping__name head_version__head head_version__version
30
+ g1 h1 base
31
+ g1 h1 600
32
+ g2 h2 base
33
+ g3 h3 base
34
+ EOF
35
+
36
+ describe DataExporter do
37
+ it "be able to import and export nested keys" do
38
+ res = Marty::DataImporter.do_import_summary(Gemini::Grouping, groupings)
39
+ expect(res).to eq({create: 3})
40
+ Gemini::Grouping.count.should == 3
41
+
42
+ res = Marty::DataImporter.do_import_summary(Gemini::Head, heads)
43
+ expect(res).to eq({create: 3})
44
+ Gemini::Head.count.should == 3
45
+
46
+ res = Marty::DataImporter.do_import_summary(Gemini::HeadVersion, head_versions)
47
+ expect(res).to eq({create: 4})
48
+ Gemini::HeadVersion.count.should == 4
49
+
50
+ res = Marty::DataImporter.do_import_summary(Gemini::GroupingHeadVersion, grouping_head_versions)
51
+ expect(res).to eq({create: 4})
52
+ Gemini::GroupingHeadVersion.count.should == 4
53
+
54
+ res = Marty::DataExporter.do_export('infinity', Gemini::GroupingHeadVersion)
55
+
56
+ expect(res).to eq [
57
+ ["grouping", "head_version__head", "head_version__version"],
58
+ ["g1", "h1", "base"],
59
+ ["g1", "h1", "600"],
60
+ ["g2", "h2", "base"],
61
+ ["g3", "h3", "base"]
62
+ ]
63
+
64
+ csv = Marty::DataExporter.to_csv(res, col_sep: "\t")
65
+
66
+ res = Marty::DataImporter.do_import_summary(Gemini::GroupingHeadVersion, csv)
67
+
68
+ expect(res).to eq({same: 4})
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,461 @@
1
+ require "spec_helper"
2
+ require "class_list"
3
+
4
+ module Marty
5
+
6
+ bud_cats =<<EOF
7
+ name
8
+ Conv Fixed 30
9
+ Conv Fixed 20
10
+ EOF
11
+
12
+ bud_cats2 =<<EOF
13
+ namex
14
+ Conv Fixed 20
15
+ EOF
16
+
17
+ fannie_bup1 =<<EOF
18
+ bud_category note_rate buy_up buy_down settlement_mm settlement_yy
19
+ Conv Fixed 30 2.250 4.42000 7.24000 12 2012
20
+ Conv Fixed 30 2.375 4.42000 7.24000 12 2012
21
+ Conv Fixed 30 2.500 4.41300 7.22800 12 2012
22
+ Conv Fixed 30 2.625 4.37500 7.16200 12 2012
23
+ Conv Fixed 30 2.750 4.32900 7.09300 12 2012
24
+ Conv Fixed 20 2.875 4.24800 6.95900 12 2012
25
+ EOF
26
+
27
+ fannie_bup1_export =
28
+ [
29
+ ["entity", "bud_category", "note_rate", "settlement_mm",
30
+ "settlement_yy", "buy_up", "buy_down"],
31
+ [nil, "Conv Fixed 30", 2.250, 12, 2012, 4.42, 7.24],
32
+ [nil, "Conv Fixed 30", 2.375, 12, 2012, 4.42, 7.24],
33
+ [nil, "Conv Fixed 30", 2.500, 12, 2012, 4.413, 7.228],
34
+ [nil, "Conv Fixed 30", 2.625, 12, 2012, 4.375, 7.162],
35
+ [nil, "Conv Fixed 30", 2.750, 12, 2012, 4.329, 7.093],
36
+ [nil, "Conv Fixed 20", 2.875, 12, 2012, 4.248, 6.959],
37
+ ]
38
+
39
+ fannie_bup2 =<<EOF
40
+ bud_category note_rate buy_up buy_down settlement_mm settlement_yy
41
+ Conv Fixed 20 2.250 4.42000 7.24000 12 2012
42
+ Conv Fixed 20 2.375 4.42000 7.24000 12 2012
43
+ Conv Fixed 30 2.500 1.111 2.222 12 2012
44
+ Conv Fixed 30 2.625 4.37500 7.16200 12 2012
45
+ \t\t\t\t\t
46
+ Conv Fixed 30 2.750 4.32900 7.09300 12 2012
47
+ \t\t\t\t\t
48
+ Conv Fixed 20 2.875 3.333 4.444 12 2012
49
+ EOF
50
+
51
+ fannie_bup3 =<<EOF
52
+ bud_category note_rate buy_up buy_down settlement_mm settlement_yy
53
+ Conv Fixed 30 2.250 1.123 2.345 12 2012
54
+ EOF
55
+
56
+ loan_programs =<<EOF
57
+ name amortization_type mortgage_type streamline_type high_balance_indicator
58
+ Conv Fixed 30 Year Fixed Conventional Not Streamlined false
59
+ Conv Fixed 30 Year HB Fixed Conventional Not Streamlined true
60
+ Conv Fixed 30 Year DURP <=80 Fixed Conventional DURP false
61
+ Conv Fixed 30 Year DURP <=80 HB Fixed Conventional DURP true
62
+ EOF
63
+
64
+ fannie_bup4 =<<EOF
65
+ loan_program bud_category note_rate buy_up buy_down settlement_mm settlement_yy
66
+ Conv Fixed 30 Year Conv Fixed 30 2.250 1.123 2.345 12 2012
67
+ EOF
68
+
69
+ fannie_bup5 =<<EOF
70
+ bud_category note_rate buy_up buy_down settlement_mm settlement_yy
71
+ Conv Fixed 20 2.250 1.123 2.345 12 2012
72
+ Conv Fixed XX 2.250 1.123 2.345 12 2012
73
+ EOF
74
+
75
+ fannie_bup6 =<<EOF
76
+ bud_category note_rate buy_up buy_down settlement_mm settlement_yy
77
+ Conv Fixed 30 2.250 4.42000 7.24000 12 2012
78
+ Conv Fixed 30 2.375 a123 7.24000 12 2012
79
+ EOF
80
+
81
+ fannie_bup7 =<<EOF
82
+ bud_category note_rate buy_up buy_down settlement_mm settlement_yy
83
+ Conv Fixed 30 $2.250 4.42% 7.24% 12 2012
84
+ Conv Fixed 30 $2.375 4.42% 7.24% 12 2012
85
+ Conv Fixed 30 $2.500 4.41300 7.22800 12 2012
86
+ Conv Fixed 30 $2.625 4.37500 7.16200 12 2012
87
+ Conv Fixed 30 $2.750 4.32900 7.09300 12 2012
88
+ Conv Fixed 20 $2.875 4.24800 6.95900 12 2012
89
+ EOF
90
+
91
+ describe DataImporter do
92
+ it "should be able to import into classes with id as uniqueness" do
93
+ pending("Fix data importer to handle at least group_id as mcfly_uniqueness")
94
+
95
+ res = Marty::DataImporter.
96
+ do_import_summary(Gemini::Simple,
97
+ [{"some_name" => "hello"}])
98
+ res.should == {create: 1}
99
+ res = Marty::DataImporter.
100
+ do_import_summary(Gemini::Simple,
101
+ [{"group_id" => Gemini::Simple.first.group_id, "some_name" => "hello"}])
102
+ res.should == {same: 1}
103
+ end
104
+
105
+ it "should be able to import fannie buyups" do
106
+ res = Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
107
+ res.should == {create: 2}
108
+ Gemini::BudCategory.count.should == 2
109
+
110
+ res = Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
111
+ res.should == {same: 2}
112
+ Gemini::BudCategory.count.should == 2
113
+
114
+ res = Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup1)
115
+ res.should == {create: 6}
116
+ Gemini::FannieBup.count.should == 6
117
+
118
+ # spot-check the import
119
+ bc = Gemini::BudCategory.find_by_name("Conv Fixed 30")
120
+ fb = Gemini::FannieBup.where(bud_category_id: bc.id, note_rate: 2.50).first
121
+ fb.buy_up.should == 4.41300
122
+ fb.buy_down.should == 7.22800
123
+
124
+ res = Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup1)
125
+ res.should == {same: 6}
126
+ Gemini::FannieBup.count.should == 6
127
+
128
+ # dups should raise an error
129
+ dup = fannie_bup1.split("\n")[-1]
130
+ lambda {
131
+ Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup1+dup)
132
+ }.should raise_error(Marty::DataImporterError)
133
+ end
134
+
135
+ it "should be able to use comma separated files" do
136
+ res = Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
137
+ res = Marty::DataImporter.
138
+ do_import_summary(Gemini::FannieBup,
139
+ fannie_bup1.gsub("\t", ","),
140
+ 'infinity',
141
+ nil,
142
+ nil,
143
+ ",",
144
+ )
145
+ res.should == {create: 6}
146
+ Gemini::FannieBup.count.should == 6
147
+ end
148
+
149
+ it "should be all-or-nothing" do
150
+ lambda {
151
+ Marty::DataImporter.
152
+ do_import_summary(Gemini::BudCategory,
153
+ bud_cats+bud_cats.sub(/name\n/, ""))
154
+ }.should raise_error(Marty::DataImporterError)
155
+ Gemini::BudCategory.count.should == 0
156
+ end
157
+
158
+ it "should be able to perform updates mixed with inserts" do
159
+ res = Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
160
+ res = Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup1)
161
+
162
+ res = Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup3)
163
+ res.should == {update: 1}
164
+
165
+ res = Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup2)
166
+ res.should == {same: 2, create: 2, update: 2, blank: 2}
167
+ end
168
+
169
+ it "should be able to import with cleaner" do
170
+ res = Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
171
+ res = Marty::DataImporter.
172
+ do_import_summary(Gemini::FannieBup,
173
+ fannie_bup1,
174
+ 'infinity',
175
+ 'import_cleaner',
176
+ )
177
+ res.should == {create: 6}
178
+
179
+ res = Marty::DataImporter.
180
+ do_import_summary(Gemini::FannieBup,
181
+ fannie_bup1,
182
+ 'infinity',
183
+ 'import_cleaner',
184
+ )
185
+ res.should == {same: 6}
186
+
187
+ res = Marty::DataImporter.
188
+ do_import_summary(Gemini::FannieBup,
189
+ fannie_bup3,
190
+ 'infinity',
191
+ 'import_cleaner',
192
+ )
193
+ res.should == {update: 1, clean: 5}
194
+
195
+ res = Marty::DataImporter.
196
+ do_import_summary(Gemini::FannieBup,
197
+ fannie_bup2,
198
+ 'infinity',
199
+ 'import_cleaner',
200
+ )
201
+ res.should == {create: 6, blank: 2, clean: 1}
202
+ end
203
+
204
+ it "should be able to import with validation" do
205
+ Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
206
+
207
+ # first load some old data
208
+ res = Marty::DataImporter.
209
+ do_import_summary(Gemini::FannieBup,
210
+ fannie_bup1,
211
+ 'infinity',
212
+ )
213
+ res.should == {create: 6}
214
+
215
+ lambda {
216
+ Marty::DataImporter.
217
+ do_import_summary(Gemini::FannieBup,
218
+ fannie_bup1.sub("2012", "2100"), # change 1st row
219
+ 'infinity',
220
+ nil,
221
+ 'import_validation',
222
+ )
223
+ }.should raise_error(Marty::DataImporterError)
224
+
225
+ res = Marty::DataImporter.
226
+ do_import_summary(Gemini::FannieBup,
227
+ fannie_bup1.gsub("2012", "2100"),
228
+ 'infinity',
229
+ nil,
230
+ 'import_validation',
231
+ )
232
+
233
+ res.should == {create: 6}
234
+
235
+ lambda {
236
+ Marty::DataImporter.
237
+ do_import_summary(Gemini::FannieBup,
238
+ fannie_bup3,
239
+ 'infinity',
240
+ 'import_cleaner',
241
+ 'import_validation',
242
+ )
243
+ }.should raise_error(Marty::DataImporterError)
244
+
245
+ res = Marty::DataImporter.
246
+ do_import_summary(Gemini::FannieBup,
247
+ fannie_bup3.gsub("2012", "2100"),
248
+ 'infinity',
249
+ 'import_cleaner',
250
+ 'import_validation',
251
+ )
252
+ res.should == {update: 1, clean: 11}
253
+ end
254
+
255
+ it "should be able to import with preprocess" do
256
+ res = Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
257
+ res = Marty::DataImporter.
258
+ do_import_summary(Gemini::FannieBup,
259
+ fannie_bup7,
260
+ 'infinity',
261
+ 'import_cleaner',
262
+ nil,
263
+ "\t",
264
+ false,
265
+ 'import_preprocess',
266
+ )
267
+ res.should == {create: 6}
268
+ end
269
+
270
+ it "should be able to import with validation - allow prior month" do
271
+ Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
272
+
273
+ # first load some data without any validation
274
+ res = Marty::DataImporter.
275
+ do_import_summary(Gemini::FannieBup, fannie_bup1, 'infinity')
276
+ res.should == {create: 6}
277
+
278
+ now = DateTime.now
279
+ cm, cy = now.month, now.year
280
+ pm1, py1 = (now - 1.month).month, (now - 1.month).year
281
+ pm2, py2 = (now - 2.months).month, (now - 2.months).year
282
+
283
+ # Load data into current mm/yy
284
+ lambda {
285
+ Marty::DataImporter.
286
+ do_import_summary(Gemini::FannieBup,
287
+ fannie_bup1.gsub("12\t2012", "#{cm}\t#{cy}"),
288
+ 'infinity',
289
+ nil,
290
+ 'import_validation',
291
+ )
292
+ }.should_not raise_error
293
+
294
+ # Load data into prior mm/yy - should fail since import_validation
295
+ # only allows current or future months
296
+ lambda {
297
+ Marty::DataImporter.
298
+ do_import_summary(Gemini::FannieBup,
299
+ fannie_bup1.gsub("12\t2012", "#{pm1}\t#{py1}"),
300
+ 'infinity',
301
+ nil,
302
+ 'import_validation',
303
+ )
304
+ }.should raise_error(Marty::DataImporterError)
305
+
306
+ # Load data into prior mm/yy - should not fail since
307
+ # import_validation_allow_prior_month is specified
308
+ lambda {
309
+ Marty::DataImporter.
310
+ do_import_summary(Gemini::FannieBup,
311
+ fannie_bup1.gsub("12\t2012", "#{pm1}\t#{py1}"),
312
+ 'infinity',
313
+ nil,
314
+ 'import_validation_allow_prior_month',
315
+ )
316
+ }.should_not raise_error
317
+
318
+ # Load data into mm/yy more than 1 month prior - should fail even
319
+ # if import_validation_allow_prior_month is specified
320
+ lambda {
321
+ Marty::DataImporter.
322
+ do_import_summary(Gemini::FannieBup,
323
+ fannie_bup1.gsub("12\t2012", "#{pm2}\t#{py2}"),
324
+ 'infinity',
325
+ nil,
326
+ 'import_validation_allow_prior_month',
327
+ )
328
+ }.should raise_error(Marty::DataImporterError)
329
+ end
330
+
331
+ it "should properly handle validation errors" do
332
+ res = Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
333
+ res = Marty::DataImporter.
334
+ do_import_summary(Gemini::LoanProgram, loan_programs)
335
+ res.should == {create: 4}
336
+
337
+ begin
338
+ Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup4)
339
+ rescue Marty::DataImporterError => exc
340
+ exc.lines.should == [0]
341
+ else
342
+ raise "should have had an exception"
343
+ end
344
+ end
345
+
346
+ it "should properly handle cases where an association item is missing" do
347
+ res = Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
348
+
349
+ begin
350
+ Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup5)
351
+ rescue Marty::DataImporterError => exc
352
+ exc.lines.should == [1]
353
+ exc.message.should =~ /Conv Fixed XX/
354
+ else
355
+ raise "should have had an exception"
356
+ end
357
+ end
358
+
359
+ it "should check for bad header" do
360
+ lambda {
361
+ Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats2)
362
+ }.should raise_error(Marty::DataImporterError, /namex/)
363
+ end
364
+
365
+ it "should handle bad data" do
366
+ res = Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
367
+ begin
368
+ res = Marty::DataImporter.
369
+ do_import_summary(Gemini::FannieBup, fannie_bup6)
370
+ rescue Marty::DataImporterError => exc
371
+ exc.lines.should == [1]
372
+ exc.message.should =~ /bad float/
373
+ else
374
+ raise "should have had an exception"
375
+ end
376
+ end
377
+
378
+ it "should be able to export" do
379
+ Marty::Script.load_scripts(nil, Date.today)
380
+ Marty::ScriptSet.clear_cache
381
+ Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
382
+ Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup1)
383
+ p = Marty::Posting.do_create("BASE", DateTime.tomorrow, '?')
384
+
385
+ engine = Marty::ScriptSet.new.get_engine("DataReport")
386
+ res = engine.evaluate("TableReport",
387
+ "result_raw",
388
+ {
389
+ "pt_name" => p.name,
390
+ "class_name" => "Gemini::FannieBup",
391
+ },
392
+ )
393
+ res[0].should == fannie_bup1_export[0]
394
+ res[1..-1].sort.should == fannie_bup1_export[1..-1].sort
395
+ end
396
+ end
397
+
398
+ describe "Blame Report without yml translations" do
399
+ before(:each) do
400
+ I18n.backend.store_translations(:en, {
401
+ attributes: {
402
+ note_rate: nil
403
+ }
404
+ })
405
+ Marty::Script.load_scripts(nil, Date.today)
406
+ Marty::ScriptSet.clear_cache
407
+ p = Marty::Posting.do_create("BASE", DateTime.yesterday, 'yesterday')
408
+ Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
409
+ Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup1)
410
+ p2 = Marty::Posting.do_create("BASE", DateTime.now, 'now is the time')
411
+ engine = Marty::ScriptSet.new.get_engine("BlameReport")
412
+ @res = engine.evaluate("DataBlameReport",
413
+ "result",
414
+ {
415
+ "pt_name1" => p.name,
416
+ "pt_name2" => p2.name
417
+ },
418
+ )
419
+ end
420
+
421
+ context 'when exporting' do
422
+ it "exports the column_name" do
423
+
424
+ expect(@res[0][1][0][1].length).to eq(12)
425
+ expect(@res[0][1][0][1][7]).to eq("note_rate")
426
+ end
427
+ end
428
+ end
429
+
430
+ describe "Blame Report with yml translations" do
431
+ before(:each) do
432
+ I18n.backend.store_translations(:en, {
433
+ attributes: {
434
+ note_rate: "Note Rate"
435
+ }
436
+ })
437
+ Marty::Script.load_scripts(nil, Date.today)
438
+ Marty::ScriptSet.clear_cache
439
+ p = Marty::Posting.do_create("BASE", DateTime.yesterday, 'yesterday')
440
+ Marty::DataImporter.do_import_summary(Gemini::BudCategory, bud_cats)
441
+ Marty::DataImporter.do_import_summary(Gemini::FannieBup, fannie_bup1)
442
+ p2 = Marty::Posting.do_create("BASE", DateTime.now, 'now is the time')
443
+ engine = Marty::ScriptSet.new.get_engine("BlameReport")
444
+ @res = engine.evaluate("DataBlameReport",
445
+ "result",
446
+ {
447
+ "pt_name1" => p.name,
448
+ "pt_name2" => p2.name
449
+ },
450
+ )
451
+ end
452
+
453
+ context 'when exporting' do
454
+ it "exports the locale value for the column header" do
455
+
456
+ expect(@res[0][1][0][1].length).to eq(12)
457
+ expect(@res[0][1][0][1][7]).to eq("Note Rate")
458
+ end
459
+ end
460
+ end
461
+ end