marty 0.5.15 → 0.5.16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +27 -0
- data/.rspec +3 -0
- data/.travis.yml +23 -0
- data/Gemfile +23 -0
- data/INDEPENDENCE_ISSUES.md +23 -0
- data/app/assets/images/marty/.gitkeep +0 -0
- data/app/components/marty/report_form.rb +9 -4
- data/gemini_deprecations.md +6 -0
- data/lib/marty/data_change.rb +99 -0
- data/lib/marty/data_conversion.rb +11 -3
- data/lib/marty/data_exporter.rb +9 -0
- data/lib/marty/version.rb +1 -1
- data/marty.gemspec +35 -0
- data/script/rails +8 -0
- data/spec/controllers/application_controller_spec.rb +52 -0
- data/spec/controllers/job_controller_spec.rb +226 -0
- data/spec/controllers/rpc_controller_spec.rb +379 -0
- data/spec/controllers/rpc_import_spec.rb +45 -0
- data/spec/dummy/README.rdoc +261 -0
- data/spec/dummy/Rakefile +7 -0
- data/spec/dummy/app/assets/javascripts/application.js +15 -0
- data/spec/dummy/app/assets/stylesheets/application.css +13 -0
- data/spec/dummy/app/controllers/application_controller.rb +3 -0
- data/spec/dummy/app/controllers/components_controller.rb +7 -0
- data/spec/dummy/app/helpers/application_helper.rb +2 -0
- data/spec/dummy/app/mailers/.gitkeep +0 -0
- data/spec/dummy/app/models/.gitkeep +0 -0
- data/spec/dummy/app/models/gemini/amortization_type.rb +5 -0
- data/spec/dummy/app/models/gemini/bud_category.rb +7 -0
- data/spec/dummy/app/models/gemini/entity.rb +2 -0
- data/spec/dummy/app/models/gemini/extras/data_import.rb +5 -0
- data/spec/dummy/app/models/gemini/extras/settlement_import.rb +28 -0
- data/spec/dummy/app/models/gemini/fannie_bup.rb +29 -0
- data/spec/dummy/app/models/gemini/grouping.rb +8 -0
- data/spec/dummy/app/models/gemini/grouping_head_version.rb +14 -0
- data/spec/dummy/app/models/gemini/head.rb +7 -0
- data/spec/dummy/app/models/gemini/head_version.rb +14 -0
- data/spec/dummy/app/models/gemini/helper.rb +44 -0
- data/spec/dummy/app/models/gemini/loan_program.rb +11 -0
- data/spec/dummy/app/models/gemini/mortgage_type.rb +5 -0
- data/spec/dummy/app/models/gemini/simple.rb +6 -0
- data/spec/dummy/app/models/gemini/streamline_type.rb +16 -0
- data/spec/dummy/app/views/layouts/application.html.erb +14 -0
- data/spec/dummy/config.ru +4 -0
- data/spec/dummy/config/application.rb +82 -0
- data/spec/dummy/config/boot.rb +10 -0
- data/spec/dummy/config/database.yml.example +10 -0
- data/spec/dummy/config/environment.rb +5 -0
- data/spec/dummy/config/environments/development.rb +35 -0
- data/spec/dummy/config/environments/production.rb +69 -0
- data/spec/dummy/config/environments/test.rb +39 -0
- data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/spec/dummy/config/initializers/delayed_job.rb +5 -0
- data/spec/dummy/config/initializers/inflections.rb +15 -0
- data/spec/dummy/config/initializers/mime_types.rb +5 -0
- data/spec/dummy/config/initializers/secret_token.rb +7 -0
- data/spec/dummy/config/initializers/session_store.rb +8 -0
- data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/spec/dummy/config/locales/en.yml +5 -0
- data/spec/dummy/config/routes.rb +12 -0
- data/spec/dummy/db/migrate/20140801000000_create_groupings.rb +11 -0
- data/spec/dummy/db/migrate/20150406171536_create_categories.rb +27 -0
- data/spec/dummy/db/migrate/20150408200916_create_loan_programs.rb +26 -0
- data/spec/dummy/db/migrate/20150408201429_create_types.rb +21 -0
- data/spec/dummy/db/migrate/20150420000001_create_heads.rb +14 -0
- data/spec/dummy/db/migrate/20150420000002_create_head_versions.rb +15 -0
- data/spec/dummy/db/migrate/20150420000003_create_grouping_head_versions.rb +12 -0
- data/spec/dummy/db/migrate/20151023000001_create_simple.rb +12 -0
- data/spec/dummy/db/seeds.rb +8 -0
- data/spec/dummy/delorean/blame_report.dl +171 -0
- data/spec/dummy/delorean/data_report.dl +105 -0
- data/spec/dummy/delorean/fields.dl +52 -0
- data/spec/dummy/delorean/styles.dl +134 -0
- data/spec/dummy/lib/assets/.gitkeep +0 -0
- data/spec/dummy/lib/class_list.rb +3 -0
- data/spec/dummy/log/.gitkeep +0 -0
- data/spec/dummy/public/404.html +26 -0
- data/spec/dummy/public/422.html +26 -0
- data/spec/dummy/public/500.html +25 -0
- data/spec/dummy/public/favicon.ico +0 -0
- data/spec/dummy/public/icons/READ.txt +3 -0
- data/spec/dummy/public/icons/application_cascade.png +0 -0
- data/spec/dummy/public/icons/application_delete.png +0 -0
- data/spec/dummy/public/icons/application_put.png +0 -0
- data/spec/dummy/public/icons/application_view_detail.png +0 -0
- data/spec/dummy/public/icons/arrow_in.png +0 -0
- data/spec/dummy/public/icons/arrow_refresh.png +0 -0
- data/spec/dummy/public/icons/database_save.png +0 -0
- data/spec/dummy/public/icons/door_in.png +0 -0
- data/spec/dummy/public/icons/door_out.png +0 -0
- data/spec/dummy/public/icons/group.png +0 -0
- data/spec/dummy/public/icons/page_lightning.png +0 -0
- data/spec/dummy/public/icons/printer.png +0 -0
- data/spec/dummy/public/icons/report_disk.png +0 -0
- data/spec/dummy/public/icons/report_go.png +0 -0
- data/spec/dummy/public/icons/report_magnify.png +0 -0
- data/spec/dummy/public/icons/script.png +0 -0
- data/spec/dummy/public/icons/script_add.png +0 -0
- data/spec/dummy/public/icons/script_go.png +0 -0
- data/spec/dummy/public/icons/script_key.png +0 -0
- data/spec/dummy/public/icons/table_go.png +0 -0
- data/spec/dummy/public/icons/time.png +0 -0
- data/spec/dummy/public/icons/time_add.png +0 -0
- data/spec/dummy/public/icons/time_go.png +0 -0
- data/spec/dummy/public/icons/timeline_marker.png +0 -0
- data/spec/dummy/public/icons/user_add.png +0 -0
- data/spec/dummy/public/icons/user_delete.png +0 -0
- data/spec/dummy/public/icons/user_edit.png +0 -0
- data/spec/dummy/public/icons/wrench.png +0 -0
- data/spec/dummy/script/delayed_job +6 -0
- data/spec/dummy/script/rails +6 -0
- data/spec/features/javascripts/job_dashboard_live_search.js.coffee +8 -0
- data/spec/features/javascripts/login.js.coffee +8 -0
- data/spec/features/jobs_dashboard_netzke_spec.rb +24 -0
- data/spec/features/jobs_dashboard_spec.rb +49 -0
- data/spec/fixtures/scripts/load_tests/script1.dl +2 -0
- data/spec/fixtures/scripts/load_tests/script2.dl +2 -0
- data/spec/job_helper.rb +102 -0
- data/spec/lib/data_exporter_spec.rb +71 -0
- data/spec/lib/data_importer_spec.rb +461 -0
- data/spec/lib/xl_spec.rb +198 -0
- data/spec/lib/xl_styles_spec.rb +115 -0
- data/spec/models/api_auth_spec.rb +187 -0
- data/spec/models/posting_spec.rb +107 -0
- data/spec/models/promise_spec.rb +65 -0
- data/spec/models/script_spec.rb +187 -0
- data/spec/models/user_spec.rb +68 -0
- data/spec/requests/routes_spec.rb +12 -0
- data/spec/spec_helper.rb +61 -0
- data/spec/support/clean_db_helpers.rb +18 -0
- data/spec/support/delayed_job_helpers.rb +12 -0
- data/spec/support/user_helpers.rb +12 -0
- metadata +139 -89
- data/app/components/marty/auth_app.rb~ +0 -51
- data/app/components/marty/auth_app/javascripts/auth_app.js~ +0 -91
- data/app/components/marty/cm_form_panel.rb~ +0 -5
- data/app/components/marty/cm_grid_panel.rb~ +0 -35
- data/app/components/marty/data_import_view.rb~ +0 -142
- data/app/components/marty/extras/layout.rb~ +0 -46
- data/app/components/marty/live_search_grid_panel.rb~ +0 -49
- data/app/components/marty/main_auth_app.rb~ +0 -238
- data/app/components/marty/mcfly_grid_panel.rb~ +0 -80
- data/app/components/marty/new_posting_form.rb~ +0 -46
- data/app/components/marty/new_posting_window.rb~ +0 -21
- data/app/components/marty/pivot_grid.rb +0 -52
- data/app/components/marty/pivot_grid/endpoints.rb +0 -45
- data/app/components/marty/pivot_grid/javascripts/extensions.js +0 -150
- data/app/components/marty/pivot_grid/javascripts/pivot_grid.js +0 -86
- data/app/components/marty/pivot_grid/services.rb +0 -44
- data/app/components/marty/posting_grid.rb~ +0 -140
- data/app/components/marty/promise_view.rb~ +0 -157
- data/app/components/marty/promise_view/stylesheets/promise_view.css~ +0 -15
- data/app/components/marty/report_form.rb~ +0 -217
- data/app/components/marty/report_select.rb~ +0 -133
- data/app/components/marty/reporting.rb~ +0 -39
- data/app/components/marty/script_detail.rb~ +0 -430
- data/app/components/marty/script_form.rb~ +0 -233
- data/app/components/marty/script_form/javascripts/Ext.ux.form.field.CodeMirror.js~ +0 -909
- data/app/components/marty/script_grid.rb~ +0 -99
- data/app/components/marty/script_tester.rb~ +0 -213
- data/app/components/marty/scripting.rb~ +0 -124
- data/app/components/marty/select_report.rb~ +0 -143
- data/app/components/marty/simple_app.rb~ +0 -101
- data/app/components/marty/tag_grid.rb~ +0 -89
- data/app/components/marty/tree_panel.rb~ +0 -256
- data/app/components/marty/tree_panel/javascripts/tree_panel.js~ +0 -317
- data/app/components/marty/user_pivot.rb +0 -128
- data/app/components/marty/user_view.rb~ +0 -188
- data/app/controllers/marty/application_controller.rb~ +0 -133
- data/app/controllers/marty/components_controller.rb~ +0 -37
- data/app/controllers/marty/job_controller.rb~ +0 -28
- data/app/controllers/marty/rpc_controller.rb~ +0 -61
- data/app/helpers/marty/script_set.rb~ +0 -59
- data/app/models/marty/api_auth.rb~ +0 -48
- data/app/models/marty/data_change.rb~ +0 -141
- data/app/models/marty/enum.rb~ +0 -16
- data/app/models/marty/import_type.rb~ +0 -48
- data/app/models/marty/poop.rb~ +0 -169
- data/app/models/marty/posting.rb~ +0 -86
- data/app/models/marty/posting_type.rb~ +0 -21
- data/app/models/marty/promise.rb~ +0 -196
- data/app/models/marty/role.rb~ +0 -10
- data/app/models/marty/script.rb~ +0 -62
- data/app/models/marty/tag.rb~ +0 -91
- data/app/models/marty/user.rb~ +0 -148
- data/app/models/marty/user_role.rb~ +0 -13
- data/app/views/layouts/marty/application.html.erb~ +0 -11
- data/config/routes.rb~ +0 -10
- data/db/migrate/019_create_marty_postings.rb~ +0 -19
- data/db/migrate/095_create_marty_tags.rb~ +0 -19
- data/lib/marty.rb~ +0 -13
- data/lib/marty/content_handler.rb~ +0 -93
- data/lib/marty/data_exporter.rb~ +0 -137
- data/lib/marty/data_importer.rb~ +0 -114
- data/lib/marty/data_row_processor.rb~ +0 -206
- data/lib/marty/drop_folder_hook.rb~ +0 -17
- data/lib/marty/folder_hook.rb~ +0 -9
- data/lib/marty/lazy_column_loader.rb~ +0 -47
- data/lib/marty/mcfly_query.rb~ +0 -188
- data/lib/marty/migrations.rb~ +0 -65
- data/lib/marty/monkey.rb~ +0 -160
- data/lib/marty/permissions.rb~ +0 -69
- data/lib/marty/promise.rb~ +0 -41
- data/lib/marty/promise_job.rb~ +0 -121
- data/lib/marty/promise_proxy.rb~ +0 -69
- data/lib/marty/util.rb~ +0 -80
- data/lib/marty/version.rb~ +0 -3
- data/lib/marty/xl.rb~ +0 -526
- data/lib/pyxll/README.txt~ +0 -16
- data/lib/pyxll/gemini.py~ +0 -110
- data/lib/pyxll/pyxll.cfg~ +0 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ee57ec5bc50741eda9ea8b4cc109221e1cf2e57c
|
4
|
+
data.tar.gz: 486efeb1bdde157cf0eff2ce7c0f67e5999727ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 44701d2a49b44b9f433e7b32f79d17d1db6d7ea1af47d660721e8f678278a919bf1f5482d6c6321ee5feef7a7b0dff751f2a50713d968a98c9adc4165091ecf8
|
7
|
+
data.tar.gz: c16b0685d1a63cd10afd0e3b6b8fefa0336c792bceba6df7946e9ed11a80c5833032728cd866354fe263da42a181d28b03ec11385fe5144ead71cbec16f55a8d
|
data/.gitignore
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
.bundle/
|
2
|
+
log/*.log
|
3
|
+
pkg/
|
4
|
+
spec/dummy/db/*.sqlite3
|
5
|
+
spec/dummy/log/*.log
|
6
|
+
spec/dummy/tmp/
|
7
|
+
spec/dummy/.sass-cache
|
8
|
+
|
9
|
+
# Ignore schema
|
10
|
+
/db/schema.rb
|
11
|
+
/spec/dummy/db/schema.rb
|
12
|
+
/spec/dummy/config/database.yml
|
13
|
+
|
14
|
+
# Ignore all logfiles and tempfiles.
|
15
|
+
/log/*.log
|
16
|
+
/spec/dummy/log/*.log
|
17
|
+
/tmp
|
18
|
+
.rvmrc
|
19
|
+
.ruby-version
|
20
|
+
.ruby-gemset
|
21
|
+
*~
|
22
|
+
*.komodoproject
|
23
|
+
/pkg
|
24
|
+
/Gemfile.lock
|
25
|
+
*.swp
|
26
|
+
|
27
|
+
/spec/dummy/public/extjs
|
data/.rspec
ADDED
data/.travis.yml
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
language: ruby
|
2
|
+
rvm:
|
3
|
+
- 1.9.3
|
4
|
+
- 2.2.2
|
5
|
+
|
6
|
+
before_script:
|
7
|
+
- cp config/database.yml.travis spec/dummy/config/database.yml
|
8
|
+
- psql -c 'create database travis_ci_test;' -U postgres
|
9
|
+
- bundle exec rake db:migrate
|
10
|
+
|
11
|
+
- sh -e /etc/init.d/xvfb start
|
12
|
+
|
13
|
+
- wget http://cdn.sencha.com/ext/gpl/ext-5.1.0-gpl.zip
|
14
|
+
- unzip -q ext-5.1.0-gpl.zip
|
15
|
+
- mv ext-5.1.0 spec/dummy/public/extjs
|
16
|
+
- ln -s `pwd`/spec/dummy/public/icons/ spec/dummy/public/extjs/icons
|
17
|
+
|
18
|
+
script:
|
19
|
+
- export DISPLAY=:99.0
|
20
|
+
- bundle exec rake
|
21
|
+
|
22
|
+
addons:
|
23
|
+
postgresql: "9.4"
|
data/Gemfile
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
|
3
|
+
# Declare your gem's dependencies in marty.gemspec.
|
4
|
+
# Bundler will treat runtime dependencies like base dependencies, and
|
5
|
+
# development dependencies will be added by default to the :development group.
|
6
|
+
gemspec
|
7
|
+
|
8
|
+
gem 'responders'
|
9
|
+
gem 'delayed_job_active_record'
|
10
|
+
gem 'daemons', '~> 1.1.9'
|
11
|
+
|
12
|
+
group :development, :test do
|
13
|
+
gem 'rails', '~> 4.2.1'
|
14
|
+
gem 'pry-rails'
|
15
|
+
gem 'rspec-rails', '~>3.0'
|
16
|
+
gem 'capybara'
|
17
|
+
gem 'selenium-webdriver'
|
18
|
+
gem 'timecop'
|
19
|
+
gem 'database_cleaner'
|
20
|
+
gem 'netzke-core'
|
21
|
+
gem 'netzke-basepack'
|
22
|
+
gem 'netzke-testing' #, path: File.expand_path('../../netzke-testing', __FILE__)
|
23
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# Things that may affect how independent Marty can be as a gem:
|
2
|
+
|
3
|
+
* User model tests shouldn't rely on `User.last` as an alternate user type.
|
4
|
+
|
5
|
+
## Things pulled from Gemini:
|
6
|
+
|
7
|
+
* Some of the category models and migrations. This may be OK since they are
|
8
|
+
only used in testing.
|
9
|
+
* Some/most/all of the models/gemini/extras. Perhaps these are more appropriate
|
10
|
+
as Marty items anyway.
|
11
|
+
* The `DataReport` and `Fields` (only parts) delorean scripts were copied into the dummy job for
|
12
|
+
the data_import spec. Currently Gemini won't be using this version of these scripts and the
|
13
|
+
data_import spec *must* remain in Gemini somewhere so we don't lose test coverage on what's
|
14
|
+
being used in production. This will probably need to remain this way until we have a solution
|
15
|
+
for making Marty be the source for these scripts (plus some of the others coming across).
|
16
|
+
At that time these scripts should be removed from the dummy job and the tests should refer
|
17
|
+
to them in their location in the Marty production code.
|
18
|
+
|
19
|
+
Also, the `Fields` script used here is a small subset of only the stuff needed by `DataReport`.
|
20
|
+
When the Marty-specific scripts are moved from Gemini `Fields will need to be broken into it's
|
21
|
+
Marty-specific and Gemini-specific parts.
|
22
|
+
* Along with those Delorean scripts more gemini models were needed (and put into spec/dummy/app/models)
|
23
|
+
Amongst them: `LoanProgram`, `*Type`, etc. They also got migrations.
|
File without changes
|
@@ -102,10 +102,15 @@ class Marty::ReportForm < Marty::Form
|
|
102
102
|
js_configure do |c|
|
103
103
|
# FIXME: Can we use POST instead of get:
|
104
104
|
# http://stackoverflow.com/questions/133925/javascript-post-request-like-a-form-submit
|
105
|
-
# arman -- I tried this solution. However, with a POST the
|
106
|
-
# component
|
107
|
-
# have access to the selected script.
|
108
|
-
#
|
105
|
+
# arman -- I tried this solution. However, with a POST, the
|
106
|
+
# component doesn't get the session cookie. Therefore, we don't
|
107
|
+
# have access to the selected script. QQQ: why not just send the
|
108
|
+
# session cookie in the header?? also see -- using XMLHttpRequest
|
109
|
+
# http://stackoverflow.com/questions/9516865/how-to-set-a-header-field-on-post-a-form
|
110
|
+
# http://stackoverflow.com/questions/9713058/sending-post-data-with-a-xmlhttprequest
|
111
|
+
# Or, perhaps use ExtJS Ajax:
|
112
|
+
# http://stackoverflow.com/questions/2917581/how-to-post-json-data-with-extjs
|
113
|
+
|
109
114
|
c.on_foreground = <<-JS
|
110
115
|
function() {
|
111
116
|
var values = this.getForm().getValues();
|
@@ -0,0 +1,6 @@
|
|
1
|
+
Things to remove/deprecate from Gemini
|
2
|
+
|
3
|
+
- The duplicate tests from the `marty` spec subdirectories:
|
4
|
+
- `lib` - `data_import_spec.rb` needs to remain in Gemini for the time being (see notes in INDEPENDENCE_ISSUES). `data_blame_spec.rb`
|
5
|
+
is Gemini-specific and should be moved into the Gemini specs proper. All other specs should be removed.
|
6
|
+
- `features/javascript` - this doesn't appear to do anything useful and can probably be removed.
|
data/lib/marty/data_change.rb
CHANGED
@@ -179,6 +179,105 @@ class Marty::DataChange
|
|
179
179
|
|
180
180
|
######################################################################
|
181
181
|
|
182
|
+
# Given a class and an array of records (hashes), figure out the
|
183
|
+
# differences in the current records for the class and the records.
|
184
|
+
# Produces a result hash with the following format:
|
185
|
+
#
|
186
|
+
# {"only_source" => [...],
|
187
|
+
# "only_input" => [...],
|
188
|
+
# "different" => [...],
|
189
|
+
# "same" => [...]}
|
190
|
+
#
|
191
|
+
# The "only_input" are hashes found in input which were not found in
|
192
|
+
# source. "same" are input hashes which were found in source and
|
193
|
+
# have identical information for fields provided. "only_source" is
|
194
|
+
# an array of source objects for which no equivalent was found in
|
195
|
+
# the input data. "different" denotes the set of objects which were
|
196
|
+
# found in both input and source but differed on some attribute
|
197
|
+
# values. "different" is an array of hashes. Array element look
|
198
|
+
# like:
|
199
|
+
#
|
200
|
+
# {"source" => {k1=>v1, k2=>v2, ...},
|
201
|
+
# "input" => {k1=>v1, k2=>v2, ...}}
|
202
|
+
|
203
|
+
delorean_fn :diff, sig: [2, 3] do
|
204
|
+
|klass, input_data, ts='infinity'|
|
205
|
+
|
206
|
+
ts = Mcfly.normalize_infinity(ts)
|
207
|
+
keys = Marty::DataConversion.assoc_keys(klass).map(&:to_s).to_set
|
208
|
+
|
209
|
+
only_source, only_input, different, same = [], [], [], []
|
210
|
+
found_sources = Set[]
|
211
|
+
|
212
|
+
input_data.each do
|
213
|
+
|input|
|
214
|
+
|
215
|
+
input_keys = input.keys
|
216
|
+
|
217
|
+
raise "non-String keys in input data" unless
|
218
|
+
input_keys.all? { |x| String === x }
|
219
|
+
|
220
|
+
begin
|
221
|
+
# convert record -- if there's an exception, it's likely that
|
222
|
+
# an association lookup failed => don't have some association
|
223
|
+
# in source. FIXME: it could be that we get an conversion
|
224
|
+
# error through not finding a non-key association. Ideally,
|
225
|
+
# if we can find the key in source, we should report this as
|
226
|
+
# "different".
|
227
|
+
conv =
|
228
|
+
Marty::DataConversion.convert_row(klass, input, ts)
|
229
|
+
rescue => exc
|
230
|
+
only_input << input
|
231
|
+
next
|
232
|
+
end
|
233
|
+
|
234
|
+
key_hash = conv.reject { |k, v| !keys.member?(k) }
|
235
|
+
|
236
|
+
source = Marty::DataConversion.find_row(klass, key_hash, ts)
|
237
|
+
|
238
|
+
if !source
|
239
|
+
# lookup of keys failed => don't have this in source
|
240
|
+
only_input << input
|
241
|
+
next
|
242
|
+
end
|
243
|
+
|
244
|
+
found_sources << source
|
245
|
+
|
246
|
+
non_key_hash = conv.reject { |k, v| keys.member?(k) }
|
247
|
+
|
248
|
+
# is source same as converted input?
|
249
|
+
if non_key_hash.all? { |k, v| v == source.send(k) }
|
250
|
+
same << input
|
251
|
+
next
|
252
|
+
end
|
253
|
+
|
254
|
+
source_export = Marty::DataExporter.export_obj(source) % input_keys
|
255
|
+
|
256
|
+
different << [
|
257
|
+
{"_origin_" => "source"} + source_export,
|
258
|
+
{"_origin_" => "input"} + input,
|
259
|
+
]
|
260
|
+
end
|
261
|
+
|
262
|
+
# now find any live source object which have not been visited
|
263
|
+
query = klass
|
264
|
+
|
265
|
+
query = query.where("obsoleted_dt >= ? AND created_dt < ?", ts, ts) if
|
266
|
+
Mcfly.has_mcfly?(klass)
|
267
|
+
|
268
|
+
query = query.where.not(id: found_sources.map(&:id))
|
269
|
+
|
270
|
+
{
|
271
|
+
"different" => different,
|
272
|
+
"same" => same,
|
273
|
+
"only_input" => only_input,
|
274
|
+
"only_source" => Marty::DataExporter.
|
275
|
+
do_export_query_result(klass, query),
|
276
|
+
}
|
277
|
+
end
|
278
|
+
|
279
|
+
######################################################################
|
280
|
+
|
182
281
|
# Given a Mcfly class_name, find all of the obsoleted Mcfly objects
|
183
282
|
# which are referenced by live (non-obsoleted) class instances.
|
184
283
|
delorean_fn :dead_refs, sig: 2 do
|
@@ -58,10 +58,18 @@ class Marty::DataConversion
|
|
58
58
|
# convert them to dates. FIXME: 'infinity' as a date in
|
59
59
|
# Rails 3.2 appears to be broken. Setting a date field to
|
60
60
|
# 'infinity' sets it to nil.
|
61
|
-
|
62
|
-
|
61
|
+
begin
|
62
|
+
v =~ FLOAT_PAT ? EXCEL_START_DATE + v.to_f :
|
63
|
+
Mcfly.is_infinity(v) ? 'infinity' : v.to_date
|
64
|
+
rescue => exc
|
65
|
+
raise "date conversion failed for #{v}"
|
66
|
+
end
|
63
67
|
when :datetime
|
64
|
-
|
68
|
+
begin
|
69
|
+
Mcfly.is_infinity(v) ? 'infinity' : v.to_datetime
|
70
|
+
rescue => exc
|
71
|
+
raise "datetime conversion failed for #{v}"
|
72
|
+
end
|
65
73
|
when :numrange, :int4range, :int8range
|
66
74
|
v.to_s
|
67
75
|
when :float_array, :json, :jsonb
|
data/lib/marty/data_exporter.rb
CHANGED
@@ -147,4 +147,13 @@ class Marty::DataExporter
|
|
147
147
|
|
148
148
|
header + qres.map {|obj| export_attrs(klass, obj).flatten(1)}
|
149
149
|
end
|
150
|
+
|
151
|
+
# Export a single object to hash -- FIXME: inefficient
|
152
|
+
# implementation
|
153
|
+
def self.export_obj(obj)
|
154
|
+
klass = obj.class
|
155
|
+
headers = export_headers(klass)
|
156
|
+
rec = export_attrs(klass, obj).flatten
|
157
|
+
Hash[ headers.zip(rec) ]
|
158
|
+
end
|
150
159
|
end
|
data/lib/marty/version.rb
CHANGED
data/marty.gemspec
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
$:.push File.expand_path("../lib", __FILE__)
|
2
|
+
|
3
|
+
require "marty/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "marty"
|
7
|
+
s.version = Marty::VERSION
|
8
|
+
s.authors = [
|
9
|
+
"Arman Bostani",
|
10
|
+
"Eric Litwin",
|
11
|
+
"Brian VanLoo",
|
12
|
+
"Iliana Toneva",
|
13
|
+
"Chad Edie",
|
14
|
+
]
|
15
|
+
s.email = ["arman.bostani@pnmac.com"]
|
16
|
+
s.homepage = "https://github.com/arman000/marty"
|
17
|
+
s.summary = "A framework for working with versioned data"
|
18
|
+
s.description = s.summary
|
19
|
+
s.files = `git ls-files`.split($\)
|
20
|
+
s.licenses = ['MIT']
|
21
|
+
|
22
|
+
s.add_dependency "pg", "~> 0.17"
|
23
|
+
|
24
|
+
s.add_dependency 'netzke-core', '~> 0.12.2'
|
25
|
+
s.add_dependency 'netzke-basepack', '~> 0.12.6'
|
26
|
+
|
27
|
+
s.add_dependency 'axlsx', '2.1.0pre'
|
28
|
+
|
29
|
+
s.add_dependency 'delorean_lang', '~> 0.1'
|
30
|
+
s.add_dependency 'mcfly', '0.0.17'
|
31
|
+
|
32
|
+
s.add_dependency 'coderay'
|
33
|
+
s.add_dependency 'net-ldap'
|
34
|
+
s.add_dependency 'rubyzip'
|
35
|
+
end
|
data/script/rails
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# This command will automatically be run when you run "rails" with Rails 3 gems installed from the root of your application.
|
3
|
+
|
4
|
+
ENGINE_ROOT = File.expand_path('../..', __FILE__)
|
5
|
+
ENGINE_PATH = File.expand_path('../../lib/marty/engine', __FILE__)
|
6
|
+
|
7
|
+
require 'rails/all'
|
8
|
+
require 'rails/engine/commands'
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Marty
|
4
|
+
describe ApplicationController do
|
5
|
+
before(:each) do
|
6
|
+
subject.logout_user
|
7
|
+
expect(Marty::User.current).to be_nil
|
8
|
+
end
|
9
|
+
|
10
|
+
describe 'authentication' do
|
11
|
+
it "should allow a registered user to log in" do
|
12
|
+
allow(Rails.configuration.marty).to receive(:auth_source).
|
13
|
+
and_return('local')
|
14
|
+
|
15
|
+
user = Marty::User.try_to_login('marty', 'marty')
|
16
|
+
subject.set_user(user)
|
17
|
+
expect(Marty::User.current).to_not be_nil
|
18
|
+
|
19
|
+
subject.logout_user
|
20
|
+
expect(Marty::User.current).to be_nil
|
21
|
+
end
|
22
|
+
|
23
|
+
it "should allow a registered user to log in when the database " +
|
24
|
+
"is in recovery mode" do
|
25
|
+
allow(Marty::Util).to receive(:db_in_recovery?).and_return(true)
|
26
|
+
allow(Rails.configuration.marty).to receive(:auth_source).
|
27
|
+
and_return('local')
|
28
|
+
|
29
|
+
user = Marty::User.try_to_login('marty', 'marty')
|
30
|
+
subject.set_user(user)
|
31
|
+
expect(Marty::User.current).to_not be_nil
|
32
|
+
|
33
|
+
subject.logout_user
|
34
|
+
expect(Marty::User.current).to be_nil
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should prevent a non-registered user from logging in" do
|
38
|
+
user = Marty::User.try_to_login('unknown_marty', 'invalid_password')
|
39
|
+
expect(user).to be_nil
|
40
|
+
expect(Marty::User.current).to be_nil
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should prevent a non-registered user from logging in when the " +
|
44
|
+
"database is in recovery mode" do
|
45
|
+
allow(Marty::Util).to receive(:db_in_recovery?).and_return(true)
|
46
|
+
user = Marty::User.try_to_login('unknown_marty', 'invalid_password')
|
47
|
+
expect(user).to be_nil
|
48
|
+
expect(Marty::User.current).to be_nil
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,226 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'marty'
|
3
|
+
require 'delorean_lang'
|
4
|
+
require 'benchmark'
|
5
|
+
require 'job_helper'
|
6
|
+
|
7
|
+
describe Marty::JobController, slow: true do
|
8
|
+
before(:each) { @routes = Marty::Engine.routes }
|
9
|
+
|
10
|
+
before(:all) do
|
11
|
+
@clean_file = "/tmp/clean_#{Process.pid}.psql"
|
12
|
+
save_clean_db(@clean_file)
|
13
|
+
# transactional fixtures interfere with queueing jobs
|
14
|
+
self.use_transactional_fixtures = false
|
15
|
+
|
16
|
+
# Needed here because shutting transactional fixtures off
|
17
|
+
# means we lose the globally set uesr
|
18
|
+
Mcfly.whodunnit = UserHelpers.system_user
|
19
|
+
|
20
|
+
Marty::Script.load_script_bodies(promise_bodies, Date.today)
|
21
|
+
|
22
|
+
start_delayed_job
|
23
|
+
end
|
24
|
+
|
25
|
+
after(:each) do
|
26
|
+
Marty::Promise.where('parent_id IS NULL').destroy_all
|
27
|
+
end
|
28
|
+
|
29
|
+
after(:all) do
|
30
|
+
restore_clean_db(@clean_file)
|
31
|
+
stop_delayed_job
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should be able to evaluate in the foreground " do
|
35
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_A)
|
36
|
+
|
37
|
+
res = engine.evaluate("Y", "d")
|
38
|
+
expect(res).to eq([
|
39
|
+
{"z"=>0.875, "a"=>{"b"=>{"e"=>0.125}}},
|
40
|
+
{"z"=>0.875, "a"=>{"b"=>{"e"=>1.125}}},
|
41
|
+
{"z"=>0.875, "a"=>{"b"=>{"e"=>2.125}}},
|
42
|
+
])
|
43
|
+
|
44
|
+
expect(Marty::Promise.unscoped.where(start_dt: nil).count).to eq 0
|
45
|
+
|
46
|
+
expect {
|
47
|
+
res = engine.evaluate("Y", "d", {"s" => 1})
|
48
|
+
# force res to be evaluated
|
49
|
+
res.to_s
|
50
|
+
}.to raise_error(RuntimeError)
|
51
|
+
|
52
|
+
sleep 5
|
53
|
+
|
54
|
+
expect(Marty::Promise.unscoped.where(start_dt: nil).count).to eq 0
|
55
|
+
end
|
56
|
+
|
57
|
+
it "should be able to run long-running tasks in separate jobs" do
|
58
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_A)
|
59
|
+
|
60
|
+
# NOTE: can't make this too small since the default
|
61
|
+
# Delayed::Worker::sleep_delay is 5 seconds
|
62
|
+
# However - delayed_job initializer config sets this to 1 for test
|
63
|
+
slp = 5
|
64
|
+
|
65
|
+
exp_res = {"d"=>[
|
66
|
+
{"z"=>slp,"a"=>{"b"=>{"e"=>1-slp}}},
|
67
|
+
{"z"=>slp,"a"=>{"b"=>{"e"=>2-slp}}},
|
68
|
+
{"z"=>slp,"a"=>{"b"=>{"e"=>3-slp}}},
|
69
|
+
]}
|
70
|
+
|
71
|
+
bench = Benchmark.measure {
|
72
|
+
res = engine.evaluate("Y", "f", {"s" => slp})
|
73
|
+
expect(res).to eq exp_res
|
74
|
+
}
|
75
|
+
|
76
|
+
# If "f" is evaluated in serial fashion, then the process would
|
77
|
+
# take slp*3+ seconds. Make sure that we had some parallel
|
78
|
+
# behavior.
|
79
|
+
expect(bench.real).to be >= slp
|
80
|
+
expect(bench.real).to be < slp*2
|
81
|
+
end
|
82
|
+
|
83
|
+
it "should be able to run long-running tasks in separate jobs (2)" do
|
84
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_H)
|
85
|
+
|
86
|
+
slp = 5 # hard-coded in script
|
87
|
+
exp_res = {"d"=>[{"a"=>1}, {"a"=>4}]}
|
88
|
+
|
89
|
+
bench = Benchmark.measure {
|
90
|
+
res = engine.background_eval("Y", {}, ["d"])
|
91
|
+
|
92
|
+
expect(res).to eq exp_res
|
93
|
+
}
|
94
|
+
|
95
|
+
expect(bench.real).to be >= slp
|
96
|
+
expect(bench.real).to be < slp*2
|
97
|
+
end
|
98
|
+
|
99
|
+
it "should be to handle non-serializable errors" do
|
100
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_C)
|
101
|
+
engine.background_eval("Z", {"p_title" => NAME_C}, ["result"])
|
102
|
+
sleep 5
|
103
|
+
|
104
|
+
promise = Marty::Promise.find_by_title(NAME_C)
|
105
|
+
|
106
|
+
get 'download', {
|
107
|
+
job_id: promise.id,
|
108
|
+
}
|
109
|
+
|
110
|
+
expect(response.content_type).to eq 'application/json'
|
111
|
+
expect(JSON.parse(response.body).keys.member?("error")).to be true
|
112
|
+
end
|
113
|
+
|
114
|
+
it "promise proxies should be stored lazily (not expanded)" do
|
115
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_E)
|
116
|
+
engine.background_eval("Z", {"p_title" => NAME_E}, ["result"])
|
117
|
+
sleep 5
|
118
|
+
|
119
|
+
promise = Marty::Promise.find_by_title(NAME_E)
|
120
|
+
|
121
|
+
res = promise.result["result"]
|
122
|
+
|
123
|
+
expect(res.length).to eq 6
|
124
|
+
|
125
|
+
# make sure that the contents are promise proxies (i.e. have
|
126
|
+
# __force__) and therefore lazy.
|
127
|
+
res.each { |r| expect(r.respond_to?(:__force__)).to be true }
|
128
|
+
end
|
129
|
+
|
130
|
+
it "should not leave zombie promises when we have exceptions" do
|
131
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_D)
|
132
|
+
engine.background_eval("Z", {"p_title" => NAME_D}, ["result"])
|
133
|
+
sleep 5
|
134
|
+
|
135
|
+
pl = Marty::Promise.unscoped.where(title: NAME_D)
|
136
|
+
|
137
|
+
expect(pl.count).to eq 2
|
138
|
+
|
139
|
+
pl.each { |p|
|
140
|
+
expect(p.result["error"]).not_to eq nil
|
141
|
+
expect(p.end_dt).not_to eq nil
|
142
|
+
}
|
143
|
+
end
|
144
|
+
|
145
|
+
it "should be able to ask controller for job result" do
|
146
|
+
title = "BG RPC"
|
147
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_A)
|
148
|
+
engine.background_eval("Y", {"p_title" => title}, ["d"])
|
149
|
+
sleep 5
|
150
|
+
|
151
|
+
promise = Marty::Promise.find_by_title(title)
|
152
|
+
|
153
|
+
res = {"d"=>[
|
154
|
+
{"z"=>0.875, "a"=>{"b"=>{"e"=>0.125}}},
|
155
|
+
{"z"=>0.875, "a"=>{"b"=>{"e"=>1.125}}},
|
156
|
+
{"z"=>0.875, "a"=>{"b"=>{"e"=>2.125}}},
|
157
|
+
]}
|
158
|
+
|
159
|
+
expect(promise.result).to eq res
|
160
|
+
|
161
|
+
get 'download', {
|
162
|
+
format: :json,
|
163
|
+
job_id: promise.id,
|
164
|
+
}
|
165
|
+
expect(response.body).to eq res.to_json
|
166
|
+
end
|
167
|
+
|
168
|
+
it "should be able to get zip results" do
|
169
|
+
title = "BG ZIP"
|
170
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_B)
|
171
|
+
engine.background_eval("Z",
|
172
|
+
{},
|
173
|
+
["result", "format", "title"],
|
174
|
+
)
|
175
|
+
sleep 5
|
176
|
+
|
177
|
+
promise = Marty::Promise.find_by_title(NAME_B)
|
178
|
+
|
179
|
+
expect(promise.result).to eq({
|
180
|
+
"result"=>[{"a"=>1, "b"=>1},{"a"=>2, "b"=>4},{"a"=>3, "b"=>9}],
|
181
|
+
"format"=>"csv",
|
182
|
+
"title"=>"PromiseB",
|
183
|
+
})
|
184
|
+
|
185
|
+
get 'download', {
|
186
|
+
job_id: promise.id,
|
187
|
+
}
|
188
|
+
|
189
|
+
expect_csv = "a,b\r\n1,1\r\n2,4\r\n3,9\r\n"
|
190
|
+
expect(response.body).to eq expect_csv
|
191
|
+
expect(response.content_type).to eq "text/csv"
|
192
|
+
|
193
|
+
get 'download', {
|
194
|
+
job_id: promise.parent_id,
|
195
|
+
}
|
196
|
+
|
197
|
+
expect(response.content_type).to eq "application/zip"
|
198
|
+
|
199
|
+
Zip::InputStream.open(StringIO.new(response.body)) {
|
200
|
+
|io|
|
201
|
+
|
202
|
+
count = 0
|
203
|
+
while (entry = io.get_next_entry)
|
204
|
+
expect(entry.name).to match /PromiseB.*\.csv/
|
205
|
+
expect(io.read).to eq expect_csv
|
206
|
+
count += 1
|
207
|
+
end
|
208
|
+
expect(count).to eq 3
|
209
|
+
}
|
210
|
+
end
|
211
|
+
|
212
|
+
it "should be able to start promises on imported nodes" do
|
213
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_F)
|
214
|
+
res = engine.evaluate("Z", "result", {})
|
215
|
+
expect(res).to eq "x"*10
|
216
|
+
end
|
217
|
+
|
218
|
+
it "promises called from regular node calls should pass parent promise id" do
|
219
|
+
engine = Marty::ScriptSet.new.get_engine(NAME_G)
|
220
|
+
res = engine.evaluate("V", "result", {})
|
221
|
+
expect(res).to eq [123]
|
222
|
+
p1 = Marty::Promise.find_by_title("#{NAME_G}")
|
223
|
+
p2 = Marty::Promise.find_by_title("#{NAME_G}2")
|
224
|
+
expect(p2.parent_id).to eq p1.id
|
225
|
+
end
|
226
|
+
end
|