marty 0.5.15 → 0.5.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c0900603f68de4a2d16347f8cd75cad55f92761
-  data.tar.gz: 0f927363e84dbdb332939f907f2396c766b3ecf2
+  metadata.gz: ee57ec5bc50741eda9ea8b4cc109221e1cf2e57c
+  data.tar.gz: 486efeb1bdde157cf0eff2ce7c0f67e5999727ae
 SHA512:
-  metadata.gz: 620c234afb693a0aea7480e7c299c854f39210256dedf43f6d1e1e0823c9cea7f36b0010dd0b4bffa7d3395375cdc02de3730c9692fda4db5ffc3100bb7d1860
-  data.tar.gz: 632298cbb98def0ddb8698fcb05eb77c37827169a9483d8a0e57a576b31a75c4f6c4359f7e7fb35d9b480314f8533dd820dd7ba02eb68bc8ecc983d5d8d35cda
+  metadata.gz: 44701d2a49b44b9f433e7b32f79d17d1db6d7ea1af47d660721e8f678278a919bf1f5482d6c6321ee5feef7a7b0dff751f2a50713d968a98c9adc4165091ecf8
+  data.tar.gz: c16b0685d1a63cd10afd0e3b6b8fefa0336c792bceba6df7946e9ed11a80c5833032728cd866354fe263da42a181d28b03ec11385fe5144ead71cbec16f55a8d

data/.gitignore

@@ -0,0 +1,27 @@
+.bundle/
+log/*.log
+pkg/
+spec/dummy/db/*.sqlite3
+spec/dummy/log/*.log
+spec/dummy/tmp/
+spec/dummy/.sass-cache
+
+# Ignore schema
+/db/schema.rb
+/spec/dummy/db/schema.rb
+/spec/dummy/config/database.yml
+
+# Ignore all logfiles and tempfiles.
+/log/*.log
+/spec/dummy/log/*.log
+/tmp
+.rvmrc
+.ruby-version
+.ruby-gemset
+*~
+*.komodoproject
+/pkg
+/Gemfile.lock
+*.swp
+
+/spec/dummy/public/extjs

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format progress
+-t ~netzke_testing

data/.travis.yml

@@ -0,0 +1,23 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.2.2
+
+before_script:
+  - cp config/database.yml.travis spec/dummy/config/database.yml
+  - psql -c 'create database travis_ci_test;' -U postgres
+  - bundle exec rake db:migrate
+
+  - sh -e /etc/init.d/xvfb start
+
+  - wget http://cdn.sencha.com/ext/gpl/ext-5.1.0-gpl.zip
+  - unzip -q ext-5.1.0-gpl.zip
+  - mv ext-5.1.0 spec/dummy/public/extjs
+  - ln -s `pwd`/spec/dummy/public/icons/ spec/dummy/public/extjs/icons
+
+script:
+  - export DISPLAY=:99.0
+  - bundle exec rake
+
+addons:
+  postgresql: "9.4"

data/Gemfile

@@ -0,0 +1,23 @@
+source "http://rubygems.org"
+
+# Declare your gem's dependencies in marty.gemspec.
+# Bundler will treat runtime dependencies like base dependencies, and
+# development dependencies will be added by default to the :development group.
+gemspec
+
+gem 'responders'
+gem 'delayed_job_active_record'
+gem 'daemons', '~> 1.1.9'
+
+group :development, :test do
+  gem 'rails', '~> 4.2.1'
+  gem 'pry-rails'
+  gem 'rspec-rails', '~>3.0'
+  gem 'capybara'
+  gem 'selenium-webdriver'
+  gem 'timecop'
+  gem 'database_cleaner'
+  gem 'netzke-core'
+  gem 'netzke-basepack'
+  gem 'netzke-testing' #, path: File.expand_path('../../netzke-testing', __FILE__)
+end

data/INDEPENDENCE_ISSUES.md

@@ -0,0 +1,23 @@
+# Things that may affect how independent Marty can be as a gem:
+
+* User model tests shouldn't rely on `User.last` as an alternate user type.
+
+## Things pulled from Gemini:
+
+* Some of the category models and migrations. This may be OK since they are
+  only used in testing.
+* Some/most/all of the models/gemini/extras. Perhaps these are more appropriate
+  as Marty items anyway.
+* The `DataReport` and `Fields` (only parts) delorean scripts were copied into the dummy job for
+  the data_import spec. Currently Gemini won't be using this version of these scripts and the
+  data_import spec *must* remain in Gemini somewhere so we don't lose test coverage on what's
+  being used in production. This will probably need to remain this way until we have a solution
+  for making Marty be the source for these scripts (plus some of the others coming across).
+  At that time these scripts should be removed from the dummy job and the tests should refer
+  to them in their location in the Marty production code.
+
+  Also, the `Fields` script used here is a small subset of only the stuff needed by `DataReport`.
+  When the Marty-specific scripts are moved from Gemini `Fields will need to be broken into it's
+  Marty-specific and Gemini-specific parts.
+* Along with those Delorean scripts more gemini models were needed (and put into spec/dummy/app/models)
+  Amongst them: `LoanProgram`, `*Type`, etc. They also got migrations.

data/app/components/marty/report_form.rb

@@ -102,10 +102,15 @@ class Marty::ReportForm < Marty::Form
   js_configure do |c|
     # FIXME: Can we use POST instead of get:
     # http://stackoverflow.com/questions/133925/javascript-post-request-like-a-form-submit
-    # arman -- I tried this solution. However, with a POST the
-    # component no longer has the session cookie.  Therefore, we don't
-    # have access to the selected script.  We likely needs to rethink
-    # this whole mechanism.
+    # arman -- I tried this solution. However, with a POST, the
+    # component doesn't get the session cookie.  Therefore, we don't
+    # have access to the selected script.  QQQ: why not just send the
+    # session cookie in the header??  also see -- using XMLHttpRequest
+    # http://stackoverflow.com/questions/9516865/how-to-set-a-header-field-on-post-a-form
+    # http://stackoverflow.com/questions/9713058/sending-post-data-with-a-xmlhttprequest
+    # Or, perhaps use ExtJS Ajax:
+    # http://stackoverflow.com/questions/2917581/how-to-post-json-data-with-extjs
+
     c.on_foreground = <<-JS
     function() {
        var values = this.getForm().getValues();

data/gemini_deprecations.md

@@ -0,0 +1,6 @@
+Things to remove/deprecate from Gemini
+
+- The duplicate tests from the `marty` spec subdirectories:
+  - `lib` - `data_import_spec.rb` needs to remain in Gemini for the time being (see notes in INDEPENDENCE_ISSUES). `data_blame_spec.rb`
+  is Gemini-specific and should be moved into the Gemini specs proper. All other specs should be removed.
+  - `features/javascript` - this doesn't appear to do anything useful and can probably be removed.

data/lib/marty/data_change.rb

@@ -179,6 +179,105 @@ class Marty::DataChange
 
   ######################################################################
 
+  # Given a class and an array of records (hashes), figure out the
+  # differences in the current records for the class and the records.
+  # Produces a result hash with the following format:
+  #
+  # {"only_source" => [...],
+  #  "only_input"  => [...],
+  #  "different"   => [...],
+  #  "same"        => [...]}
+  #
+  # The "only_input" are hashes found in input which were not found in
+  # source.  "same" are input hashes which were found in source and
+  # have identical information for fields provided.  "only_source" is
+  # an array of source objects for which no equivalent was found in
+  # the input data.  "different" denotes the set of objects which were
+  # found in both input and source but differed on some attribute
+  # values.  "different" is an array of hashes.  Array element look
+  # like:
+  #
+  # {"source" => {k1=>v1, k2=>v2, ...},
+  #  "input"  => {k1=>v1, k2=>v2, ...}}
+
+  delorean_fn :diff, sig: [2, 3] do
+    |klass, input_data, ts='infinity'|
+
+    ts = Mcfly.normalize_infinity(ts)
+    keys = Marty::DataConversion.assoc_keys(klass).map(&:to_s).to_set
+
+    only_source, only_input, different, same = [], [], [], []
+    found_sources = Set[]
+
+    input_data.each do
+      |input|
+
+      input_keys = input.keys
+
+      raise "non-String keys in input data" unless
+        input_keys.all? { |x| String === x }
+
+      begin
+        # convert record -- if there's an exception, it's likely that
+        # an association lookup failed => don't have some association
+        # in source.  FIXME: it could be that we get an conversion
+        # error through not finding a non-key association.  Ideally,
+        # if we can find the key in source, we should report this as
+        # "different".
+        conv =
+          Marty::DataConversion.convert_row(klass, input, ts)
+      rescue => exc
+        only_input << input
+        next
+      end
+
+      key_hash = conv.reject { |k, v| !keys.member?(k) }
+
+      source = Marty::DataConversion.find_row(klass, key_hash, ts)
+
+      if !source
+        # lookup of keys failed => don't have this in source
+        only_input << input
+        next
+      end
+
+      found_sources << source
+
+      non_key_hash = conv.reject { |k, v| keys.member?(k) }
+
+      # is source same as converted input?
+      if non_key_hash.all? { |k, v| v == source.send(k) }
+        same << input
+        next
+      end
+
+      source_export = Marty::DataExporter.export_obj(source) % input_keys
+
+      different << [
+        {"_origin_" => "source"} + source_export,
+        {"_origin_" => "input"} + input,
+      ]
+    end
+
+    # now find any live source object which have not been visited
+    query = klass
+
+    query = query.where("obsoleted_dt >= ? AND created_dt < ?", ts, ts) if
+      Mcfly.has_mcfly?(klass)
+
+    query = query.where.not(id: found_sources.map(&:id))
+
+    {
+      "different"   => different,
+      "same"        => same,
+      "only_input"  => only_input,
+      "only_source" => Marty::DataExporter.
+                      do_export_query_result(klass, query),
+    }
+  end
+
+  ######################################################################
+
   # Given a Mcfly class_name, find all of the obsoleted Mcfly objects
   # which are referenced by live (non-obsoleted) class instances.
   delorean_fn :dead_refs, sig: 2 do

data/lib/marty/data_conversion.rb

@@ -58,10 +58,18 @@ class Marty::DataConversion
       # convert them to dates. FIXME: 'infinity' as a date in
       # Rails 3.2 appears to be broken. Setting a date field to
       # 'infinity' sets it to nil.
-      v =~ FLOAT_PAT ? EXCEL_START_DATE + v.to_f :
-        Mcfly.is_infinity(v) ? 'infinity' : v.to_date
+      begin
+        v =~ FLOAT_PAT ? EXCEL_START_DATE + v.to_f :
+          Mcfly.is_infinity(v) ? 'infinity' : v.to_date
+      rescue => exc
+        raise "date conversion failed for #{v}"
+      end
     when :datetime
-      Mcfly.is_infinity(v) ? 'infinity' : v.to_datetime
+      begin
+        Mcfly.is_infinity(v) ? 'infinity' : v.to_datetime
+      rescue => exc
+        raise "datetime conversion failed for #{v}"
+      end
     when :numrange, :int4range, :int8range
       v.to_s
     when :float_array, :json, :jsonb

data/lib/marty/data_exporter.rb

@@ -147,4 +147,13 @@ class Marty::DataExporter
 
     header + qres.map {|obj| export_attrs(klass, obj).flatten(1)}
   end
+
+  # Export a single object to hash -- FIXME: inefficient
+  # implementation
+  def self.export_obj(obj)
+    klass = obj.class
+    headers = export_headers(klass)
+    rec = export_attrs(klass, obj).flatten
+    Hash[ headers.zip(rec) ]
+  end
 end

data/lib/marty/version.rb

@@ -1,3 +1,3 @@
 module Marty
-  VERSION = "0.5.15"
+  VERSION = "0.5.16"
 end

data/marty.gemspec

@@ -0,0 +1,35 @@
+$:.push File.expand_path("../lib", __FILE__)
+
+require "marty/version"
+
+Gem::Specification.new do |s|
+  s.name        = "marty"
+  s.version     = Marty::VERSION
+  s.authors     = [
+                   "Arman Bostani",
+                   "Eric Litwin",
+                   "Brian VanLoo",
+                   "Iliana Toneva",
+                   "Chad Edie",
+                  ]
+  s.email       = ["arman.bostani@pnmac.com"]
+  s.homepage    = "https://github.com/arman000/marty"
+  s.summary     = "A framework for working with versioned data"
+  s.description = s.summary
+  s.files       = `git ls-files`.split($\)
+  s.licenses    = ['MIT']
+
+  s.add_dependency "pg", "~> 0.17"
+
+  s.add_dependency 'netzke-core', '~> 0.12.2'
+  s.add_dependency 'netzke-basepack', '~> 0.12.6'
+
+  s.add_dependency 'axlsx', '2.1.0pre'
+
+  s.add_dependency 'delorean_lang', '~> 0.1'
+  s.add_dependency 'mcfly', '0.0.17'
+
+  s.add_dependency 'coderay'
+  s.add_dependency 'net-ldap'
+  s.add_dependency 'rubyzip'
+end

data/script/rails

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails 3 gems installed from the root of your application.
+
+ENGINE_ROOT = File.expand_path('../..', __FILE__)
+ENGINE_PATH = File.expand_path('../../lib/marty/engine', __FILE__)
+
+require 'rails/all'
+require 'rails/engine/commands'

data/spec/controllers/application_controller_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+module Marty
+  describe ApplicationController do
+    before(:each) do
+      subject.logout_user
+      expect(Marty::User.current).to be_nil
+    end
+
+    describe 'authentication' do
+      it "should allow a registered user to log in" do
+        allow(Rails.configuration.marty).to receive(:auth_source).
+          and_return('local')
+
+        user = Marty::User.try_to_login('marty', 'marty')
+        subject.set_user(user)
+        expect(Marty::User.current).to_not be_nil
+
+        subject.logout_user
+        expect(Marty::User.current).to be_nil
+      end
+
+      it "should allow a registered user to log in when the database " +
+        "is in recovery mode" do
+        allow(Marty::Util).to receive(:db_in_recovery?).and_return(true)
+        allow(Rails.configuration.marty).to receive(:auth_source).
+          and_return('local')
+
+        user = Marty::User.try_to_login('marty', 'marty')
+        subject.set_user(user)
+        expect(Marty::User.current).to_not be_nil
+
+        subject.logout_user
+        expect(Marty::User.current).to be_nil
+      end
+
+      it "should prevent a non-registered user from logging in" do
+        user = Marty::User.try_to_login('unknown_marty', 'invalid_password')
+        expect(user).to be_nil
+        expect(Marty::User.current).to be_nil
+      end
+
+      it "should prevent a non-registered user from logging in when the " +
+        "database is in recovery mode" do
+        allow(Marty::Util).to receive(:db_in_recovery?).and_return(true)
+        user = Marty::User.try_to_login('unknown_marty', 'invalid_password')
+        expect(user).to be_nil
+        expect(Marty::User.current).to be_nil
+      end
+    end
+  end
+end

data/spec/controllers/job_controller_spec.rb

@@ -0,0 +1,226 @@
+require 'spec_helper'
+require 'marty'
+require 'delorean_lang'
+require 'benchmark'
+require 'job_helper'
+
+describe Marty::JobController, slow: true do
+  before(:each) { @routes = Marty::Engine.routes }
+
+  before(:all) do
+    @clean_file = "/tmp/clean_#{Process.pid}.psql"
+    save_clean_db(@clean_file)
+    # transactional fixtures interfere with queueing jobs
+    self.use_transactional_fixtures = false
+
+    # Needed here because shutting transactional fixtures off
+    # means we lose the globally set uesr
+    Mcfly.whodunnit = UserHelpers.system_user
+
+    Marty::Script.load_script_bodies(promise_bodies, Date.today)
+
+    start_delayed_job
+  end
+
+  after(:each) do
+    Marty::Promise.where('parent_id IS NULL').destroy_all
+  end
+
+  after(:all) do
+    restore_clean_db(@clean_file)
+    stop_delayed_job
+  end
+
+  it "should be able to evaluate in the foreground " do
+    engine = Marty::ScriptSet.new.get_engine(NAME_A)
+
+    res = engine.evaluate("Y", "d")
+    expect(res).to eq([
+                       {"z"=>0.875, "a"=>{"b"=>{"e"=>0.125}}},
+                       {"z"=>0.875, "a"=>{"b"=>{"e"=>1.125}}},
+                       {"z"=>0.875, "a"=>{"b"=>{"e"=>2.125}}},
+                      ])
+
+    expect(Marty::Promise.unscoped.where(start_dt: nil).count).to eq 0
+
+    expect {
+      res = engine.evaluate("Y", "d", {"s" => 1})
+      # force res to be evaluated
+      res.to_s
+    }.to raise_error(RuntimeError)
+
+    sleep 5
+
+    expect(Marty::Promise.unscoped.where(start_dt: nil).count).to eq 0
+  end
+
+  it "should be able to run long-running tasks in separate jobs" do
+    engine = Marty::ScriptSet.new.get_engine(NAME_A)
+
+    # NOTE: can't make this too small since the default
+    # Delayed::Worker::sleep_delay is 5 seconds
+    # However - delayed_job initializer config sets this to 1 for test
+    slp = 5
+
+    exp_res = {"d"=>[
+                     {"z"=>slp,"a"=>{"b"=>{"e"=>1-slp}}},
+                     {"z"=>slp,"a"=>{"b"=>{"e"=>2-slp}}},
+                     {"z"=>slp,"a"=>{"b"=>{"e"=>3-slp}}},
+                    ]}
+
+    bench = Benchmark.measure {
+      res = engine.evaluate("Y", "f", {"s" => slp})
+      expect(res).to eq exp_res
+    }
+
+    # If "f" is evaluated in serial fashion, then the process would
+    # take slp*3+ seconds.  Make sure that we had some parallel
+    # behavior.
+    expect(bench.real).to be >= slp
+    expect(bench.real).to be < slp*2
+  end
+
+  it "should be able to run long-running tasks in separate jobs (2)" do
+    engine = Marty::ScriptSet.new.get_engine(NAME_H)
+
+    slp = 5 # hard-coded in script
+    exp_res = {"d"=>[{"a"=>1}, {"a"=>4}]}
+
+    bench = Benchmark.measure {
+      res = engine.background_eval("Y", {}, ["d"])
+
+      expect(res).to eq exp_res
+    }
+
+    expect(bench.real).to be >= slp
+    expect(bench.real).to be < slp*2
+  end
+
+  it "should be to handle non-serializable errors" do
+    engine = Marty::ScriptSet.new.get_engine(NAME_C)
+    engine.background_eval("Z", {"p_title" => NAME_C}, ["result"])
+    sleep 5
+
+    promise = Marty::Promise.find_by_title(NAME_C)
+
+    get 'download', {
+      job_id: promise.id,
+    }
+
+    expect(response.content_type).to eq 'application/json'
+    expect(JSON.parse(response.body).keys.member?("error")).to be true
+  end
+
+  it "promise proxies should be stored lazily (not expanded)" do
+    engine = Marty::ScriptSet.new.get_engine(NAME_E)
+    engine.background_eval("Z", {"p_title" => NAME_E}, ["result"])
+    sleep 5
+
+    promise = Marty::Promise.find_by_title(NAME_E)
+
+    res = promise.result["result"]
+
+    expect(res.length).to eq 6
+
+    # make sure that the contents are promise proxies (i.e. have
+    # __force__) and therefore lazy.
+    res.each { |r| expect(r.respond_to?(:__force__)).to be true }
+  end
+
+  it "should not leave zombie promises when we have exceptions" do
+    engine = Marty::ScriptSet.new.get_engine(NAME_D)
+    engine.background_eval("Z", {"p_title" => NAME_D}, ["result"])
+    sleep 5
+
+    pl = Marty::Promise.unscoped.where(title: NAME_D)
+
+    expect(pl.count).to eq 2
+
+    pl.each { |p|
+      expect(p.result["error"]).not_to eq nil
+      expect(p.end_dt).not_to eq nil
+    }
+  end
+
+  it "should be able to ask controller for job result" do
+    title = "BG RPC"
+    engine = Marty::ScriptSet.new.get_engine(NAME_A)
+    engine.background_eval("Y", {"p_title" => title}, ["d"])
+    sleep 5
+
+    promise = Marty::Promise.find_by_title(title)
+
+    res = {"d"=>[
+                 {"z"=>0.875, "a"=>{"b"=>{"e"=>0.125}}},
+                 {"z"=>0.875, "a"=>{"b"=>{"e"=>1.125}}},
+                 {"z"=>0.875, "a"=>{"b"=>{"e"=>2.125}}},
+                ]}
+
+    expect(promise.result).to eq res
+
+    get 'download', {
+      format: :json,
+      job_id: promise.id,
+    }
+    expect(response.body).to eq res.to_json
+  end
+
+  it "should be able to get zip results" do
+    title = "BG ZIP"
+    engine = Marty::ScriptSet.new.get_engine(NAME_B)
+    engine.background_eval("Z",
+                           {},
+                           ["result", "format", "title"],
+                           )
+    sleep 5
+
+    promise = Marty::Promise.find_by_title(NAME_B)
+
+    expect(promise.result).to eq({
+      "result"=>[{"a"=>1, "b"=>1},{"a"=>2, "b"=>4},{"a"=>3, "b"=>9}],
+      "format"=>"csv",
+      "title"=>"PromiseB",
+    })
+
+    get 'download', {
+      job_id: promise.id,
+    }
+
+    expect_csv = "a,b\r\n1,1\r\n2,4\r\n3,9\r\n"
+    expect(response.body).to eq expect_csv
+    expect(response.content_type).to eq "text/csv"
+
+    get 'download', {
+      job_id: promise.parent_id,
+    }
+
+    expect(response.content_type).to eq "application/zip"
+
+    Zip::InputStream.open(StringIO.new(response.body)) {
+      |io|
+
+      count = 0
+      while (entry = io.get_next_entry)
+        expect(entry.name).to match /PromiseB.*\.csv/
+        expect(io.read).to eq expect_csv
+        count += 1
+      end
+      expect(count).to eq 3
+    }
+  end
+
+  it "should be able to start promises on imported nodes" do
+    engine = Marty::ScriptSet.new.get_engine(NAME_F)
+    res = engine.evaluate("Z", "result", {})
+    expect(res).to eq "x"*10
+  end
+
+  it "promises called from regular node calls should pass parent promise id" do
+    engine = Marty::ScriptSet.new.get_engine(NAME_G)
+    res = engine.evaluate("V", "result", {})
+    expect(res).to eq [123]
+    p1 = Marty::Promise.find_by_title("#{NAME_G}")
+    p2 = Marty::Promise.find_by_title("#{NAME_G}2")
+    expect(p2.parent_id).to eq p1.id
+  end
+end