marty 2.4.5 → 2.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 47d75b731f7458392df1acc9383452faf199aeb2f414fecf70c208f7700fae46
-  data.tar.gz: 1d50e28afa9e48074f4c87d74ce9c23834ce995ffa3c13142bf00a19d4af51b0
+SHA1:
+  metadata.gz: bd09d3bb234821197b1ae5dd78b70e8d07db080f
+  data.tar.gz: 4a2b19ce73da8012c2cd8157661e56fef22d2fd6
 SHA512:
-  metadata.gz: d144bce2db55a1cd1023bf223839da9d5f5570b520926d06951e60a244563d4c5c2eaca8b9dd990bee4dd846895fc6cb05e4f07b92f691970f15cad7a0ff52c7
-  data.tar.gz: 2ce052dae89ccda7d9648d4ce2ea751b226281d7c7bb5ea3db369f52886e8ec83591941f0a58c1122067b81b2456b0b06a39217425b017f47dbfcce297378f8a
+  metadata.gz: 5862a5cc24a94a11a71ba2d649440d6636768b8b18872ae1f1653cfeef588d34d81069486cdb45bf9a2793a22bf9aef172df3bbf9f9e4fad243138b2be04f9b9
+  data.tar.gz: fc3048d404e60b186f3ab9bc49063369f98fd05f30a161ee5736ed116224ec544a3059218e60cf8eacccf1685f7a1f4bfaf15793569fc09d804b731a32e8ac27

data/Dockerfile.dummy

@@ -0,0 +1,40 @@
+# ONLY FOR DEVELOPMENT PURPOSES
+# dockerfile is not suitable to produce a production-grade docker image
+FROM ruby:2.5.1
+
+WORKDIR /opt/app
+
+ADD https://dl.yarnpkg.com/debian/pubkey.gpg /tmp/yarn-pubkey.gpg
+RUN apt-key add /tmp/yarn-pubkey.gpg && rm /tmp/yarn-pubkey.gpg
+RUN echo 'deb http://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list
+
+RUN apt-get update && apt-get install -qq -y --no-install-recommends \
+      build-essential libpq-dev curl
+
+RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -
+RUN apt-get update && apt-get install -qq -y --no-install-recommends nodejs yarn
+
+# Install chrome for integration tests
+# RUN curl -L -o google-chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \
+#     && dpkg -i google-chrome.deb || true \
+#     && apt-get install -f -y \
+#     && sed -i 's|HERE/chrome\"|HERE/chrome\" --disable-setuid-sandbox|g' /opt/google/chrome/google-chrome \
+#     && rm google-chrome.deb
+
+ENV BUNDLE_PATH /bundle_box/bundle
+ENV GEM_HOME /bundle_box/bundle
+ENV PATH /bundle_box/bundle/bin:$PATH
+# ENV NODE_PATH /bundle_box/node
+
+COPY . .
+
+# RUN bundle install
+# RUN yarn install
+
+# RUN mv .rbenv-vars.example .rbenv-vars
+# RUN chmod +x entrypoint.sh
+
+# ENV RAILS_ENV=development
+# ENV PORT=3001
+# EXPOSE 3001
+

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    marty (2.4.4)
+    marty (2.4.7)
       aws-sigv4 (~> 1.0, >= 1.0.2)
       axlsx (= 3.0.0pre)
       coderay

data/Makefile

@@ -0,0 +1,2 @@
+include make-dummy.mk
+

data/README.md

@@ -56,6 +56,22 @@ $ rake marty:delete_scripts
 Make sure that extjs is installed (or symbolically linked) in the
 dummy app at spec/dummy/public.
 
+Docker doesn't support symlinks, so in order to run it in Docker you will have to copy extjs files.
+
+```bash
+$ cp -r PATH/TO/YOUR/EXTJS/FILES spec/dummy/public/extjs
+```
+
+You can run it with docker:
+
+```bash
+$ make dummy-app-initialise-docker
+
+$ make dummy-app-start
+```
+
+To run without docker:
+
 Marty currently only runs with postgresql. To be able to run the tests
 you will first need to create a `database.yml` file in `spec/dummy/config`.
 You can use the example file by doing:

data/app/controllers/marty/delayed_job_controller.rb

@@ -0,0 +1,23 @@
+class Marty::DelayedJobController < ActionController::Base
+  def trigger
+    work_off_job if delayed_job.present?
+    render json: { status: :ok }, status: :ok
+  end
+
+  private
+
+  def delayed_job
+    return unless params['id'].present?
+    @delayed_job ||= ::Delayed::Job.find_by(id: params['id'])
+  end
+
+  def work_off_job
+    return if delayed_job.locked_at.present?
+
+    ::Delayed::Job.where(id: delayed_job.id).
+      update_all(locked_at: ::Delayed::Job.db_time_now, locked_by: 'Lambda')
+
+    w = ::Delayed::Worker.new
+    w.run(delayed_job)
+  end
+end

data/app/controllers/marty/rpc_controller.rb

@@ -25,7 +25,7 @@ class Marty::RpcController < ActionController::Base
       return result = massaged_params if massaged_params.include?(:error)
 
       api_params = api.process_params(massaged_params)
-      auth       = api.is_authorized?(massaged_params)
+      auth       = api.is_authorized?(api_params)
       return result = {error: "Permission denied"} unless auth
 
       start_time = Time.zone.now

data/app/models/marty/delorean_rule.rb

@@ -96,7 +96,8 @@ class Marty::DeloreanRule < Marty::BaseRule
                   result.err_section)
         end
         result.cg_hash = Hash[result.cg_keys.zip(result.cg_vals)]
-        return result.cg_hash.select{|k,v| !v} unless result.cg_vals.all?
+        fails = result.cg_hash.select{|k,v| ![v].flatten.first}
+        return fails if fails.present?
       end
 
       grids_computed = false

data/app/models/marty/promise.rb

@@ -136,6 +136,8 @@ class Marty::Promise < Marty::Base
   end
 
   def wait_for_result(timeout)
+    # FIXME: Not sure that comparing result with empty hash if a good idea
+    # perhaps it's better to use .present? or .blank?
     return self.result if self.result != {}
 
     begin
@@ -148,25 +150,7 @@ class Marty::Promise < Marty::Base
       if !last.start_dt
         job = Marty::Promise.job_by_id(last.job_id)
 
-        # log "AAAA #{Process.pid} #{last} #{job}"
-
-        if !job || job.locked_at
-          # job has been locked, so it looks like it started already
-          # and we need to wait for it.
-          wait_for_my_notify(Marty::Promise::DEFAULT_JOB_TIMEOUT)
-        else
-          # work off the job instead of waiting for a real worker to
-          # pick it up.
-          # log "OFF0 #{Process.pid} #{last}"
-          begin
-            work_off_job(job)
-          rescue => exc
-            # log "OFFERR #{exc}"
-            res = Delorean::Engine.grok_runtime_exception(exc)
-            last.set_result(res)
-          end
-          # log "OFF1 #{Process.pid} #{last}"
-        end
+        wait_for_my_notify(Marty::Promise::DEFAULT_JOB_TIMEOUT)
 
         last = latest
 
@@ -184,19 +168,13 @@ class Marty::Promise < Marty::Base
       # at this point, we know the promise has already started
       if !last.end_dt
         wait_for_my_notify(timeout)
-        # log "UUUU #{Process.pid} #{id} #{Time.now.to_f}"
         last = latest
 
-        # log "XXXX #{Process.pid} #{Time.now.to_f} #{last}"
-
         if !last.end_dt
-          # log "TO22 #{Process.pid} #{last}"
           return {"error" => "promise #{last.id} timed out (didn't end)"}
         end
       end
 
-      # log "RRRR #{Process.pid} #{last} #{Time.now.to_f}"
-
       last.result
     ensure
       # Stop listening to the promise notifications

data/config/routes.rb

@@ -6,5 +6,6 @@ Marty::Engine.routes.draw do
   match via: [:get, :post], "rpc/evaluate(.:format)" => "rpc", as: :rpc
   match via: [:get, :post], "report(.:format)" => "report#index", as: :report
   get  'job/download' => 'job', as: :job
+  post 'delayed_job/trigger' => 'delayed_job#trigger', as: :trigger_delayed_job
   get  'diag', to: 'diagnostic/#op'
 end

data/docker-compose.dummy.yml

@@ -0,0 +1,41 @@
+version: "2"
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile.dummy
+    command: bash -c 'cd spec/dummy && rm -f tmp/pids/server.pid && rails s -b 0.0.0.0'
+    ports:
+      - "3000:3000"
+    environment:
+      - "PORT=3000"
+      - "PATH=/bundle_box/bundle/bin:/usr/local/bundle/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+      - "DEVELOPMENT_DATABASE_URL=postgres://postgres@postgres/marty_dev"
+      - "TEST_DATABASE_URL=postgres://postgres@postgres/marty_test"
+    # env_file: ".rbenv-vars"
+    depends_on:
+      - "postgres"
+    volumes:
+      - .:/opt/app:delegated
+
+      # - '~/.bash_history:/root/.bash_history'
+      # - '~/.pry_history:/root/.pry_history'
+    volumes_from:
+      - bundle_box
+    tty: true
+    stdin_open: true
+
+  postgres:
+    image: "clkao/postgres-plv8:9.6-2.0"
+    volumes:
+      - postgresql-data:/var/lib/postgresql/data
+
+  bundle_box:
+    image: busybox
+    volumes:
+      - /bundle_box
+
+volumes:
+  postgresql-data:
+

data/lib/marty/promise_job.rb

@@ -107,6 +107,17 @@ class Marty::PromiseJob < Struct.new(:promise,
   # def log(msg)
   #   open('/tmp/dj.out', 'a') { |f| f.puts msg }
   # end
+  #
+  def enqueue(job)
+    config = Rails.configuration.marty
+    hooks = config.promise_job_enqueue_hooks
+
+    return if hooks.blank?
+
+    hooks.each do |hook|
+      hook.call(job)
+    end
+  end
 
   def perform
     # log "PERF #{Process.pid} #{title}"

data/lib/marty/railtie.rb

@@ -3,5 +3,6 @@ module Marty
     config.marty = ActiveSupport::OrderedOptions.new
     config.marty.default_posting_type = 'BASE'
     config.marty.extjs_theme = 'classic'
+    config.marty.promise_job_enqueue_hooks = []
   end
 end

data/lib/marty/version.rb

@@ -1,3 +1,3 @@
 module Marty
-  VERSION = "2.4.5"
+  VERSION = "2.4.6"
 end

data/make-dummy.mk

@@ -0,0 +1,42 @@
+dummy-app-build:
+	docker-compose --file=docker-compose.dummy.yml build
+
+dummy-app:
+	docker-compose --file=docker-compose.dummy.yml up
+
+dummy-app-start:
+	docker-compose --file=docker-compose.dummy.yml up app
+
+dummy-app-stop:
+	docker-compose --file=docker-compose.dummy.yml stop
+
+dummy-app-down:
+	docker-compose --file=docker-compose.dummy.yml down
+
+dummy-app-bash:
+	docker-compose --file=docker-compose.dummy.yml run --rm app bash
+
+dummy-app-console:
+	docker-compose --file=docker-compose.dummy.yml run --rm app bin/rails c
+
+dummy-app-initialise-docker:
+	make dummy-app-install
+	make dummy-app-db-prepare
+
+dummy-app-install:
+	make dummy-app-install-bundle
+
+dummy-app-install-bundle:
+	docker-compose --file=docker-compose.dummy.yml run --rm app bundle
+
+dummy-app-db-drop:
+	docker-compose --file=docker-compose.dummy.yml run --rm app rake db:drop
+
+dummy-app-db-prepare:
+	cp spec/dummy/config/database.yml.example spec/dummy/config/database.yml
+	docker-compose --file=docker-compose.dummy.yml run --rm app bundle exec rake db:create db:migrate db:seed app:marty:load_scripts
+
+dummy-app-db-seed:
+	docker-compose --file=docker-compose.dummy.yml run --rm app rake db:seed
+
+

data/spec/controllers/delayed_job_controller_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+require 'marty'
+require 'delorean_lang'
+require 'benchmark'
+require 'job_helper'
+require 'support/empty_job'
+
+describe Marty::DelayedJobController, slow: false do
+  before(:each) { @routes = Marty::Engine.routes }
+
+  describe "#trigger" do
+    before do
+      @job = ::Delayed::Job.enqueue EmptyJob.new
+    end
+
+    it "should be able to execute existing job" do
+      expect(::Delayed::Job.exists?(@job.id)).to be true
+      post :trigger, params: { id: @job.id }
+      expect(response).to have_http_status(:ok)
+      expect(::Delayed::Job.exists?(@job.id)).to be false
+    end
+
+    it "should return ok if triggered job is locked" do
+      @job.update!(locked_at: Time.zone.now)
+      post :trigger, params: { id: @job.id }
+      expect(response).to have_http_status(:ok)
+    end
+
+    it "should return ok if triggered job is missing" do
+      @job.destroy!
+      post :trigger, params: { id: @job.id }
+      expect(response).to have_http_status(:ok)
+    end
+  end
+end

data/spec/dummy/config/database.yml.example

@@ -3,8 +3,10 @@ common: &common
 
 development:
   <<: *common
+  url: <%= ENV['DEVELOPMENT_DATABASE_URL'] %>
   database: marty_dev
 
 test:
   <<: *common
   database: marty_test
+  url: <%= ENV['TEST_DATABASE_URL'] %>

data/spec/fixtures/csv/rule/MyRule.csv

@@ -10,6 +10,6 @@ Rule2,SimpleRule,2017-2-1 14:00:00,2017-4-1,true,"{""g_array"":[""G1V2""],""g_si
 Rule2a,SimpleRule,2017-2-1 14:00:00,2017-4-1,true,"{""g_array"":[""G1V2""],""g_single"":""G2V3"",""g_string"":""abc"",""g_bool"":true,""g_range"":""(,50]"",""g_integer"":99}",,,"{""simple_result"":""\""b value\"""", ""sr2"":""true"", ""sr3"": ""123""}"
 Rule2b,SimpleRule,2017-2-1 14:00:00,2017-4-1,true,"{""g_array"":[""G1V2""],""g_single"":""G2V3"",""g_string"":""abc"",""g_bool"":true,""g_range"":""(,50]"",""g_integer"":999}",,"{""grid1"":""DataGrid1"",""grid2"":""DataGrid2""}",
 Rule2c,SimpleRule,2017-2-1 14:00:00,2017-4-1,true,"{""g_array"":[""G1V2""],""g_single"":""G2V3"",""g_string"":""abc"",""g_bool"":false,""g_range"":""(,50]"",""g_integer"":10}",,"{""grid1"":""DataGrid1"",""grid2"":""DataGrid2""}",
-Rule3,ComplexRule,2017-3-1 00:00:00,2017-4-1,false,"{""g_array"":[""G1V2"",""G1V3""],""g_string"":""def"",""g_integer"":11}","{""cguard1"":""1==1"",""cguard2"":""param2 == 'abc'""}","{""grid1"":""DataGrid1"",""grid2"":""DataGrid2""}","{""simple_result"":""\""c value\"""",""computed_value"":""if paramb\nthen param1 / (grid1_grid_result||1)\nelse (grid2_grid_result||1) / param1""}"
+Rule3,ComplexRule,2017-3-1 00:00:00,2017-4-1,false,"{""g_array"":[""G1V2"",""G1V3""],""g_string"":""def"",""g_integer"":11}","{""cguard1"":""1==1"",""cguard2"":""[param2 == 'abc', \""a string\""]""}","{""grid1"":""DataGrid1"",""grid2"":""DataGrid2""}","{""simple_result"":""\""c value\"""",""computed_value"":""if paramb\nthen param1 / (grid1_grid_result||1)\nelse (grid2_grid_result||1) / param1""}"
 Rule4,ComplexRule,2017-4-1 15:00:01,2017-5-1,,"{""g_array"":[""G1V2"",""G1V3""],""g_string"":""Hi Mom"",""g_integer"":11}","{""cguard1"":""1==1"",""cguard2"":""param2 == \""abc\""""}","{""grid1"":""DataGrid1"",""grid2"":""DataGrid2""}","{""computed_name_grid"":""\""DataGrid\""+\""X\"""", ""simple_result"":""computed_name_grid"",""grid_sum"":""computed_name_grid_result + (grid1_grid_result||0)+(grid2_grid_result||0)""}"
-Rule5,ComplexRule,2017-4-2 15:00:01,2017-5-1,,"{""g_string"":""zzz"",""g_integer"":3757,""g_has_default"":""foo""}","{""cguard1"":""1==1""}",,"{""flavor"": ""[\""cherry\"",\""lemon\""][param2]"",""other_grid"": ""\""DataGrid4\"""",""final_value"": ""other_grid_result * 3""}"
+Rule5,ComplexRule,2017-4-2 15:00:01,2017-5-1,,"{""g_string"":""zzz"",""g_integer"":3757,""g_has_default"":""foo""}","{""cguard1"":""[1==1, \""a string\""]""}",,"{""flavor"": ""[\""cherry\"",\""lemon\""][param2]"",""other_grid"": ""\""DataGrid4\"""",""final_value"": ""other_grid_result * 3""}"

data/spec/lib/logger_spec.rb

@@ -15,8 +15,12 @@ module Marty
     end
 
     it "log has its own connection" do
-      expect(Marty::Log.connection).not_to equal(Marty::Posting.connection)
-      expect(Marty::Posting.connection).to equal(Marty::Script.connection)
+      log_conn     = Marty::Log.connection.raw_connection
+      posting_conn = Marty::Posting.connection.raw_connection
+      script_conn  = Marty::Script.connection.raw_connection
+
+      expect(log_conn).not_to equal(posting_conn)
+      expect(posting_conn).to equal(script_conn)
     end
 
     it "logs" do

data/spec/models/rule_spec.rb

@@ -286,7 +286,7 @@ module Marty::RuleSpec
       it "computed guards work" do
         c = complex.compute(@ruleopts_myrule, {"pt"=>Time.zone.now,
                                                'param2'=>'def'})
-        expect(c).to eq({"cguard2"=>false})
+        expect(c).to eq({"cguard2"=>[false, "a string"]})
       end
       it "returns simple results via #fixed_results" do
         expect(simple.fixed_results["simple_result"]).to eq("b value")

data/spec/support/empty_job.rb

@@ -0,0 +1,5 @@
+# Used for testing delayed job triggers
+class EmptyJob
+  def perform
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: marty
 version: !ruby/object:Gem::Version
-  version: 2.4.5
+  version: 2.4.6
 platform: ruby
 authors:
 - Arman Bostani
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-10 00:00:00.000000000 Z
+date: 2019-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -186,10 +186,12 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- Dockerfile.dummy
 - Gemfile
 - Gemfile.lock
 - INDEPENDENCE_ISSUES.md
 - MIT-LICENSE
+- Makefile
 - README.md
 - Rakefile
 - app/assets/images/marty/.gitkeep
@@ -401,6 +403,7 @@ files:
 - app/components/marty/user_view.rb
 - app/controllers/marty/application_controller.rb
 - app/controllers/marty/components_controller.rb
+- app/controllers/marty/delayed_job_controller.rb
 - app/controllers/marty/diagnostic/controller.rb
 - app/controllers/marty/job_controller.rb
 - app/controllers/marty/report_controller.rb
@@ -488,6 +491,7 @@ files:
 - db/seeds.rb
 - delorean/diagnostics.dl
 - delorean/script_report.dl
+- docker-compose.dummy.yml
 - gemini_deprecations.md
 - lib/marty.rb
 - lib/marty/aws/base.rb
@@ -520,6 +524,7 @@ files:
 - lib/pyxll/pyxll.cfg
 - lib/pyxll/sample.xlsx
 - lib/tasks/marty_tasks.rake
+- make-dummy.mk
 - marty.gemspec
 - other/marty/api/base.rb
 - other/marty/diagnostic/aws/ec2_instance.rb
@@ -540,6 +545,7 @@ files:
 - other/marty/diagnostic/version.rb
 - script/rails
 - spec/controllers/application_controller_spec.rb
+- spec/controllers/delayed_job_controller_spec.rb
 - spec/controllers/diagnostic/controller_spec.rb
 - spec/controllers/job_controller_spec.rb
 - spec/controllers/rpc_controller_spec.rb
@@ -1693,6 +1699,7 @@ files:
 - spec/support/custom_selectors.rb
 - spec/support/delayed_job_helpers.rb
 - spec/support/download_helper.rb
+- spec/support/empty_job.rb
 - spec/support/helper.rb
 - spec/support/netzke.rb
 - spec/support/performance_helper.rb
@@ -1723,7 +1730,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: A framework for working with versioned data