sp-job 0.2.3 → 0.3.22

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/bin/configure +40 -0
  4. data/lib/sp-job.rb +21 -2
  5. data/lib/sp/job/back_burner.rb +350 -68
  6. data/lib/sp/job/broker.rb +18 -16
  7. data/lib/sp/job/broker_http_client.rb +80 -20
  8. data/lib/sp/job/broker_oauth2_client.rb +12 -4
  9. data/lib/sp/job/common.rb +876 -62
  10. data/lib/sp/job/configure/configure.rb +640 -0
  11. data/lib/sp/job/curl_http_client.rb +100 -0
  12. data/lib/sp/job/easy_http_client.rb +94 -0
  13. data/lib/sp/job/http_client.rb +51 -0
  14. data/lib/sp/job/job_db_adapter.rb +38 -36
  15. data/lib/sp/job/jsonapi_error.rb +31 -74
  16. data/lib/sp/job/jwt.rb +55 -5
  17. data/lib/sp/job/mail_queue.rb +9 -2
  18. data/lib/sp/job/manticore_http_client.rb +94 -0
  19. data/lib/sp/job/pg_connection.rb +90 -10
  20. data/lib/sp/job/query_params.rb +45 -0
  21. data/lib/sp/job/rfc822.rb +13 -0
  22. data/lib/sp/job/session.rb +239 -0
  23. data/lib/sp/job/unique_file.rb +37 -1
  24. data/lib/sp/job/uploaded_image_converter.rb +27 -19
  25. data/lib/sp/job/worker.rb +51 -1
  26. data/lib/sp/job/worker_thread.rb +22 -7
  27. data/lib/sp/jsonapi.rb +24 -0
  28. data/lib/sp/jsonapi/adapters/base.rb +177 -0
  29. data/lib/sp/jsonapi/adapters/db.rb +26 -0
  30. data/lib/sp/jsonapi/adapters/raw_db.rb +96 -0
  31. data/lib/sp/jsonapi/exceptions.rb +54 -0
  32. data/lib/sp/jsonapi/model/base.rb +31 -0
  33. data/lib/sp/jsonapi/model/concerns/attributes.rb +91 -0
  34. data/lib/sp/jsonapi/model/concerns/model.rb +39 -0
  35. data/lib/sp/jsonapi/model/concerns/persistence.rb +212 -0
  36. data/lib/sp/jsonapi/model/concerns/serialization.rb +57 -0
  37. data/lib/sp/jsonapi/parameters.rb +54 -0
  38. data/lib/sp/jsonapi/service.rb +96 -0
  39. data/lib/tasks/configure.rake +2 -496
  40. data/sp-job.gemspec +3 -2
  41. metadata +24 -2
@@ -0,0 +1,26 @@
1
+ module SP
2
+ module JSONAPI
3
+ module Adapters
4
+
5
+ class Db < RawDb
6
+
7
+ protected
8
+
9
+ def get_error_response(path, error) ; HashWithIndifferentAccess.new(error_response(path, error)) ; end
10
+
11
+ private
12
+
13
+ def is_error?(result) ; !result[:errors].blank? ; end
14
+
15
+ def process_result(result)
16
+ result = HashWithIndifferentAccess.new(result)
17
+ result[:response] = JSON.parse(result[:response])
18
+ raise SP::JSONAPI::Exceptions::GenericModelError.new(result[:response]) if is_error?(result[:response])
19
+ [ result[:http_status], result[:response] ]
20
+ end
21
+
22
+ end
23
+
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,96 @@
1
+ module SP
2
+ module JSONAPI
3
+ module Adapters
4
+
5
+ class RawDb < Base
6
+
7
+ protected
8
+
9
+ def unwrap_response(response)
10
+ # As the method request() is EXACTLY the same as request!(), and it cannot be reverted without affecting lots of changes already made in the app's controllers...
11
+ # Allow for response being both a [ status, result ] pair (as of old) OR a single result (as of now)
12
+ if response.is_a?(Array)
13
+ status = response[0].to_i
14
+ result = response[1]
15
+ raise SP::JSONAPI::Exceptions::GenericModelError.new(result) if status != SP::JSONAPI::Status::OK
16
+ result
17
+ else
18
+ # No raise here, we do not know the status...
19
+ response
20
+ end
21
+ end
22
+
23
+ def get_error_response(path, error) ; error_response(path, error).to_json ; end
24
+
25
+ def do_request(method, path, params)
26
+ process_result(do_request_on_the_db(method, path, params))
27
+ end
28
+
29
+ def explicit_do_request(exp_subentity_schema, exp_subentity_prefix, method, path, params)
30
+ process_result(explicit_do_request_on_the_db(exp_subentity_schema, exp_subentity_prefix, method, path, params))
31
+ end
32
+
33
+ def specific_service_do_request(method, path, params, service_params)
34
+ process_result(specific_service_do_request_on_the_db(method, path, params, service_params))
35
+ end
36
+
37
+ private
38
+ def user_id ; "'#{service.parameters.user_id}'" ; end
39
+ def entity_id ; "'#{service.parameters.entity_id}'" ; end
40
+ def entity_schema ; service.parameters.entity_schema.nil? ? 'NULL' : "'#{service.parameters.entity_schema}'" ; end
41
+ def sharded_schema ; service.parameters.sharded_schema.nil? ? 'NULL' : "'#{service.parameters.sharded_schema}'" ; end
42
+ def subentity_schema ; service.parameters.subentity_schema.nil? ? 'NULL' : "'#{service.parameters.subentity_schema}'" ; end
43
+ def subentity_prefix ; service.parameters.subentity_prefix.nil? ? 'NULL' : "'#{service.parameters.subentity_prefix}'" ; end
44
+
45
+ def process_result(result)
46
+ raise SP::JSONAPI::Exceptions::GenericModelError.new(result) if is_error?(result)
47
+ [ SP::JSONAPI::Status::OK, result ]
48
+ end
49
+
50
+ # Implement the JSONAPI request by direct querying of the JSONAPI function in the database
51
+ def do_request_on_the_db(method, path, params)
52
+ jsonapi_query = if method == 'GET'
53
+ %Q[ SELECT * FROM public.jsonapi('#{method}', '#{url_with_params_for_query(path, params)}', '', #{user_id}, #{entity_id}, #{entity_schema}, #{sharded_schema}, #{subentity_schema}, #{subentity_prefix}) ]
54
+ else
55
+ %Q[ SELECT * FROM public.jsonapi('#{method}', '#{url(path)}', '#{params_for_body(params)}', #{user_id}, #{entity_id}, #{entity_schema}, #{sharded_schema}, #{subentity_schema}, #{subentity_prefix}) ]
56
+ end
57
+ response = service.connection.exec jsonapi_query
58
+ response.first if response.first
59
+ end
60
+
61
+ def explicit_do_request_on_the_db(exp_subentity_schema, exp_subentity_prefix, method, path, params)
62
+ _subentity_schema = "'#{exp_subentity_schema}'"
63
+ _subentity_prefix = "'#{exp_subentity_prefix}'"
64
+
65
+ jsonapi_query = if method == 'GET'
66
+ %Q[ SELECT * FROM public.jsonapi('#{method}', '#{url_with_params_for_query(path, params)}', '', #{user_id}, #{entity_id}, #{entity_schema}, #{sharded_schema}, #{_subentity_schema}, #{_subentity_prefix}) ]
67
+ else
68
+ %Q[ SELECT * FROM public.jsonapi('#{method}', '#{url(path)}', '#{params_for_body(params)}', #{user_id}, #{entity_id}, #{entity_schema}, #{sharded_schema}, #{_subentity_schema}, #{_subentity_prefix}) ]
69
+ end
70
+ response = service.connection.exec jsonapi_query
71
+ response.first if response.first
72
+ end
73
+
74
+ def specific_service_do_request_on_the_db(method, path, params, service_params)
75
+ _user_id = "'"+service_params["user_id"]+"'"
76
+ _entity_id = "'"+service_params["entity_id"]+"'"
77
+ _entity_schema = service_params["entity_schema"].blank? ? 'NULL' : "'"+service_params["entity_schema"]+"'"
78
+ _sharded_schema = service_params["sharded_schema"].blank? ? 'NULL' : "'"+service_params["sharded_schema"]+"'"
79
+ _subentity_schema = service_params["subentity_schema"].blank? ? 'NULL' : "'"+service_params["subentity_schema"]+"'"
80
+ _subentity_prefix = service_params["subentity_prefix"].blank? ? 'NULL' : "'"+service_params["subentity_prefix"]+"'"
81
+
82
+ jsonapi_query = if method == 'GET'
83
+ %Q[ SELECT * FROM public.jsonapi('#{method}', '#{url_with_params_for_query(path, params)}', '', #{_user_id}, #{_entity_id}, #{_entity_schema}, #{_sharded_schema}, #{_subentity_schema}, #{_subentity_prefix}) ]
84
+ else
85
+ %Q[ SELECT * FROM public.jsonapi('#{method}', '#{url(path)}', '#{params_for_body(params)}', #{_user_id}, #{_entity_id}, #{_entity_schema}, #{_sharded_schema}, #{_subentity_schema}, #{_subentity_prefix}) ]
86
+ end
87
+ response = service.connection.exec jsonapi_query
88
+ response.first if response.first
89
+ end
90
+
91
+ def is_error?(result) ; result =~ /^\s*{\s*"errors"\s*:/ ; end
92
+ end
93
+
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,54 @@
1
+ module SP
2
+ module JSONAPI
3
+ module Exceptions
4
+
5
+ # JSONAPI service and configuration errors
6
+
7
+ class ServiceSetupError < StandardError ; ; end
8
+ class ServiceProtocolError < StandardError ; ; end
9
+ class InvalidResourceConfigurationError < StandardError ; ; end
10
+ class InvalidResourcePublisherError < StandardError ; ; end
11
+ class DuplicateResourceError < StandardError ; ; end
12
+ class SaveConfigurationError < StandardError ; ; end
13
+ class InvalidJSONAPIKeyError < StandardError ; ; end
14
+
15
+ # JSONAPI model querying errors
16
+
17
+ class GenericModelError < StandardError
18
+
19
+ attr_reader :id
20
+ attr_reader :status
21
+ attr_reader :result
22
+
23
+ def initialize(result, nested = $!)
24
+ @result = result
25
+ errors = get_result_errors()
26
+ @status = (errors.map { |error| error[:status].to_i }.max) || 403
27
+ message = errors.first[:detail]
28
+ super(message, nested)
29
+ end
30
+
31
+ def internal_error
32
+ errors = get_result_errors()
33
+ if errors.length != 1
34
+ @result.to_json
35
+ else
36
+ errors.first[:meta]['internal-error'] if errors.first[:meta]
37
+ end
38
+ end
39
+
40
+ def inspect()
41
+ description = super()
42
+ description = description + " (#{internal_error})" if internal_error
43
+ description
44
+ end
45
+
46
+ private
47
+
48
+ def get_result_errors() ; (result.is_a?(Hash) ? result : HashWithIndifferentAccess.new(JSON.parse(result)))[:errors] ; end
49
+
50
+ end
51
+
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,31 @@
1
+ require 'sp/jsonapi/model/concerns/model'
2
+
3
+ module SP
4
+ module JSONAPI
5
+ module Model
6
+
7
+ class Base
8
+ include SP::JSONAPI::Model::Concerns::Model
9
+
10
+ @@adapter = nil
11
+ @@include_root_in_json = true
12
+
13
+ def self.adapter ; @@adapter ; end
14
+ def self.adapter=(value) ; @@adapter = value ; end
15
+
16
+ def self.include_root_in_json ; @@include_root_in_json ; end
17
+ def self.include_root_in_json=(value) ; @@include_root_in_json = value ; end
18
+
19
+ def self.inherited(child)
20
+ child.resource_name = child.name.demodulize.underscore.pluralize
21
+ child.autogenerated_id = true
22
+ end
23
+
24
+ def self.i18n_scope
25
+ :activerecord
26
+ end
27
+ end
28
+
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,91 @@
1
+ module SP
2
+ module JSONAPI
3
+ module Model
4
+ module Concerns
5
+ module Attributes
6
+ def self.included(base)
7
+ base.extend ClassMethods
8
+ end
9
+
10
+ module ClassMethods
11
+
12
+ def attributes
13
+ if !@attributes && superclass.respond_to?(:attributes)
14
+ @attributes = []
15
+ @attributes += superclass.attributes
16
+ end
17
+ @attributes = [] if !@attributes
18
+ @attributes
19
+ end
20
+
21
+ def attr_accessible(name)
22
+ attributes << name if !attributes.include?(name)
23
+ attr_accessor name
24
+ end
25
+ end
26
+
27
+ def initialize(new_attributes = nil)
28
+ assign_attributes(new_attributes) if new_attributes
29
+ yield self if block_given?
30
+ end
31
+
32
+ def attributes=(new_attributes)
33
+ return unless new_attributes.is_a?(Hash)
34
+ assign_attributes(new_attributes)
35
+ end
36
+
37
+ private
38
+
39
+ def assign_attributes(new_attributes)
40
+ return if new_attributes.blank?
41
+
42
+ new_attributes = new_attributes.stringify_keys
43
+ nested_parameter_attributes = []
44
+
45
+ new_attributes.each do |k, v|
46
+ if respond_to?("#{k}=")
47
+ if v.is_a?(Hash)
48
+ nested_parameter_attributes << [ k, v ]
49
+ else
50
+ send("#{k}=", v)
51
+ end
52
+ # else
53
+ # raise(ActiveRecord::UnknownAttributeError, "unknown attribute: #{k}")
54
+ end
55
+ end
56
+
57
+ # Assign any deferred nested attributes after the base attributes have been set
58
+ nested_parameter_attributes.each do |k,v|
59
+ send("#{k}=", v)
60
+ end
61
+ end
62
+
63
+ # Returns an <tt>#inspect</tt>-like string for the value of the
64
+ # attribute +attr_name+. String attributes are truncated upto 50
65
+ # characters, and Date and Time attributes are returned in the
66
+ # <tt>:db</tt> format. Other attributes return the value of
67
+ # <tt>#inspect</tt> without modification.
68
+ #
69
+ # person = Person.create!(:name => "David Heinemeier Hansson " * 3)
70
+ #
71
+ # person.attribute_for_inspect(:name)
72
+ # # => '"David Heinemeier Hansson David Heinemeier Hansson D..."'
73
+ #
74
+ # person.attribute_for_inspect(:created_at)
75
+ # # => '"2009-01-12 04:48:57"'
76
+ def attribute_for_inspect(name)
77
+ value = self.send(name)
78
+ if value.is_a?(String) && value.length > 50
79
+ "#{value[0..50]}...".inspect
80
+ elsif value.is_a?(Date) || value.is_a?(Time)
81
+ %("#{value.to_s(:db)}")
82
+ else
83
+ value.inspect
84
+ end
85
+ end
86
+
87
+ end
88
+ end
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,39 @@
1
+ require 'sp/jsonapi/model/concerns/attributes'
2
+ require 'sp/jsonapi/model/concerns/serialization'
3
+ require 'sp/jsonapi/model/concerns/persistence'
4
+
5
+ module SP
6
+ module JSONAPI
7
+ module Model
8
+ module Concerns
9
+ module Model
10
+
11
+ def self.included klass
12
+ klass.extend ClassMethods
13
+
14
+ klass.class_eval do
15
+ include Attributes
16
+ include Serialization
17
+ include Persistence
18
+ end
19
+ end
20
+
21
+ module ClassMethods
22
+
23
+ def inspect
24
+ "#{super}(#{self.attributes.join(', ')})"
25
+ end
26
+ end
27
+
28
+ # Returns the contents of the record as a nicely formatted string.
29
+ def inspect
30
+ # attrs = self.class.attributes
31
+ inspection = self.class.attributes.collect { |name| "#{name}: #{attribute_for_inspect(name)}" }.compact.join(", ")
32
+ "#<#{self.class} #{inspection}>"
33
+ end
34
+
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,212 @@
1
+ module SP
2
+ module JSONAPI
3
+ module Model
4
+ module Concerns
5
+ module Persistence
6
+
7
+ def self.included(base)
8
+ base.extend ClassMethods
9
+ end
10
+
11
+ module ClassMethods
12
+
13
+ # Define resource configuration accessors at the class (and subclass) level (static).
14
+ # These attribute values are NOT inherited by subclasses, each subclass MUST define their own.
15
+ # Instances can access these attributes at the class level only.
16
+ attr_accessor :resource_name
17
+ attr_accessor :autogenerated_id
18
+ attr_accessor :non_persistent_attributes
19
+
20
+ def find!(id, conditions = nil) ; get(id, conditions) ; end
21
+
22
+ def find_explicit!(exp_subentity_schema, exp_subentity_prefix, id, conditions = nil)
23
+ get_explicit(exp_subentity_schema, exp_subentity_prefix, id, conditions)
24
+ end
25
+
26
+ def find(id, conditions = nil)
27
+ begin
28
+ get(id, conditions)
29
+ rescue Exception => e
30
+ return nil
31
+ end
32
+ end
33
+
34
+ def query!(condition) ; get_all(condition) ; end
35
+ def query_explicit!(exp_subentity_schema, exp_subentity_prefix, condition) ; get_all_explicit(exp_subentity_schema, exp_subentity_prefix, condition) ; end
36
+
37
+ def query(condition)
38
+ begin
39
+ get_all(condition)
40
+ rescue Exception => e
41
+ nil
42
+ end
43
+ end
44
+
45
+ def first!(condition = "")
46
+ condition += (condition.blank? ? "" : "&") + "page[size]=1"
47
+ get_all(condition).first
48
+ end
49
+
50
+ def first(condition = "")
51
+ begin
52
+ condition += (condition.blank? ? "" : "&") + "page[size]=1"
53
+ get_all(condition).first
54
+ rescue Exception => e
55
+ nil
56
+ end
57
+ end
58
+
59
+ def all! ; get_all("") ; end
60
+
61
+ def all
62
+ begin
63
+ get_all("")
64
+ rescue Exception => e
65
+ nil
66
+ end
67
+ end
68
+
69
+ private
70
+
71
+ def get_explicit(exp_subentity_schema, exp_subentity_prefix, id, conditions = nil)
72
+ result = self.adapter.get_explicit!(exp_subentity_schema, exp_subentity_prefix, "#{self.resource_name}/#{id.to_s}", conditions)
73
+ jsonapi_result_to_instance(result[:data], result)
74
+ end
75
+
76
+ def get(id, conditions = nil)
77
+ result = self.adapter.get("#{self.resource_name}/#{id.to_s}", conditions)
78
+ jsonapi_result_to_instance(result[:data], result)
79
+ end
80
+
81
+ def get_all(condition)
82
+ got = []
83
+ result = self.adapter.get(self.resource_name, condition)
84
+ if result
85
+ got = result[:data].map do |item|
86
+ data = { data: item }
87
+ data.merge(included: result[:included]) if result[:included]
88
+ jsonapi_result_to_instance(item, data)
89
+ end
90
+ end
91
+ got
92
+ end
93
+
94
+ def get_all_explicit(exp_subentity_schema, exp_subentity_prefix, condition)
95
+ got = []
96
+ result = self.adapter.get_explicit!(exp_subentity_schema, exp_subentity_prefix, self.resource_name, condition)
97
+ if result
98
+ got = result[:data].map do |item|
99
+ data = { data: item }
100
+ data.merge(included: result[:included]) if result[:included]
101
+ jsonapi_result_to_instance(item, data)
102
+ end
103
+ end
104
+ got
105
+ end
106
+
107
+ def jsonapi_result_to_instance(result, data)
108
+ if result
109
+ instance = self.new(result.merge(result[:attributes]).except(:attributes))
110
+ instance.send :_data=, data
111
+ end
112
+ instance
113
+ end
114
+ end
115
+
116
+ # Instance methods
117
+
118
+ def new_record?
119
+ if self.class.autogenerated_id || self.id.nil?
120
+ self.id.nil?
121
+ else
122
+ self.class.find(self.id).nil?
123
+ end
124
+ end
125
+
126
+ def save!
127
+ if new_record?
128
+ create!
129
+ else
130
+ update!
131
+ end
132
+ end
133
+
134
+ def save_explicit!(exp_subentity_schema, exp_subentity_prefix)
135
+ if new_record?
136
+ create!(exp_subentity_schema, exp_subentity_prefix)
137
+ else
138
+ update!(exp_subentity_schema, exp_subentity_prefix)
139
+ end
140
+ end
141
+
142
+ def destroy!
143
+ if !new_record?
144
+ self.class.adapter.delete(path_for_id)
145
+ end
146
+ end
147
+
148
+ def destroy_explicit!(exp_subentity_schema, exp_subentity_prefix)
149
+ if !new_record?
150
+ self.class.adapter.delete_explicit!(exp_subentity_schema, exp_subentity_prefix, path_for_id)
151
+ end
152
+ end
153
+
154
+ alias :delete! :destroy!
155
+
156
+ def create!(exp_subentity_schema = nil, exp_subentity_prefix = nil)
157
+ if self.class.autogenerated_id
158
+ params = {
159
+ data: {
160
+ type: self.class.resource_name,
161
+ attributes: get_persistent_json.reject { |k,v| k == :id || v.nil? }
162
+ }
163
+ }
164
+ else
165
+ params = {
166
+ data: {
167
+ type: self.class.resource_name,
168
+ attributes: get_persistent_json.reject { |k,v| v.nil? }
169
+ }
170
+ }
171
+ end
172
+ result = if !exp_subentity_schema.blank? || !exp_subentity_prefix.blank?
173
+ self.class.adapter.post_explicit!(exp_subentity_schema, exp_subentity_prefix, self.class.resource_name, params)
174
+ else
175
+ self.class.adapter.post(self.class.resource_name, params)
176
+ end
177
+ # Set the id to the newly created id
178
+ self.id = result[:data][:id]
179
+ end
180
+
181
+ def update!(exp_subentity_schema = nil, exp_subentity_prefix = nil)
182
+ params = {
183
+ data: {
184
+ type: self.class.resource_name,
185
+ id: self.id.to_s,
186
+ attributes: get_persistent_json.reject { |k,v| k == :id }
187
+ }
188
+ }
189
+ result = if !exp_subentity_schema.blank? || !exp_subentity_prefix.blank?
190
+ self.class.adapter.patch_explicit!(exp_subentity_schema, exp_subentity_prefix, path_for_id, params)
191
+ else
192
+ self.class.adapter.patch(path_for_id, params)
193
+ end
194
+ end
195
+
196
+ def get_persistent_json
197
+ as_json.reject { |k| !self.class.attributes.include?(k) || (self.class.non_persistent_attributes || []).include?(k) }
198
+ end
199
+
200
+ protected
201
+
202
+ attr_accessor :_data
203
+
204
+ private
205
+
206
+ def path_for_id ; "#{self.class.resource_name}/#{self.id.to_s}" ; end
207
+
208
+ end
209
+ end
210
+ end
211
+ end
212
+ end