sufia-models 6.2.0 → 6.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. checksums.yaml +4 -4
  2. data/app/actors/sufia/generic_file/actor.rb +4 -7
  3. data/app/jobs/active_fedora_id_based_job.rb +1 -1
  4. data/app/jobs/active_fedora_pid_based_job.rb +1 -2
  5. data/app/jobs/audit_job.rb +1 -1
  6. data/app/jobs/batch_update_job.rb +6 -6
  7. data/app/jobs/create_derivatives_job.rb +2 -3
  8. data/app/jobs/import_url_job.rb +8 -5
  9. data/app/jobs/ingest_local_file_job.rb +1 -1
  10. data/app/models/batch.rb +15 -21
  11. data/app/models/checksum_audit_log.rb +0 -1
  12. data/app/models/concerns/sufia/ability.rb +5 -0
  13. data/app/models/concerns/sufia/collection_behavior.rb +3 -2
  14. data/app/models/concerns/sufia/file_stat_utils.rb +19 -21
  15. data/app/models/concerns/sufia/generic_file/batches.rb +1 -3
  16. data/app/models/concerns/sufia/generic_file/characterization.rb +11 -16
  17. data/app/models/concerns/sufia/generic_file/content.rb +0 -1
  18. data/app/models/concerns/sufia/generic_file/derivatives.rb +2 -2
  19. data/app/models/concerns/sufia/generic_file/export.rb +50 -59
  20. data/app/models/concerns/sufia/generic_file/full_text_indexing.rb +15 -18
  21. data/app/models/concerns/sufia/generic_file/metadata.rb +0 -2
  22. data/app/models/concerns/sufia/generic_file/mime_types.rb +9 -9
  23. data/app/models/concerns/sufia/generic_file/permissions.rb +0 -1
  24. data/app/models/concerns/sufia/generic_file/proxy_deposit.rb +0 -1
  25. data/app/models/concerns/sufia/generic_file/querying.rb +9 -5
  26. data/app/models/concerns/sufia/generic_file/trophies.rb +1 -1
  27. data/app/models/concerns/sufia/generic_file/versions.rb +0 -4
  28. data/app/models/concerns/sufia/model_methods.rb +0 -1
  29. data/app/models/concerns/sufia/user.rb +15 -15
  30. data/app/models/concerns/sufia/user_usage_stats.rb +0 -2
  31. data/app/models/datastreams/fits_datastream.rb +25 -25
  32. data/app/models/domain_term.rb +2 -3
  33. data/app/models/featured_work.rb +3 -5
  34. data/app/models/file_download_stat.rb +3 -4
  35. data/app/models/file_usage.rb +10 -11
  36. data/app/models/file_view_stat.rb +3 -3
  37. data/app/models/follow.rb +1 -1
  38. data/app/models/geo_names_resource.rb +3 -3
  39. data/app/models/group.rb +1 -3
  40. data/app/models/local_authority.rb +26 -28
  41. data/app/models/proxy_deposit_request.rb +9 -9
  42. data/app/models/single_use_link.rb +10 -18
  43. data/app/models/sufia/download.rb +2 -2
  44. data/app/models/sufia/pageview.rb +1 -1
  45. data/app/models/trophy.rb +2 -4
  46. data/app/services/sufia/analytics.rb +10 -11
  47. data/app/services/sufia/generic_file_audit_service.rb +11 -12
  48. data/app/services/sufia/repository_audit_service.rb +1 -1
  49. data/config/locales/sufia.en.yml +2 -0
  50. data/lib/generators/sufia/models/abstract_migration_generator.rb +7 -6
  51. data/lib/generators/sufia/models/install_generator.rb +3 -3
  52. data/lib/generators/sufia/models/templates/config/arkivo_constraint.rb +1 -1
  53. data/lib/generators/sufia/models/templates/config/clamav.rb +1 -1
  54. data/lib/generators/sufia/models/templates/config/redis_config.rb +13 -5
  55. data/lib/generators/sufia/models/templates/config/resque_admin.rb +2 -2
  56. data/lib/generators/sufia/models/templates/config/resque_config.rb +1 -1
  57. data/lib/generators/sufia/models/templates/config/sufia.rb +10 -4
  58. data/lib/generators/sufia/models/templates/migrations/create_checksum_audit_logs.rb +1 -1
  59. data/lib/generators/sufia/models/templates/migrations/create_file_download_stats.rb +1 -1
  60. data/lib/generators/sufia/models/templates/migrations/create_file_view_stats.rb +1 -1
  61. data/lib/generators/sufia/models/templates/migrations/create_local_authorities.rb +1 -1
  62. data/lib/generators/sufia/models/update_content_blocks_generator.rb +0 -1
  63. data/lib/generators/sufia/models/upgrade600_generator.rb +0 -1
  64. data/lib/generators/sufia/models/user_stats_generator.rb +2 -2
  65. data/lib/sufia/messages.rb +17 -17
  66. data/lib/sufia/models.rb +1 -1
  67. data/lib/sufia/models/active_fedora/redis.rb +1 -4
  68. data/lib/sufia/models/active_record/redis.rb +2 -3
  69. data/lib/sufia/models/engine.rb +12 -7
  70. data/lib/sufia/models/file_content/versions.rb +0 -1
  71. data/lib/sufia/models/resque.rb +2 -2
  72. data/lib/sufia/models/stats/user_stat_importer.rb +65 -67
  73. data/lib/sufia/models/user_local_directory_behavior.rb +9 -13
  74. data/lib/sufia/models/utils.rb +1 -2
  75. data/lib/sufia/models/version.rb +1 -1
  76. data/lib/sufia/permissions.rb +0 -1
  77. data/lib/sufia/permissions/readable.rb +0 -1
  78. data/lib/sufia/permissions/writable.rb +20 -23
  79. data/lib/tasks/sufia-models_tasks.rake +18 -0
  80. data/sufia-models.gemspec +1 -1
  81. metadata +5 -5
@@ -1,9 +1,8 @@
1
1
  module Sufia
2
2
  class UserStatImporter
3
-
4
3
  UserRecord = Struct.new("UserRecord", :id, :user_key, :last_stats_update)
5
4
 
6
- def initialize(options={})
5
+ def initialize(options = {})
7
6
  @verbose = options[:verbose]
8
7
  @logging = options[:logging]
9
8
  @delay_secs = options[:delay_secs].to_f
@@ -35,94 +34,93 @@ module Sufia
35
34
  end
36
35
 
37
36
  # Returns an array of users sorted by the date of their last
38
- # stats update. Users that have not been recently updated
37
+ # stats update. Users that have not been recently updated
39
38
  # will be at the top of the array.
40
39
  def sorted_users
41
40
  users = []
42
41
  ::User.find_each do |user|
43
42
  users.push(UserRecord.new(user.id, user.user_key, date_since_last_cache(user)))
44
43
  end
45
- users.sort {|a, b| a.last_stats_update <=> b.last_stats_update}
44
+ users.sort { |a, b| a.last_stats_update <=> b.last_stats_update }
46
45
  end
47
46
 
48
- private
47
+ private
49
48
 
50
- def delay
51
- sleep @delay_secs
52
- end
49
+ def delay
50
+ sleep @delay_secs
51
+ end
53
52
 
54
- def rescue_and_retry(fail_message)
55
- retry_count = 0
56
- begin
57
- return yield
58
- rescue StandardError => e
59
- retry_count += 1
60
- if retry_count < @number_of_retries
61
- delay
62
- retry
63
- else
64
- log_message fail_message
65
- log_message "Last exception #{e}"
53
+ def rescue_and_retry(fail_message)
54
+ retry_count = 0
55
+ begin
56
+ return yield
57
+ rescue StandardError => e
58
+ retry_count += 1
59
+ if retry_count < @number_of_retries
60
+ delay
61
+ retry
62
+ else
63
+ log_message fail_message
64
+ log_message "Last exception #{e}"
65
+ end
66
66
  end
67
67
  end
68
- end
69
68
 
70
- def date_since_last_cache(user)
71
- last_cached_stat = UserStat.where(user_id: user.id).order(date: :asc).last
69
+ def date_since_last_cache(user)
70
+ last_cached_stat = UserStat.where(user_id: user.id).order(date: :asc).last
72
71
 
73
- if last_cached_stat
74
- last_cached_stat.date + 1.day
75
- else
76
- Sufia.config.analytic_start_date
72
+ if last_cached_stat
73
+ last_cached_stat.date + 1.day
74
+ else
75
+ Sufia.config.analytic_start_date
76
+ end
77
77
  end
78
- end
79
78
 
80
- def file_ids_for_user(user)
81
- ids = []
82
- ::GenericFile.find_in_batches("#{Solrizer.solr_name('depositor', :symbol)}:\"#{user.user_key}\"", fl:"id") do |group|
83
- ids.concat group.map { |doc| doc["id"] }
79
+ def file_ids_for_user(user)
80
+ ids = []
81
+ ::GenericFile.find_in_batches("#{Solrizer.solr_name('depositor', :symbol)}:\"#{user.user_key}\"", fl: "id") do |group|
82
+ ids.concat(group.map { |doc| doc["id"] })
83
+ end
84
+ ids
84
85
  end
85
- ids
86
- end
87
86
 
88
- # For each date, add the view and download counts for this
89
- # file to the view & download sub-totals for that day.
90
- # The resulting hash will look something like this:
91
- # {"2014-11-30 00:00:00 UTC" => {:views=>2, :downloads=>5},
92
- # "2014-12-01 00:00:00 UTC" => {:views=>4, :downloads=>4}}
93
- def tally_results(file_stats, stat_name, total_stats)
94
- file_stats.each do |stats|
95
- # Exclude the stats from today since it will only be a partial day's worth of data
96
- break if stats.date == Date.today
97
-
98
- date_key = stats.date.to_s
99
- old_count = total_stats[date_key] ? total_stats[date_key].fetch(stat_name) { 0 } : 0
100
- new_count = old_count + stats.method(stat_name).call
101
-
102
- old_values = total_stats[date_key] || {}
103
- total_stats.store(date_key, old_values)
104
- total_stats[date_key].store(stat_name, new_count)
87
+ # For each date, add the view and download counts for this
88
+ # file to the view & download sub-totals for that day.
89
+ # The resulting hash will look something like this:
90
+ # {"2014-11-30 00:00:00 UTC" => {:views=>2, :downloads=>5},
91
+ # "2014-12-01 00:00:00 UTC" => {:views=>4, :downloads=>4}}
92
+ def tally_results(file_stats, stat_name, total_stats)
93
+ file_stats.each do |stats|
94
+ # Exclude the stats from today since it will only be a partial day's worth of data
95
+ break if stats.date == Date.today
96
+
97
+ date_key = stats.date.to_s
98
+ old_count = total_stats[date_key] ? total_stats[date_key].fetch(stat_name) { 0 } : 0
99
+ new_count = old_count + stats.method(stat_name).call
100
+
101
+ old_values = total_stats[date_key] || {}
102
+ total_stats.store(date_key, old_values)
103
+ total_stats[date_key].store(stat_name, new_count)
104
+ end
105
+ total_stats
105
106
  end
106
- total_stats
107
- end
108
107
 
109
- def create_or_update_user_stats(stats, user)
110
- stats.each do |date_string, data|
111
- date = Time.zone.parse(date_string)
108
+ def create_or_update_user_stats(stats, user)
109
+ stats.each do |date_string, data|
110
+ date = Time.zone.parse(date_string)
112
111
 
113
- user_stat = UserStat.where(user_id: user.id).where(date: date).first
114
- user_stat ||= UserStat.new(user_id: user.id, date: date)
112
+ user_stat = UserStat.where(user_id: user.id).where(date: date).first
113
+ user_stat ||= UserStat.new(user_id: user.id, date: date)
115
114
 
116
- user_stat.file_views = data[:views] || 0
117
- user_stat.file_downloads = data[:downloads] || 0
118
- user_stat.save!
115
+ user_stat.file_views = data[:views] || 0
116
+ user_stat.file_downloads = data[:downloads] || 0
117
+ user_stat.save!
118
+ end
119
119
  end
120
- end
121
-
122
- def log_message(message)
123
- puts message if @verbose
124
- Rails.logger.info "#{self.class}: #{message}" if @logging
125
- end
126
120
 
121
+ def log_message(message)
122
+ puts message if @verbose
123
+ Rails.logger.info "#{self.class}: #{message}" if @logging
124
+ end
127
125
  end
128
126
  end
@@ -1,29 +1,25 @@
1
- # To enable local file ingest,
2
- # - Make User model define .directory method that returns a String corresponding to the User's personal import directory on the server. This can be a simple ActiveRecord attribute on the User model, or it can be something more elaborate.
1
+ # To enable local file ingest,
2
+ # - Make User model define .directory method that returns a String corresponding to the User's personal import directory on the server. This can be a simple ActiveRecord attribute on the User model, or it can be something more elaborate.
3
3
  # - Include this module in your User model, or define a .files() method that behaves the same
4
4
  # - Set Sufia.config.enable_local_ingest to true
5
- #
5
+ #
6
6
  module Sufia::UserLocalDirectoryBehavior
7
-
8
7
  # You can use this validator in your User model.
9
- # Ensures that a string defining the path to the user's directory has been provided
8
+ # Ensures that a string defining the path to the user's directory has been provided
10
9
  # and corresponds to a real directory on the server.
11
10
  # @example
12
11
  # validate :directory_must_exist
13
12
  def directory_must_exist
14
- unless directory.blank? || File.directory?(directory)
15
- errors.add(:directory, "must be an existing directory")
16
- end
13
+ return if directory.blank? || File.directory?(directory)
14
+ errors.add(:directory, "must be an existing directory")
17
15
  end
18
16
 
19
17
  # List the contents of the user's directory on the server
20
18
  # Indicates whether each item is a directory or not.
21
19
  def files
22
20
  return [] unless directory.present? && File.directory?(directory)
23
- Dir[File.join(directory, '*')].inject([]) do |accum, val|
24
- accum << { name: File.basename(val), directory: File.directory?(val)}
25
- accum
21
+ Dir[File.join(directory, '*')].each_with_object([]) do |val, accum|
22
+ accum << { name: File.basename(val), directory: File.directory?(val) }
26
23
  end
27
24
  end
28
-
29
- end
25
+ end
@@ -15,7 +15,6 @@ module Sufia
15
15
  end
16
16
 
17
17
  module ClassMethods
18
-
19
18
  # retry the block if the conditional call is true unless we hit the maximum tries
20
19
  #
21
20
  # @param number_of_tries [enumerator] maximum number of times to retry the block
@@ -32,7 +31,7 @@ module Sufia
32
31
  return result unless condition.call
33
32
  sleep(Sufia.config.retry_unless_sleep) if Sufia.config.retry_unless_sleep > 0
34
33
  end
35
- raise RuntimeError, "retry_unless could not complete successfully. Try upping the # of tries?"
34
+ raise "retry_unless could not complete successfully. Try upping the # of tries?"
36
35
  end
37
36
  end
38
37
  end
@@ -1,5 +1,5 @@
1
1
  module Sufia
2
2
  module Models
3
- VERSION = "6.2.0"
3
+ VERSION = "6.3.0"
4
4
  end
5
5
  end
@@ -4,6 +4,5 @@ module Sufia
4
4
 
5
5
  autoload :Writable
6
6
  autoload :Readable
7
-
8
7
  end
9
8
  end
@@ -14,7 +14,6 @@ module Sufia
14
14
  def private?
15
15
  !(public? || registered?)
16
16
  end
17
-
18
17
  end
19
18
  end
20
19
  end
@@ -3,7 +3,7 @@ module Sufia
3
3
  module Writable
4
4
  extend ActiveSupport::Concern
5
5
 
6
- #we're overriding the permissions= method which is in Hydra::AccessControls::Permissions
6
+ # we're overriding the permissions= method which is in Hydra::AccessControls::Permissions
7
7
  include Hydra::AccessControls::Permissions
8
8
  include Hydra::AccessControls::Visibility
9
9
 
@@ -14,20 +14,19 @@ module Sufia
14
14
  def paranoid_permissions
15
15
  valid = true
16
16
  paranoid_edit_permissions.each do |validation|
17
- if validation[:condition].call(self)
18
- errors[validation[:key]] ||= []
19
- errors[validation[:key]] << validation[:message]
20
- valid = false
21
- end
17
+ next unless validation[:condition].call(self)
18
+ errors[validation[:key]] ||= []
19
+ errors[validation[:key]] << validation[:message]
20
+ valid = false
22
21
  end
23
- return valid
22
+ valid
24
23
  end
25
24
 
26
25
  def paranoid_edit_permissions
27
26
  [
28
- {key: :edit_users, message: 'Depositor must have edit access', condition: lambda { |obj| !obj.edit_users.include?(obj.depositor) }},
29
- {key: :edit_groups, message: 'Public cannot have edit access', condition: lambda { |obj| obj.edit_groups.include?('public') }},
30
- {key: :edit_groups, message: 'Registered cannot have edit access', condition: lambda { |obj| obj.edit_groups.include?('registered') }}
27
+ { key: :edit_users, message: 'Depositor must have edit access', condition: ->(obj) { !obj.edit_users.include?(obj.depositor) } },
28
+ { key: :edit_groups, message: 'Public cannot have edit access', condition: ->(obj) { obj.edit_groups.include?('public') } },
29
+ { key: :edit_groups, message: 'Registered cannot have edit access', condition: ->(obj) { obj.edit_groups.include?('registered') } }
31
30
  ]
32
31
  end
33
32
 
@@ -56,20 +55,18 @@ module Sufia
56
55
 
57
56
  private
58
57
 
59
- def permission_hash
60
- old_perms = self.permissions
61
- user_perms = {}
62
- old_perms.select{|r| r[:type] == 'user'}.each do |r|
63
- user_perms[r[:name]] = r[:access]
64
- end
65
- user_perms
66
- group_perms = {}
67
- old_perms.select{|r| r[:type] == 'group'}.each do |r|
68
- group_perms[r[:name]] = r[:access]
58
+ def permission_hash
59
+ old_perms = permissions
60
+ user_perms = {}
61
+ old_perms.select { |r| r[:type] == 'user' }.each do |r|
62
+ user_perms[r[:name]] = r[:access]
63
+ end
64
+ group_perms = {}
65
+ old_perms.select { |r| r[:type] == 'group' }.each do |r|
66
+ group_perms[r[:name]] = r[:access]
67
+ end
68
+ { 'person' => user_perms, 'group' => group_perms }
69
69
  end
70
- {'person'=>user_perms, 'group'=>group_perms}
71
- end
72
-
73
70
  end
74
71
  end
75
72
  end
@@ -1,5 +1,9 @@
1
1
  require 'net/http'
2
2
 
3
+ # Pull in tasks from AF::Noid
4
+ af_noid = Gem::Specification.find_by_name 'active_fedora-noid'
5
+ load "#{af_noid.gem_dir}/lib/tasks/noid_tasks.rake"
6
+
3
7
  namespace :solr do
4
8
  desc "Enqueue a job to resolrize the repository objects"
5
9
  task reindex: :environment do
@@ -8,6 +12,14 @@ namespace :solr do
8
12
  end
9
13
 
10
14
  namespace :sufia do
15
+ namespace :noid do
16
+ desc 'Migrate minter state file'
17
+ task migrate_statefile: :environment do
18
+ ENV['AFNOID_STATEFILE'] = Sufia.config.minter_statefile
19
+ Rake::Task['active_fedora:noid:migrate_statefile'].invoke if needs_migration?(Sufia.config.minter_statefile)
20
+ end
21
+ end
22
+
11
23
  namespace :user do
12
24
  desc 'Populate user tokens'
13
25
  task tokens: :environment do
@@ -94,3 +106,9 @@ def download_from_maven url, dst
94
106
  end
95
107
  end
96
108
  end
109
+
110
+ def needs_migration?(statefile)
111
+ !!YAML.load(File.open(statefile).read)
112
+ rescue Psych::SyntaxError, Errno::ENOENT
113
+ false
114
+ end
data/sufia-models.gemspec CHANGED
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
33
33
  spec.add_dependency "active-fedora", "~> 9.1", ">= 9.1.1"
34
34
  spec.add_dependency "hydra-collections", [">= 5.0.2", "< 6.0"]
35
35
  spec.add_dependency 'hydra-derivatives', '~> 1.0'
36
- spec.add_dependency 'active_fedora-noid', '~> 0.1'
36
+ spec.add_dependency 'active_fedora-noid', '~> 1.0'
37
37
  spec.add_dependency 'nest', '~> 1.1'
38
38
  spec.add_dependency 'resque', '~> 1.23'
39
39
  spec.add_dependency 'resque-pool', '~> 0.3'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sufia-models
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.2.0
4
+ version: 6.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Friesen
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-07-09 00:00:00.000000000 Z
11
+ date: 2015-08-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -140,14 +140,14 @@ dependencies:
140
140
  requirements:
141
141
  - - "~>"
142
142
  - !ruby/object:Gem::Version
143
- version: '0.1'
143
+ version: '1.0'
144
144
  type: :runtime
145
145
  prerelease: false
146
146
  version_requirements: !ruby/object:Gem::Requirement
147
147
  requirements:
148
148
  - - "~>"
149
149
  - !ruby/object:Gem::Version
150
- version: '0.1'
150
+ version: '1.0'
151
151
  - !ruby/object:Gem::Dependency
152
152
  name: nest
153
153
  requirement: !ruby/object:Gem::Requirement
@@ -482,7 +482,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
482
482
  version: '0'
483
483
  requirements: []
484
484
  rubyforge_project:
485
- rubygems_version: 2.4.6
485
+ rubygems_version: 2.4.8
486
486
  signing_key:
487
487
  specification_version: 4
488
488
  summary: Models and services for sufia