geoblacklight_admin 0.0.1 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +59 -36
- data/Rakefile +47 -14
- data/app/assets/javascripts/geoblacklight_admin.js +2 -1
- data/app/assets/stylesheets/geoblacklight_admin/_core.scss +5 -0
- data/app/assets/stylesheets/geoblacklight_admin/modules/_chosen.scss +425 -0
- data/app/assets/stylesheets/geoblacklight_admin/modules/_forms.scss +108 -0
- data/app/controllers/admin/admin_controller.rb +4 -2
- data/app/controllers/admin/advanced_search_controller.rb +123 -103
- data/app/controllers/admin/api_controller.rb +123 -108
- data/app/controllers/admin/bookmarks_controller.rb +4 -4
- data/app/controllers/admin/bulk_actions_controller.rb +9 -4
- data/app/controllers/admin/concerns/upload_util.rb +2 -0
- data/app/controllers/admin/document_accesses_controller.rb +10 -4
- data/app/controllers/admin/document_assets_controller.rb +14 -11
- data/app/controllers/admin/document_downloads_controller.rb +12 -10
- data/app/controllers/admin/documents_controller.rb +32 -15
- data/app/controllers/admin/elements_controller.rb +6 -5
- data/app/controllers/admin/form_elements_controller.rb +4 -4
- data/app/controllers/admin/ids_controller.rb +107 -87
- data/app/controllers/admin/import_documents_controller.rb +1 -1
- data/app/controllers/admin/imports_controller.rb +7 -3
- data/app/controllers/admin/mappings_controller.rb +6 -4
- data/app/controllers/admin/notifications_controller.rb +1 -1
- data/app/controllers/admin/search_controller.rb +3 -2
- data/app/controllers/admin/users_controller.rb +2 -2
- data/app/helpers/form_input_helper.rb +4 -4
- data/app/helpers/geoblacklight_admin_helper.rb +1 -1
- data/app/indexers/document_indexer.rb +1 -1
- data/app/jobs/export_job.rb +9 -6
- data/app/jobs/export_json_bulk_job.rb +95 -0
- data/app/jobs/export_json_job.rb +14 -12
- data/app/models/admin/bookmark.rb +18 -0
- data/app/models/api_search_builder.rb +3 -1
- data/app/models/blacklight_api.rb +6 -6
- data/app/models/blacklight_api_facets.rb +7 -2
- data/app/models/blacklight_api_ids.rb +3 -4
- data/app/models/bulk_action.rb +1 -1
- data/app/models/document/bbox_validator.rb +9 -5
- data/app/models/document/date_range_validator.rb +13 -10
- data/app/models/document/geom_validator.rb +4 -2
- data/app/models/document.rb +15 -7
- data/app/models/element.rb +9 -7
- data/app/models/form_control.rb +2 -0
- data/app/models/form_element.rb +2 -0
- data/app/models/form_feature.rb +2 -0
- data/app/models/form_group.rb +2 -0
- data/app/models/form_header.rb +2 -0
- data/app/models/geoblacklight_admin/schema.rb +5 -2
- data/app/models/geoblacklight_admin.rb +2 -0
- data/app/models/import/csv_duplicates_validator.rb +2 -1
- data/app/models/import/csv_header_validator.rb +2 -1
- data/app/models/import.rb +4 -7
- data/app/models/user.rb +1 -1
- data/app/services/export_csv_service.rb +5 -1
- data/app/views/admin/advanced_search/facets.json.jbuilder +9 -8
- data/app/views/admin/advanced_search/index.json.jbuilder +8 -7
- data/app/views/admin/api/_field.json.jbuilder +1 -1
- data/app/views/admin/api/fetch.json.jbuilder +8 -7
- data/app/views/admin/api/index.json.jbuilder +8 -7
- data/app/views/admin/bookmarks/index.html.erb +2 -2
- data/app/views/admin/bulk_actions/index.html.erb +3 -3
- data/app/views/admin/bulk_actions/show.html.erb +1 -1
- data/app/views/admin/document_accesses/_form.html.erb +1 -1
- data/app/views/admin/document_accesses/destroy_all.html.erb +3 -3
- data/app/views/admin/document_accesses/edit.html.erb +1 -1
- data/app/views/admin/document_accesses/import.html.erb +2 -2
- data/app/views/admin/document_accesses/index.html.erb +9 -9
- data/app/views/admin/document_accesses/new.html.erb +1 -1
- data/app/views/admin/document_downloads/_form.html.erb +1 -1
- data/app/views/admin/document_downloads/destroy_all.html.erb +2 -2
- data/app/views/admin/document_downloads/edit.html.erb +2 -3
- data/app/views/admin/document_downloads/import.html.erb +1 -1
- data/app/views/admin/document_downloads/index.html.erb +9 -9
- data/app/views/admin/document_downloads/new.html.erb +1 -1
- data/app/views/admin/document_downloads/show.html.erb +2 -2
- data/app/views/admin/documents/_document_bookmark.html.erb +1 -1
- data/app/views/admin/documents/_form_nav.html.erb +3 -1
- data/app/views/admin/documents/_json_btaa_aardvark.jbuilder +1 -3
- data/app/views/admin/documents/_json_file.jbuilder +14 -0
- data/app/views/admin/documents/_result_selected_options.html.erb +3 -1
- data/app/views/admin/documents/edit.html.erb +1 -1
- data/app/views/admin/documents/features/_multiple_download_links.html.erb +2 -2
- data/app/views/admin/documents/fetch.json_file.jbuilder +9 -0
- data/app/views/admin/documents/index.html.erb +2 -4
- data/app/views/admin/documents/index.json_file.jbuilder +9 -0
- data/app/views/admin/documents/versions.html.erb +1 -1
- data/app/views/admin/elements/_element.json.jbuilder +4 -1
- data/app/views/admin/elements/_form.html.erb +1 -1
- data/app/views/admin/elements/edit.html.erb +2 -2
- data/app/views/admin/elements/index.html.erb +4 -4
- data/app/views/admin/elements/index.json.jbuilder +2 -0
- data/app/views/admin/elements/new.html.erb +1 -1
- data/app/views/admin/elements/show.html.erb +6 -6
- data/app/views/admin/elements/show.json.jbuilder +2 -0
- data/app/views/admin/form_elements/_form.html.erb +1 -1
- data/app/views/admin/form_elements/_form_element.json.jbuilder +2 -0
- data/app/views/admin/form_elements/edit.html.erb +2 -2
- data/app/views/admin/form_elements/index.html.erb +2 -2
- data/app/views/admin/form_elements/index.json.jbuilder +2 -0
- data/app/views/admin/form_elements/new.html.erb +1 -1
- data/app/views/admin/form_elements/show.html.erb +2 -2
- data/app/views/admin/form_elements/show.json.jbuilder +2 -0
- data/app/views/admin/ids/fetch.json.jbuilder +8 -7
- data/app/views/admin/ids/index.json.jbuilder +3 -2
- data/app/views/admin/import_documents/show.html.haml +1 -1
- data/app/views/admin/imports/_import.json.jbuilder +2 -1
- data/app/views/admin/imports/_show_failed_tab.html.erb +2 -2
- data/app/views/admin/imports/_show_success_tab.html.erb +1 -1
- data/app/views/admin/imports/edit.html.erb +3 -3
- data/app/views/admin/imports/index.html.erb +3 -3
- data/app/views/admin/layouts/application.html.erb +4 -4
- data/app/views/admin/mappings/_form.html.erb +1 -1
- data/app/views/admin/mappings/_mapping.json.jbuilder +2 -1
- data/app/views/admin/mappings/edit.html.erb +2 -2
- data/app/views/admin/mappings/new.html.erb +1 -1
- data/app/views/admin/mappings/show.html.erb +2 -2
- data/app/views/admin/notifications/index.html.erb +3 -3
- data/app/views/admin/search/index.html.erb +2 -2
- data/app/views/admin/shared/_navbar.html.erb +11 -6
- data/app/views/admin/shared/_toast.html.erb +1 -1
- data/app/views/admin/users/index.html.erb +1 -1
- data/app/views/catalog/_show_gbl_admin.html.erb +3 -0
- data/config/locales/documents.en.yml +3 -0
- data/config/routes.rb +37 -33
- data/db/migrate/20230316183001_add_geoblacklight_admin_gem.rb +409 -0
- data/db/seeds.rb +7 -6
- data/db/seeds_elements.csv +56 -55
- data/db/seeds_elements.numbers +0 -0
- data/db/seeds_form_elements.csv +65 -64
- data/db/seeds_form_elements.numbers +0 -0
- data/lib/generators/geoblacklight_admin/config_generator.rb +61 -94
- data/lib/generators/geoblacklight_admin/install_generator.rb +3 -12
- data/lib/generators/geoblacklight_admin/jobs_generator.rb +3 -2
- data/lib/generators/geoblacklight_admin/models_generator.rb +0 -1
- data/lib/generators/geoblacklight_admin/templates/_show_sidebar.html.erb +19 -0
- data/lib/generators/geoblacklight_admin/templates/_user_util_links.html.erb +1 -1
- data/lib/generators/geoblacklight_admin/templates/config/database.yml +9 -5
- data/lib/generators/geoblacklight_admin/templates/config/geomg_aardvark_schema.json +12 -6
- data/lib/generators/geoblacklight_admin/templates/config/initializers/devise.rb +315 -0
- data/lib/generators/geoblacklight_admin/templates/config/initializers/geoblacklight_admin.rb +1 -2
- data/lib/generators/geoblacklight_admin/templates/config/initializers/kithe.rb +2 -0
- data/lib/generators/geoblacklight_admin/templates/config/initializers/mime_types.rb +15 -0
- data/lib/generators/geoblacklight_admin/templates/config/initializers/pagy.rb +152 -68
- data/lib/generators/geoblacklight_admin/templates/config/initializers/simple_form.rb +178 -0
- data/lib/generators/geoblacklight_admin/templates/config/initializers/simple_form_bootstrap.rb +365 -0
- data/lib/generators/geoblacklight_admin/templates/config/settings.yml +293 -0
- data/lib/generators/geoblacklight_admin/templates/javascript/controllers/results_controller.js +10 -0
- data/lib/generators/geoblacklight_admin/templates/package.json +3 -0
- data/lib/generators/geoblacklight_admin/templates/solr/conf/schema.xml +3 -3
- data/lib/generators/geoblacklight_admin/views_generator.rb +2 -3
- data/lib/geoblacklight_admin/engine.rb +5 -6
- data/lib/geoblacklight_admin/version.rb +1 -1
- data/lib/geoblacklight_admin.rb +1 -1
- data/lib/tasks/geoblacklight_admin.rake +15 -11
- metadata +101 -63
- data/.editorconfig +0 -18
- data/.gitignore +0 -75
- data/.standard.yml +0 -4
- data/Gemfile +0 -59
- data/LICENSE +0 -21
- data/app/assets/images/.keep +0 -0
- data/app/controllers/admin/concerns/.keep +0 -0
- data/app/models/bookmark.rb +0 -17
- data/app/models/concerns/.keep +0 -0
- data/db/migrate/20230316183001_add_geoblacklight_admin.rb +0 -399
- data/geoblacklight_admin.gemspec +0 -67
- data/lib/generators/geoblacklight_admin/example_docs_generator.rb +0 -18
- data/lib/generators/geoblacklight_admin/templates/.env.development.example +0 -28
- data/lib/generators/geoblacklight_admin/templates/.solr_wrapper.yml +0 -7
- data/lib/generators/geoblacklight_admin/templates/solr/snapshots/.keep +0 -0
- data/lib/tasks/.keep +0 -0
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +0 -35
- data/template.rb +0 -16
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/confirmations/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/invitations/edit.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/invitations/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/mailer/invitation_instructions.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/mailer/invitation_instructions.text.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/passwords/edit.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/passwords/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/registrations/edit.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/registrations/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/sessions/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/shared/_links.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/unlocks/new.html.erb +0 -0
@@ -34,7 +34,10 @@ module Admin
|
|
34
34
|
|
35
35
|
respond_to do |format|
|
36
36
|
if @import.save
|
37
|
-
format.html
|
37
|
+
format.html do
|
38
|
+
redirect_to admin_import_mappings_path(@import),
|
39
|
+
notice: "Import was successful. Please set your import mapping rules."
|
40
|
+
end
|
38
41
|
format.json { render :show, status: :created, location: @import }
|
39
42
|
else
|
40
43
|
format.html { render :new }
|
@@ -82,7 +85,8 @@ module Admin
|
|
82
85
|
# Never trust parameters from the scary internet, only allow the white list through.
|
83
86
|
|
84
87
|
def permittable_params
|
85
|
-
%i[type name filename source description row_count encoding content_type extension validity validation_result
|
88
|
+
%i[type name filename source description row_count encoding content_type extension validity validation_result
|
89
|
+
csv_file run]
|
86
90
|
end
|
87
91
|
|
88
92
|
def import_params
|
@@ -102,4 +106,4 @@ module Admin
|
|
102
106
|
)
|
103
107
|
end
|
104
108
|
end
|
105
|
-
end
|
109
|
+
end
|
@@ -37,7 +37,9 @@ module Admin
|
|
37
37
|
|
38
38
|
respond_to do |format|
|
39
39
|
if @mapping.save
|
40
|
-
format.html
|
40
|
+
format.html do
|
41
|
+
redirect_to admin_import_mapping_path(@import, @mapping), notice: "Mapping was successfully created."
|
42
|
+
end
|
41
43
|
format.json { render :show, status: :created, location: @mapping }
|
42
44
|
else
|
43
45
|
format.html { render :new }
|
@@ -51,7 +53,7 @@ module Admin
|
|
51
53
|
def update
|
52
54
|
respond_to do |format|
|
53
55
|
if @mapping.update(mapping_params)
|
54
|
-
format.html { redirect_to
|
56
|
+
format.html { redirect_to admin_import_mappings_path(@mapping.import), notice: "Mapping was successfully updated." }
|
55
57
|
format.json { render :show, status: :ok, location: @mapping }
|
56
58
|
else
|
57
59
|
format.html { render :edit }
|
@@ -65,7 +67,7 @@ module Admin
|
|
65
67
|
def destroy
|
66
68
|
@mapping.destroy
|
67
69
|
respond_to do |format|
|
68
|
-
format.html { redirect_to
|
70
|
+
format.html { redirect_to admin_import_mappings_url(@import), notice: "Mapping was successfully destroyed." }
|
69
71
|
format.json { head :no_content }
|
70
72
|
end
|
71
73
|
end
|
@@ -92,4 +94,4 @@ module Admin
|
|
92
94
|
)
|
93
95
|
end
|
94
96
|
end
|
95
|
-
end
|
97
|
+
end
|
@@ -4,7 +4,8 @@
|
|
4
4
|
module Admin
|
5
5
|
class SearchController < Admin::AdminController
|
6
6
|
def index
|
7
|
-
@
|
7
|
+
@request = "#{request.protocol}#{request.host}:#{request.port}"
|
8
|
+
@facet_options = BlacklightApiFacets.new(@request).facets
|
8
9
|
end
|
9
10
|
end
|
10
|
-
end
|
11
|
+
end
|
@@ -47,7 +47,7 @@ module FormInputHelper
|
|
47
47
|
category_list = [existing_category_value] + category_list unless category_list.include?(existing_category_value)
|
48
48
|
|
49
49
|
# Turn category list into values and human labels, using i18n or rails humanizing.
|
50
|
-
category_list = category_list.collect
|
50
|
+
category_list = category_list.collect do |key|
|
51
51
|
value = if key.nil?
|
52
52
|
key
|
53
53
|
elsif model_class.respond_to?(:model_name) && key.present?
|
@@ -59,15 +59,15 @@ module FormInputHelper
|
|
59
59
|
end
|
60
60
|
|
61
61
|
[value, key]
|
62
|
-
|
62
|
+
end.to_h
|
63
63
|
|
64
64
|
tag.div(class: "form-row category-and-value") do
|
65
65
|
tag.div(class: "col-left category") {
|
66
66
|
builder.input category_key, collection: category_list, label: false, include_blank: false
|
67
67
|
} +
|
68
|
-
tag.div(class: "col-sm value")
|
68
|
+
tag.div(class: "col-sm value") do
|
69
69
|
builder.input value_key, label: false, input_html: {data: input_data_attributes}
|
70
|
-
|
70
|
+
end
|
71
71
|
end
|
72
72
|
end
|
73
73
|
end
|
@@ -5,7 +5,7 @@ module GeoblacklightAdminHelper
|
|
5
5
|
# @TODO:
|
6
6
|
# Cannot generate app if uncommented...
|
7
7
|
# Uncomment after app is generated to fix view errors
|
8
|
-
|
8
|
+
include ::Pagy::Frontend if defined?(Pagy)
|
9
9
|
|
10
10
|
# jbuilder helper
|
11
11
|
def no_json_blanks(value)
|
data/app/jobs/export_job.rb
CHANGED
@@ -6,8 +6,9 @@ require "csv"
|
|
6
6
|
class ExportJob < ApplicationJob
|
7
7
|
queue_as :default
|
8
8
|
|
9
|
-
def perform(current_user, query_params, export_service)
|
9
|
+
def perform(request, current_user, query_params, export_service)
|
10
10
|
logger.debug("\n\n Background Job: ♞")
|
11
|
+
logger.debug("Request: #{request.inspect}")
|
11
12
|
logger.debug("User: #{current_user.inspect}")
|
12
13
|
logger.debug("Query: #{query_params.inspect}")
|
13
14
|
logger.debug("Export Service: #{export_service.inspect}")
|
@@ -17,7 +18,7 @@ class ExportJob < ApplicationJob
|
|
17
18
|
ActionCable.server.broadcast("export_channel", {data: "Hello from Export Job!"})
|
18
19
|
|
19
20
|
# Query params into Doc ids
|
20
|
-
document_ids = query_params[:ids] || crawl_query(query_params)
|
21
|
+
document_ids = query_params[:ids] || crawl_query(request, query_params)
|
21
22
|
|
22
23
|
logger.debug("Document Ids: #{document_ids}")
|
23
24
|
|
@@ -41,7 +42,8 @@ class ExportJob < ApplicationJob
|
|
41
42
|
notification.deliver(current_user)
|
42
43
|
|
43
44
|
# Attach CSV file (can only attach after persisted)
|
44
|
-
notification.record.file.attach(io: @tempfile, filename: "geomg-export-#{Time.zone.today}.csv",
|
45
|
+
notification.record.file.attach(io: @tempfile, filename: "geomg-export-#{Time.zone.today}.csv",
|
46
|
+
content_type: "text/csv")
|
45
47
|
|
46
48
|
# Update UI
|
47
49
|
ActionCable.server.broadcast("export_channel", {
|
@@ -55,15 +57,16 @@ class ExportJob < ApplicationJob
|
|
55
57
|
})
|
56
58
|
end
|
57
59
|
|
58
|
-
def crawl_query(query_params, doc_ids = [])
|
60
|
+
def crawl_query(request, query_params, doc_ids = [])
|
59
61
|
logger.debug("\n\n CRAWL Query: #{query_params}")
|
60
|
-
|
62
|
+
logger.debug("\n\n CRAWL Query Request: #{request}")
|
63
|
+
api_results = BlacklightApiIds.new(request, query_params)
|
61
64
|
logger.debug("API Results: #{api_results.results.inspect}")
|
62
65
|
|
63
66
|
doc_ids << api_results.results.pluck("id")
|
64
67
|
|
65
68
|
unless api_results.meta["pages"]["next_page"].nil?
|
66
|
-
crawl_query(query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
69
|
+
crawl_query(request, query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
67
70
|
doc_ids)
|
68
71
|
end
|
69
72
|
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "csv"
|
4
|
+
|
5
|
+
# ExportJsonBulkJob
|
6
|
+
class ExportJsonBulkJob < ApplicationJob
|
7
|
+
queue_as :default
|
8
|
+
|
9
|
+
def perform(request, current_user, query_params, export_service)
|
10
|
+
logger.debug("\n\n Background Job: ♞")
|
11
|
+
logger.debug("Request: #{request.inspect}")
|
12
|
+
logger.debug("User: #{current_user.inspect}")
|
13
|
+
logger.debug("Query: #{query_params.inspect}")
|
14
|
+
logger.debug("Export Service: #{export_service.inspect}")
|
15
|
+
logger.debug("\n\n")
|
16
|
+
|
17
|
+
# Test broadcast
|
18
|
+
ActionCable.server.broadcast("export_channel", {data: "Hello from Export Job!"})
|
19
|
+
|
20
|
+
# Query params into Doc ids
|
21
|
+
document_ids = query_params[:ids] || crawl_query(request, query_params)
|
22
|
+
|
23
|
+
logger.debug("Document Ids: #{document_ids}")
|
24
|
+
|
25
|
+
# Send progress
|
26
|
+
documents = export_service.call(document_ids)
|
27
|
+
|
28
|
+
begin
|
29
|
+
# Array of JSON
|
30
|
+
@json_array = []
|
31
|
+
|
32
|
+
documents.each do |doc|
|
33
|
+
json_output = Admin::DocumentsController.render("_json_file.jbuilder",
|
34
|
+
locals: {document: doc})
|
35
|
+
|
36
|
+
json_obj = JSON.parse(json_output)
|
37
|
+
Rails.logger.debug json_obj
|
38
|
+
|
39
|
+
# Remove nil/null values from JSON
|
40
|
+
json_obj.compact!
|
41
|
+
|
42
|
+
@json_array << JSON.pretty_generate(json_obj)
|
43
|
+
|
44
|
+
rescue NoMethodError => e
|
45
|
+
Rails.logger.debug { "==== Error! - #{doc.friendlier_id} ====" }
|
46
|
+
Rails.logger.debug e.inspect
|
47
|
+
next
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Write into tempfile
|
52
|
+
@tempfile = Tempfile.new(["export-#{Time.zone.today}", ".json"]).tap do |file|
|
53
|
+
file.write("[#{@json_array.join(",")}]")
|
54
|
+
end
|
55
|
+
@tempfile.rewind
|
56
|
+
|
57
|
+
# Create notification
|
58
|
+
# Message: "Download Type|Row Count|Button Label"
|
59
|
+
notification = ExportNotification.with(message: "JSON FILE |#{ActionController::Base.helpers.number_with_delimiter(documents.size)} rows|JSON")
|
60
|
+
|
61
|
+
# Deliver notification
|
62
|
+
notification.deliver(current_user)
|
63
|
+
|
64
|
+
# Attach JSON file (can only attach after persisted)
|
65
|
+
notification.record.file.attach(io: @tempfile, filename: "geomg-export-#{Time.zone.today}.json",
|
66
|
+
content_type: "application/json")
|
67
|
+
|
68
|
+
# Update UI
|
69
|
+
ActionCable.server.broadcast("export_channel", {
|
70
|
+
data: "Notification ready!",
|
71
|
+
actions: [
|
72
|
+
{
|
73
|
+
method: "RefreshNotifications",
|
74
|
+
payload: current_user.notifications.unread.count
|
75
|
+
}
|
76
|
+
]
|
77
|
+
})
|
78
|
+
end
|
79
|
+
|
80
|
+
def crawl_query(request, query_params, doc_ids = [])
|
81
|
+
logger.debug("\n\n CRAWL Query: #{query_params}")
|
82
|
+
logger.debug("\n\n CRAWL Query Request: #{request}")
|
83
|
+
api_results = BlacklightApiIds.new(request, query_params)
|
84
|
+
logger.debug("API Results: #{api_results.results.inspect}")
|
85
|
+
|
86
|
+
doc_ids << api_results.results.pluck("id")
|
87
|
+
|
88
|
+
unless api_results.meta["pages"]["next_page"].nil?
|
89
|
+
crawl_query(request, query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
90
|
+
doc_ids)
|
91
|
+
end
|
92
|
+
|
93
|
+
doc_ids
|
94
|
+
end
|
95
|
+
end
|
data/app/jobs/export_json_job.rb
CHANGED
@@ -3,22 +3,23 @@
|
|
3
3
|
require "json"
|
4
4
|
require "pathname"
|
5
5
|
|
6
|
-
#
|
6
|
+
# ExportJsonJob
|
7
7
|
class ExportJsonJob < ApplicationJob
|
8
8
|
queue_as :default
|
9
9
|
|
10
|
-
def perform(current_user, query_params, export_service)
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
10
|
+
def perform(request, current_user, query_params, export_service)
|
11
|
+
logger.debug("\n\n Background Job: ♞")
|
12
|
+
logger.debug("Request: #{request.inspect}")
|
13
|
+
logger.debug("User: #{current_user.inspect}")
|
14
|
+
logger.debug("Query: #{query_params.inspect}")
|
15
|
+
logger.debug("Export Service: #{export_service.inspect}")
|
16
|
+
logger.debug("\n\n")
|
16
17
|
|
17
18
|
# Test broadcast
|
18
19
|
ActionCable.server.broadcast("export_channel", {data: "Hello from Export JSON Job!"})
|
19
20
|
|
20
21
|
# Query params into Doc ids
|
21
|
-
document_ids = query_params[:ids] || crawl_query(query_params)
|
22
|
+
document_ids = query_params[:ids] || crawl_query(request, query_params)
|
22
23
|
|
23
24
|
# Rails.logger.debug("Document Ids: #{document_ids}")
|
24
25
|
|
@@ -47,7 +48,7 @@ class ExportJsonJob < ApplicationJob
|
|
47
48
|
tree.dirname.mkpath
|
48
49
|
Rails.logger.debug tree.inspect
|
49
50
|
|
50
|
-
json_output = DocumentsController.render("_#{query_params[:format]}",
|
51
|
+
json_output = Admin::DocumentsController.render("_#{query_params[:format]}",
|
51
52
|
locals: {document: doc})
|
52
53
|
|
53
54
|
json_obj = JSON.parse(json_output)
|
@@ -103,15 +104,16 @@ class ExportJsonJob < ApplicationJob
|
|
103
104
|
})
|
104
105
|
end
|
105
106
|
|
106
|
-
def crawl_query(query_params, doc_ids = [])
|
107
|
+
def crawl_query(request, query_params, doc_ids = [])
|
107
108
|
logger.debug("\n\n CRAWL Query: #{query_params}")
|
108
|
-
|
109
|
+
logger.debug("\n\n CRAWL Query Request: #{request}")
|
110
|
+
api_results = BlacklightApiIds.new(request, query_params)
|
109
111
|
logger.debug("API Results: #{api_results.results.inspect}")
|
110
112
|
|
111
113
|
doc_ids << api_results.results.pluck("id")
|
112
114
|
|
113
115
|
unless api_results.meta["pages"]["next_page"].nil?
|
114
|
-
crawl_query(query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
116
|
+
crawl_query(request, query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
115
117
|
doc_ids)
|
116
118
|
end
|
117
119
|
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Admin::Bookmark
|
4
|
+
module Admin
|
5
|
+
class Bookmark < ApplicationRecord
|
6
|
+
belongs_to :user, polymorphic: true
|
7
|
+
belongs_to :document, polymorphic: true
|
8
|
+
validates :user_id, presence: true
|
9
|
+
|
10
|
+
def document_id
|
11
|
+
document_id
|
12
|
+
end
|
13
|
+
|
14
|
+
def document_type
|
15
|
+
Kithe::Model
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -1,9 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
class ApiSearchBuilder < Blacklight::SearchBuilder
|
2
4
|
include Blacklight::Solr::SearchBuilderBehavior
|
3
5
|
include BlacklightAdvancedSearch::AdvancedSearchBuilder
|
4
6
|
# include B1gDateRangeQueryConcern::RangeLimitBuilder
|
5
7
|
|
6
|
-
self.default_processor_chain += [
|
8
|
+
self.default_processor_chain += %i[add_advanced_parse_q_to_solr add_advanced_search_to_solr]
|
7
9
|
|
8
10
|
# Remove GBL Spatial, otherwise it will hide all suppressed docs
|
9
11
|
# from API search results
|
@@ -1,28 +1,28 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "httparty"
|
4
|
+
|
3
5
|
# BlacklightApi
|
4
6
|
class BlacklightApi
|
5
7
|
include HTTParty
|
6
8
|
default_timeout 300
|
7
9
|
|
8
|
-
|
9
|
-
|
10
|
-
def initialize(**args)
|
10
|
+
def initialize(request, args)
|
11
11
|
defaults = {
|
12
12
|
q: "*",
|
13
13
|
page: 1,
|
14
14
|
sort: "score+desc%2C+dc_title_sort+asc",
|
15
15
|
rows: 20
|
16
16
|
}
|
17
|
-
|
18
|
-
@options = defaults.merge(
|
17
|
+
@request = request
|
18
|
+
@options = defaults.merge(args)
|
19
19
|
append_facets(@options[:f], @options)
|
20
20
|
append_daterange(@options[:f], @options)
|
21
21
|
@options.compact!
|
22
22
|
end
|
23
23
|
|
24
24
|
def fetch
|
25
|
-
@fetch ||= self.class.get("
|
25
|
+
@fetch ||= self.class.get("#{@request}#{BLACKLIGHT_JSON_API}", query: @options)
|
26
26
|
end
|
27
27
|
|
28
28
|
def results
|
@@ -1,12 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "httparty"
|
4
|
+
|
3
5
|
# BlacklightApiFacets
|
4
6
|
class BlacklightApiFacets
|
5
7
|
include HTTParty
|
6
|
-
|
8
|
+
|
9
|
+
def initialize(request)
|
10
|
+
@request = request
|
11
|
+
end
|
7
12
|
|
8
13
|
def fetch
|
9
|
-
@fetch ||= self.class.get("
|
14
|
+
@fetch ||= self.class.get("#{@request}#{BLACKLIGHT_JSON_API_FACETS}", query: {})
|
10
15
|
end
|
11
16
|
|
12
17
|
def facets
|
@@ -5,9 +5,7 @@ class BlacklightApiIds
|
|
5
5
|
include HTTParty
|
6
6
|
default_timeout 300
|
7
7
|
|
8
|
-
|
9
|
-
|
10
|
-
def initialize(args = {})
|
8
|
+
def initialize(request, args = {})
|
11
9
|
defaults = {
|
12
10
|
q: "*",
|
13
11
|
page: 1,
|
@@ -15,6 +13,7 @@ class BlacklightApiIds
|
|
15
13
|
rows: 1000
|
16
14
|
}
|
17
15
|
|
16
|
+
@request = request
|
18
17
|
@options = defaults.merge(**args)
|
19
18
|
append_facets(@options[:f], @options)
|
20
19
|
append_daterange(@options[:f], @options)
|
@@ -24,7 +23,7 @@ class BlacklightApiIds
|
|
24
23
|
def fetch
|
25
24
|
Rails.logger.debug { "BlacklightApiIds > fetch > query: #{@options.inspect}" }
|
26
25
|
|
27
|
-
@fetch ||= self.class.get("
|
26
|
+
@fetch ||= self.class.get("#{@request}#{BLACKLIGHT_JSON_API_IDS}", query: @options)
|
28
27
|
end
|
29
28
|
|
30
29
|
def results
|
data/app/models/bulk_action.rb
CHANGED
@@ -69,7 +69,7 @@ class BulkAction < ApplicationRecord
|
|
69
69
|
def api_documents(uri)
|
70
70
|
qargs = Rack::Utils.parse_nested_query(uri.query)
|
71
71
|
query_params = {q: qargs["q"], f: qargs["f"], page: qargs["page"], rows: 1_000_000}
|
72
|
-
api_documents = BlacklightApiIds.new(**query_params)
|
72
|
+
api_documents = BlacklightApiIds.new(request, **query_params)
|
73
73
|
create_documents(api_documents.load_all)
|
74
74
|
end
|
75
75
|
|
@@ -14,7 +14,10 @@ class Document
|
|
14
14
|
valid_geom = true
|
15
15
|
|
16
16
|
# Sane for W,S,E,N?
|
17
|
-
|
17
|
+
unless record.send(GeoblacklightAdmin::Schema.instance.solr_fields[:bounding_box]).nil?
|
18
|
+
proper_bounding_box(record,
|
19
|
+
valid_geom)
|
20
|
+
end
|
18
21
|
|
19
22
|
valid_geom
|
20
23
|
end
|
@@ -57,10 +60,11 @@ class Document
|
|
57
60
|
# Reject ENVELOPE(-118.00.0000,-88.00.0000,51.00.0000,42.00.0000
|
58
61
|
# - Double period float-ish things?
|
59
62
|
geom.each do |val|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
63
|
+
next unless val.count(".") >= 2
|
64
|
+
|
65
|
+
valid_geom = false
|
66
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:bounding_box],
|
67
|
+
"invalid ENVELOPE(W,E,N,S) syntax - found multiple periods in a coordinate value.")
|
64
68
|
end
|
65
69
|
end
|
66
70
|
|
@@ -18,7 +18,8 @@ class Document
|
|
18
18
|
|
19
19
|
# Must split into two via '-'
|
20
20
|
if date_range.split("-").size != 2
|
21
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
21
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
22
|
+
"invalid date range present - missing YYYY-YYYY value")
|
22
23
|
valid_date_ranges = false
|
23
24
|
return valid_date_ranges
|
24
25
|
end
|
@@ -26,28 +27,30 @@ class Document
|
|
26
27
|
start_date, end_date = date_range.split("-")
|
27
28
|
|
28
29
|
unless valid_yyyy(start_date)
|
29
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
30
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
31
|
+
"invalid start date - bad YYYY-YYYY value, only integers or the wildcard '*' allowed")
|
30
32
|
valid_date_ranges = false
|
31
33
|
end
|
32
34
|
|
33
35
|
unless valid_yyyy(end_date)
|
34
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
36
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
37
|
+
"invalid end date - bad YYYY-YYYY value, only integers or the wildcard '*' allowed")
|
35
38
|
valid_date_ranges = false
|
36
39
|
end
|
37
40
|
|
38
41
|
# Allow only 1 wildcard
|
39
42
|
if start_date == "*" && end_date == "*"
|
40
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
43
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
44
|
+
"invalid only one wildcard can be used")
|
41
45
|
valid_date_ranges = false
|
42
46
|
end
|
43
47
|
|
44
48
|
# Avoid 1996-1946
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
end
|
49
|
+
next unless start_date[/\d+/] && end_date[/\d+/] && start_date.to_i > end_date.to_i
|
50
|
+
|
51
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
52
|
+
"invalid start date must be less than the end date")
|
53
|
+
valid_date_ranges = false
|
51
54
|
end
|
52
55
|
valid_date_ranges
|
53
56
|
end
|
@@ -37,7 +37,8 @@ class Document
|
|
37
37
|
end
|
38
38
|
|
39
39
|
unless valid_geom
|
40
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:geometry],
|
40
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:geometry],
|
41
|
+
"Invalid envelope: #{error_message}")
|
41
42
|
end
|
42
43
|
|
43
44
|
valid_geom
|
@@ -60,7 +61,8 @@ class Document
|
|
60
61
|
# Guard against a whole world polygons
|
61
62
|
if geom == "POLYGON((-180 90, 180 90, 180 -90, -180 -90, -180 90))"
|
62
63
|
valid_geom = false
|
63
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:geometry],
|
64
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:geometry],
|
65
|
+
"Invalid polygon: all points are coplanar input, Solr will not index")
|
64
66
|
end
|
65
67
|
|
66
68
|
valid_geom
|
data/app/models/document.rb
CHANGED
@@ -11,7 +11,8 @@ class Document < Kithe::Work
|
|
11
11
|
belongs_to :import, optional: true
|
12
12
|
|
13
13
|
# Statesman
|
14
|
-
has_many :document_transitions, foreign_key: "kithe_model_id", autosave: false, dependent: :destroy,
|
14
|
+
has_many :document_transitions, foreign_key: "kithe_model_id", autosave: false, dependent: :destroy,
|
15
|
+
inverse_of: :document
|
15
16
|
|
16
17
|
# DocumentAccesses
|
17
18
|
has_many :document_accesses, primary_key: "friendlier_id", foreign_key: "friendlier_id", autosave: false, dependent: :destroy,
|
@@ -69,7 +70,9 @@ class Document < Kithe::Work
|
|
69
70
|
# Index Transformations - *_json functions
|
70
71
|
def references_json
|
71
72
|
references = ActiveSupport::HashWithIndifferentAccess.new
|
72
|
-
send(GeoblacklightAdmin::Schema.instance.solr_fields[:reference]).each
|
73
|
+
send(GeoblacklightAdmin::Schema.instance.solr_fields[:reference]).each do |ref|
|
74
|
+
references[Document::Reference::REFERENCE_VALUES[ref.category.to_sym][:uri]] = ref.value
|
75
|
+
end
|
73
76
|
references = apply_downloads(references)
|
74
77
|
references.to_json
|
75
78
|
end
|
@@ -79,7 +82,10 @@ class Document < Kithe::Work
|
|
79
82
|
# Make sure downloads exist!
|
80
83
|
if document_downloads.present?
|
81
84
|
multiple_downloads = multiple_downloads_array
|
82
|
-
|
85
|
+
if dct_downloads.present?
|
86
|
+
multiple_downloads << {label: download_text(send(GeoblacklightAdmin::Schema.instance.solr_fields[:format])),
|
87
|
+
url: dct_downloads}
|
88
|
+
end
|
83
89
|
references[:"http://schema.org/downloadUrl"] = multiple_downloads
|
84
90
|
end
|
85
91
|
references
|
@@ -178,7 +184,9 @@ class Document < Kithe::Work
|
|
178
184
|
end
|
179
185
|
|
180
186
|
def dct_references_s_to_csv(key, destination)
|
181
|
-
send(destination).detect
|
187
|
+
send(destination).detect do |ref|
|
188
|
+
ref.category == GeoblacklightAdmin::Schema.instance.dct_references_mappings[key]
|
189
|
+
end.value
|
182
190
|
rescue NoMethodError
|
183
191
|
nil
|
184
192
|
end
|
@@ -219,9 +227,9 @@ class Document < Kithe::Work
|
|
219
227
|
end
|
220
228
|
|
221
229
|
def set_geometry
|
222
|
-
|
223
|
-
|
224
|
-
|
230
|
+
return unless locn_geometry.blank? && self&.dcat_bbox&.present?
|
231
|
+
|
232
|
+
self.locn_geometry = derive_polygon
|
225
233
|
end
|
226
234
|
|
227
235
|
# Convert GEOM for Solr Indexing
|
data/app/models/element.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
class Element < ApplicationRecord
|
2
4
|
serialize :html_attributes
|
3
5
|
|
@@ -21,13 +23,13 @@ class Element < ApplicationRecord
|
|
21
23
|
# Validations
|
22
24
|
validates :label, :solr_field, :field_type, presence: true
|
23
25
|
|
24
|
-
FIELD_TYPES = [
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
]
|
26
|
+
FIELD_TYPES = %w[
|
27
|
+
string
|
28
|
+
text
|
29
|
+
integer
|
30
|
+
boolean
|
31
|
+
datetime
|
32
|
+
].freeze
|
31
33
|
|
32
34
|
# Find by solr_field shortcut
|
33
35
|
def self.at(field)
|