geoblacklight_admin 0.0.1 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +59 -36
- data/Rakefile +47 -14
- data/app/assets/javascripts/geoblacklight_admin.js +2 -1
- data/app/assets/stylesheets/geoblacklight_admin/_core.scss +5 -0
- data/app/assets/stylesheets/geoblacklight_admin/modules/_chosen.scss +425 -0
- data/app/assets/stylesheets/geoblacklight_admin/modules/_forms.scss +108 -0
- data/app/controllers/admin/admin_controller.rb +4 -2
- data/app/controllers/admin/advanced_search_controller.rb +123 -103
- data/app/controllers/admin/api_controller.rb +123 -108
- data/app/controllers/admin/bookmarks_controller.rb +4 -4
- data/app/controllers/admin/bulk_actions_controller.rb +9 -4
- data/app/controllers/admin/concerns/upload_util.rb +2 -0
- data/app/controllers/admin/document_accesses_controller.rb +10 -4
- data/app/controllers/admin/document_assets_controller.rb +14 -11
- data/app/controllers/admin/document_downloads_controller.rb +12 -10
- data/app/controllers/admin/documents_controller.rb +32 -15
- data/app/controllers/admin/elements_controller.rb +6 -5
- data/app/controllers/admin/form_elements_controller.rb +4 -4
- data/app/controllers/admin/ids_controller.rb +107 -87
- data/app/controllers/admin/import_documents_controller.rb +1 -1
- data/app/controllers/admin/imports_controller.rb +7 -3
- data/app/controllers/admin/mappings_controller.rb +6 -4
- data/app/controllers/admin/notifications_controller.rb +1 -1
- data/app/controllers/admin/search_controller.rb +3 -2
- data/app/controllers/admin/users_controller.rb +2 -2
- data/app/helpers/form_input_helper.rb +4 -4
- data/app/helpers/geoblacklight_admin_helper.rb +1 -1
- data/app/indexers/document_indexer.rb +1 -1
- data/app/jobs/export_job.rb +9 -6
- data/app/jobs/export_json_bulk_job.rb +95 -0
- data/app/jobs/export_json_job.rb +14 -12
- data/app/models/admin/bookmark.rb +18 -0
- data/app/models/api_search_builder.rb +3 -1
- data/app/models/blacklight_api.rb +6 -6
- data/app/models/blacklight_api_facets.rb +7 -2
- data/app/models/blacklight_api_ids.rb +3 -4
- data/app/models/bulk_action.rb +1 -1
- data/app/models/document/bbox_validator.rb +9 -5
- data/app/models/document/date_range_validator.rb +13 -10
- data/app/models/document/geom_validator.rb +4 -2
- data/app/models/document.rb +15 -7
- data/app/models/element.rb +9 -7
- data/app/models/form_control.rb +2 -0
- data/app/models/form_element.rb +2 -0
- data/app/models/form_feature.rb +2 -0
- data/app/models/form_group.rb +2 -0
- data/app/models/form_header.rb +2 -0
- data/app/models/geoblacklight_admin/schema.rb +5 -2
- data/app/models/geoblacklight_admin.rb +2 -0
- data/app/models/import/csv_duplicates_validator.rb +2 -1
- data/app/models/import/csv_header_validator.rb +2 -1
- data/app/models/import.rb +4 -7
- data/app/models/user.rb +1 -1
- data/app/services/export_csv_service.rb +5 -1
- data/app/views/admin/advanced_search/facets.json.jbuilder +9 -8
- data/app/views/admin/advanced_search/index.json.jbuilder +8 -7
- data/app/views/admin/api/_field.json.jbuilder +1 -1
- data/app/views/admin/api/fetch.json.jbuilder +8 -7
- data/app/views/admin/api/index.json.jbuilder +8 -7
- data/app/views/admin/bookmarks/index.html.erb +2 -2
- data/app/views/admin/bulk_actions/index.html.erb +3 -3
- data/app/views/admin/bulk_actions/show.html.erb +1 -1
- data/app/views/admin/document_accesses/_form.html.erb +1 -1
- data/app/views/admin/document_accesses/destroy_all.html.erb +3 -3
- data/app/views/admin/document_accesses/edit.html.erb +1 -1
- data/app/views/admin/document_accesses/import.html.erb +2 -2
- data/app/views/admin/document_accesses/index.html.erb +9 -9
- data/app/views/admin/document_accesses/new.html.erb +1 -1
- data/app/views/admin/document_downloads/_form.html.erb +1 -1
- data/app/views/admin/document_downloads/destroy_all.html.erb +2 -2
- data/app/views/admin/document_downloads/edit.html.erb +2 -3
- data/app/views/admin/document_downloads/import.html.erb +1 -1
- data/app/views/admin/document_downloads/index.html.erb +9 -9
- data/app/views/admin/document_downloads/new.html.erb +1 -1
- data/app/views/admin/document_downloads/show.html.erb +2 -2
- data/app/views/admin/documents/_document_bookmark.html.erb +1 -1
- data/app/views/admin/documents/_form_nav.html.erb +3 -1
- data/app/views/admin/documents/_json_btaa_aardvark.jbuilder +1 -3
- data/app/views/admin/documents/_json_file.jbuilder +14 -0
- data/app/views/admin/documents/_result_selected_options.html.erb +3 -1
- data/app/views/admin/documents/edit.html.erb +1 -1
- data/app/views/admin/documents/features/_multiple_download_links.html.erb +2 -2
- data/app/views/admin/documents/fetch.json_file.jbuilder +9 -0
- data/app/views/admin/documents/index.html.erb +2 -4
- data/app/views/admin/documents/index.json_file.jbuilder +9 -0
- data/app/views/admin/documents/versions.html.erb +1 -1
- data/app/views/admin/elements/_element.json.jbuilder +4 -1
- data/app/views/admin/elements/_form.html.erb +1 -1
- data/app/views/admin/elements/edit.html.erb +2 -2
- data/app/views/admin/elements/index.html.erb +4 -4
- data/app/views/admin/elements/index.json.jbuilder +2 -0
- data/app/views/admin/elements/new.html.erb +1 -1
- data/app/views/admin/elements/show.html.erb +6 -6
- data/app/views/admin/elements/show.json.jbuilder +2 -0
- data/app/views/admin/form_elements/_form.html.erb +1 -1
- data/app/views/admin/form_elements/_form_element.json.jbuilder +2 -0
- data/app/views/admin/form_elements/edit.html.erb +2 -2
- data/app/views/admin/form_elements/index.html.erb +2 -2
- data/app/views/admin/form_elements/index.json.jbuilder +2 -0
- data/app/views/admin/form_elements/new.html.erb +1 -1
- data/app/views/admin/form_elements/show.html.erb +2 -2
- data/app/views/admin/form_elements/show.json.jbuilder +2 -0
- data/app/views/admin/ids/fetch.json.jbuilder +8 -7
- data/app/views/admin/ids/index.json.jbuilder +3 -2
- data/app/views/admin/import_documents/show.html.haml +1 -1
- data/app/views/admin/imports/_import.json.jbuilder +2 -1
- data/app/views/admin/imports/_show_failed_tab.html.erb +2 -2
- data/app/views/admin/imports/_show_success_tab.html.erb +1 -1
- data/app/views/admin/imports/edit.html.erb +3 -3
- data/app/views/admin/imports/index.html.erb +3 -3
- data/app/views/admin/layouts/application.html.erb +4 -4
- data/app/views/admin/mappings/_form.html.erb +1 -1
- data/app/views/admin/mappings/_mapping.json.jbuilder +2 -1
- data/app/views/admin/mappings/edit.html.erb +2 -2
- data/app/views/admin/mappings/new.html.erb +1 -1
- data/app/views/admin/mappings/show.html.erb +2 -2
- data/app/views/admin/notifications/index.html.erb +3 -3
- data/app/views/admin/search/index.html.erb +2 -2
- data/app/views/admin/shared/_navbar.html.erb +11 -6
- data/app/views/admin/shared/_toast.html.erb +1 -1
- data/app/views/admin/users/index.html.erb +1 -1
- data/app/views/catalog/_show_gbl_admin.html.erb +3 -0
- data/config/locales/documents.en.yml +3 -0
- data/config/routes.rb +37 -33
- data/db/migrate/20230316183001_add_geoblacklight_admin_gem.rb +409 -0
- data/db/seeds.rb +7 -6
- data/db/seeds_elements.csv +56 -55
- data/db/seeds_elements.numbers +0 -0
- data/db/seeds_form_elements.csv +65 -64
- data/db/seeds_form_elements.numbers +0 -0
- data/lib/generators/geoblacklight_admin/config_generator.rb +61 -94
- data/lib/generators/geoblacklight_admin/install_generator.rb +3 -12
- data/lib/generators/geoblacklight_admin/jobs_generator.rb +3 -2
- data/lib/generators/geoblacklight_admin/models_generator.rb +0 -1
- data/lib/generators/geoblacklight_admin/templates/_show_sidebar.html.erb +19 -0
- data/lib/generators/geoblacklight_admin/templates/_user_util_links.html.erb +1 -1
- data/lib/generators/geoblacklight_admin/templates/config/database.yml +9 -5
- data/lib/generators/geoblacklight_admin/templates/config/geomg_aardvark_schema.json +12 -6
- data/lib/generators/geoblacklight_admin/templates/config/initializers/devise.rb +315 -0
- data/lib/generators/geoblacklight_admin/templates/config/initializers/geoblacklight_admin.rb +1 -2
- data/lib/generators/geoblacklight_admin/templates/config/initializers/kithe.rb +2 -0
- data/lib/generators/geoblacklight_admin/templates/config/initializers/mime_types.rb +15 -0
- data/lib/generators/geoblacklight_admin/templates/config/initializers/pagy.rb +152 -68
- data/lib/generators/geoblacklight_admin/templates/config/initializers/simple_form.rb +178 -0
- data/lib/generators/geoblacklight_admin/templates/config/initializers/simple_form_bootstrap.rb +365 -0
- data/lib/generators/geoblacklight_admin/templates/config/settings.yml +293 -0
- data/lib/generators/geoblacklight_admin/templates/javascript/controllers/results_controller.js +10 -0
- data/lib/generators/geoblacklight_admin/templates/package.json +3 -0
- data/lib/generators/geoblacklight_admin/templates/solr/conf/schema.xml +3 -3
- data/lib/generators/geoblacklight_admin/views_generator.rb +2 -3
- data/lib/geoblacklight_admin/engine.rb +5 -6
- data/lib/geoblacklight_admin/version.rb +1 -1
- data/lib/geoblacklight_admin.rb +1 -1
- data/lib/tasks/geoblacklight_admin.rake +15 -11
- metadata +101 -63
- data/.editorconfig +0 -18
- data/.gitignore +0 -75
- data/.standard.yml +0 -4
- data/Gemfile +0 -59
- data/LICENSE +0 -21
- data/app/assets/images/.keep +0 -0
- data/app/controllers/admin/concerns/.keep +0 -0
- data/app/models/bookmark.rb +0 -17
- data/app/models/concerns/.keep +0 -0
- data/db/migrate/20230316183001_add_geoblacklight_admin.rb +0 -399
- data/geoblacklight_admin.gemspec +0 -67
- data/lib/generators/geoblacklight_admin/example_docs_generator.rb +0 -18
- data/lib/generators/geoblacklight_admin/templates/.env.development.example +0 -28
- data/lib/generators/geoblacklight_admin/templates/.solr_wrapper.yml +0 -7
- data/lib/generators/geoblacklight_admin/templates/solr/snapshots/.keep +0 -0
- data/lib/tasks/.keep +0 -0
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +0 -35
- data/template.rb +0 -16
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/confirmations/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/invitations/edit.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/invitations/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/mailer/invitation_instructions.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/mailer/invitation_instructions.text.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/passwords/edit.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/passwords/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/registrations/edit.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/registrations/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/sessions/new.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/shared/_links.html.erb +0 -0
- /data/{app/views/admin → lib/generators/geoblacklight_admin/templates}/devise/unlocks/new.html.erb +0 -0
|
@@ -34,7 +34,10 @@ module Admin
|
|
|
34
34
|
|
|
35
35
|
respond_to do |format|
|
|
36
36
|
if @import.save
|
|
37
|
-
format.html
|
|
37
|
+
format.html do
|
|
38
|
+
redirect_to admin_import_mappings_path(@import),
|
|
39
|
+
notice: "Import was successful. Please set your import mapping rules."
|
|
40
|
+
end
|
|
38
41
|
format.json { render :show, status: :created, location: @import }
|
|
39
42
|
else
|
|
40
43
|
format.html { render :new }
|
|
@@ -82,7 +85,8 @@ module Admin
|
|
|
82
85
|
# Never trust parameters from the scary internet, only allow the white list through.
|
|
83
86
|
|
|
84
87
|
def permittable_params
|
|
85
|
-
%i[type name filename source description row_count encoding content_type extension validity validation_result
|
|
88
|
+
%i[type name filename source description row_count encoding content_type extension validity validation_result
|
|
89
|
+
csv_file run]
|
|
86
90
|
end
|
|
87
91
|
|
|
88
92
|
def import_params
|
|
@@ -102,4 +106,4 @@ module Admin
|
|
|
102
106
|
)
|
|
103
107
|
end
|
|
104
108
|
end
|
|
105
|
-
end
|
|
109
|
+
end
|
|
@@ -37,7 +37,9 @@ module Admin
|
|
|
37
37
|
|
|
38
38
|
respond_to do |format|
|
|
39
39
|
if @mapping.save
|
|
40
|
-
format.html
|
|
40
|
+
format.html do
|
|
41
|
+
redirect_to admin_import_mapping_path(@import, @mapping), notice: "Mapping was successfully created."
|
|
42
|
+
end
|
|
41
43
|
format.json { render :show, status: :created, location: @mapping }
|
|
42
44
|
else
|
|
43
45
|
format.html { render :new }
|
|
@@ -51,7 +53,7 @@ module Admin
|
|
|
51
53
|
def update
|
|
52
54
|
respond_to do |format|
|
|
53
55
|
if @mapping.update(mapping_params)
|
|
54
|
-
format.html { redirect_to
|
|
56
|
+
format.html { redirect_to admin_import_mappings_path(@mapping.import), notice: "Mapping was successfully updated." }
|
|
55
57
|
format.json { render :show, status: :ok, location: @mapping }
|
|
56
58
|
else
|
|
57
59
|
format.html { render :edit }
|
|
@@ -65,7 +67,7 @@ module Admin
|
|
|
65
67
|
def destroy
|
|
66
68
|
@mapping.destroy
|
|
67
69
|
respond_to do |format|
|
|
68
|
-
format.html { redirect_to
|
|
70
|
+
format.html { redirect_to admin_import_mappings_url(@import), notice: "Mapping was successfully destroyed." }
|
|
69
71
|
format.json { head :no_content }
|
|
70
72
|
end
|
|
71
73
|
end
|
|
@@ -92,4 +94,4 @@ module Admin
|
|
|
92
94
|
)
|
|
93
95
|
end
|
|
94
96
|
end
|
|
95
|
-
end
|
|
97
|
+
end
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
module Admin
|
|
5
5
|
class SearchController < Admin::AdminController
|
|
6
6
|
def index
|
|
7
|
-
@
|
|
7
|
+
@request = "#{request.protocol}#{request.host}:#{request.port}"
|
|
8
|
+
@facet_options = BlacklightApiFacets.new(@request).facets
|
|
8
9
|
end
|
|
9
10
|
end
|
|
10
|
-
end
|
|
11
|
+
end
|
|
@@ -47,7 +47,7 @@ module FormInputHelper
|
|
|
47
47
|
category_list = [existing_category_value] + category_list unless category_list.include?(existing_category_value)
|
|
48
48
|
|
|
49
49
|
# Turn category list into values and human labels, using i18n or rails humanizing.
|
|
50
|
-
category_list = category_list.collect
|
|
50
|
+
category_list = category_list.collect do |key|
|
|
51
51
|
value = if key.nil?
|
|
52
52
|
key
|
|
53
53
|
elsif model_class.respond_to?(:model_name) && key.present?
|
|
@@ -59,15 +59,15 @@ module FormInputHelper
|
|
|
59
59
|
end
|
|
60
60
|
|
|
61
61
|
[value, key]
|
|
62
|
-
|
|
62
|
+
end.to_h
|
|
63
63
|
|
|
64
64
|
tag.div(class: "form-row category-and-value") do
|
|
65
65
|
tag.div(class: "col-left category") {
|
|
66
66
|
builder.input category_key, collection: category_list, label: false, include_blank: false
|
|
67
67
|
} +
|
|
68
|
-
tag.div(class: "col-sm value")
|
|
68
|
+
tag.div(class: "col-sm value") do
|
|
69
69
|
builder.input value_key, label: false, input_html: {data: input_data_attributes}
|
|
70
|
-
|
|
70
|
+
end
|
|
71
71
|
end
|
|
72
72
|
end
|
|
73
73
|
end
|
|
@@ -5,7 +5,7 @@ module GeoblacklightAdminHelper
|
|
|
5
5
|
# @TODO:
|
|
6
6
|
# Cannot generate app if uncommented...
|
|
7
7
|
# Uncomment after app is generated to fix view errors
|
|
8
|
-
|
|
8
|
+
include ::Pagy::Frontend if defined?(Pagy)
|
|
9
9
|
|
|
10
10
|
# jbuilder helper
|
|
11
11
|
def no_json_blanks(value)
|
data/app/jobs/export_job.rb
CHANGED
|
@@ -6,8 +6,9 @@ require "csv"
|
|
|
6
6
|
class ExportJob < ApplicationJob
|
|
7
7
|
queue_as :default
|
|
8
8
|
|
|
9
|
-
def perform(current_user, query_params, export_service)
|
|
9
|
+
def perform(request, current_user, query_params, export_service)
|
|
10
10
|
logger.debug("\n\n Background Job: ♞")
|
|
11
|
+
logger.debug("Request: #{request.inspect}")
|
|
11
12
|
logger.debug("User: #{current_user.inspect}")
|
|
12
13
|
logger.debug("Query: #{query_params.inspect}")
|
|
13
14
|
logger.debug("Export Service: #{export_service.inspect}")
|
|
@@ -17,7 +18,7 @@ class ExportJob < ApplicationJob
|
|
|
17
18
|
ActionCable.server.broadcast("export_channel", {data: "Hello from Export Job!"})
|
|
18
19
|
|
|
19
20
|
# Query params into Doc ids
|
|
20
|
-
document_ids = query_params[:ids] || crawl_query(query_params)
|
|
21
|
+
document_ids = query_params[:ids] || crawl_query(request, query_params)
|
|
21
22
|
|
|
22
23
|
logger.debug("Document Ids: #{document_ids}")
|
|
23
24
|
|
|
@@ -41,7 +42,8 @@ class ExportJob < ApplicationJob
|
|
|
41
42
|
notification.deliver(current_user)
|
|
42
43
|
|
|
43
44
|
# Attach CSV file (can only attach after persisted)
|
|
44
|
-
notification.record.file.attach(io: @tempfile, filename: "geomg-export-#{Time.zone.today}.csv",
|
|
45
|
+
notification.record.file.attach(io: @tempfile, filename: "geomg-export-#{Time.zone.today}.csv",
|
|
46
|
+
content_type: "text/csv")
|
|
45
47
|
|
|
46
48
|
# Update UI
|
|
47
49
|
ActionCable.server.broadcast("export_channel", {
|
|
@@ -55,15 +57,16 @@ class ExportJob < ApplicationJob
|
|
|
55
57
|
})
|
|
56
58
|
end
|
|
57
59
|
|
|
58
|
-
def crawl_query(query_params, doc_ids = [])
|
|
60
|
+
def crawl_query(request, query_params, doc_ids = [])
|
|
59
61
|
logger.debug("\n\n CRAWL Query: #{query_params}")
|
|
60
|
-
|
|
62
|
+
logger.debug("\n\n CRAWL Query Request: #{request}")
|
|
63
|
+
api_results = BlacklightApiIds.new(request, query_params)
|
|
61
64
|
logger.debug("API Results: #{api_results.results.inspect}")
|
|
62
65
|
|
|
63
66
|
doc_ids << api_results.results.pluck("id")
|
|
64
67
|
|
|
65
68
|
unless api_results.meta["pages"]["next_page"].nil?
|
|
66
|
-
crawl_query(query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
|
69
|
+
crawl_query(request, query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
|
67
70
|
doc_ids)
|
|
68
71
|
end
|
|
69
72
|
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "csv"
|
|
4
|
+
|
|
5
|
+
# ExportJsonBulkJob
|
|
6
|
+
class ExportJsonBulkJob < ApplicationJob
|
|
7
|
+
queue_as :default
|
|
8
|
+
|
|
9
|
+
def perform(request, current_user, query_params, export_service)
|
|
10
|
+
logger.debug("\n\n Background Job: ♞")
|
|
11
|
+
logger.debug("Request: #{request.inspect}")
|
|
12
|
+
logger.debug("User: #{current_user.inspect}")
|
|
13
|
+
logger.debug("Query: #{query_params.inspect}")
|
|
14
|
+
logger.debug("Export Service: #{export_service.inspect}")
|
|
15
|
+
logger.debug("\n\n")
|
|
16
|
+
|
|
17
|
+
# Test broadcast
|
|
18
|
+
ActionCable.server.broadcast("export_channel", {data: "Hello from Export Job!"})
|
|
19
|
+
|
|
20
|
+
# Query params into Doc ids
|
|
21
|
+
document_ids = query_params[:ids] || crawl_query(request, query_params)
|
|
22
|
+
|
|
23
|
+
logger.debug("Document Ids: #{document_ids}")
|
|
24
|
+
|
|
25
|
+
# Send progress
|
|
26
|
+
documents = export_service.call(document_ids)
|
|
27
|
+
|
|
28
|
+
begin
|
|
29
|
+
# Array of JSON
|
|
30
|
+
@json_array = []
|
|
31
|
+
|
|
32
|
+
documents.each do |doc|
|
|
33
|
+
json_output = Admin::DocumentsController.render("_json_file.jbuilder",
|
|
34
|
+
locals: {document: doc})
|
|
35
|
+
|
|
36
|
+
json_obj = JSON.parse(json_output)
|
|
37
|
+
Rails.logger.debug json_obj
|
|
38
|
+
|
|
39
|
+
# Remove nil/null values from JSON
|
|
40
|
+
json_obj.compact!
|
|
41
|
+
|
|
42
|
+
@json_array << JSON.pretty_generate(json_obj)
|
|
43
|
+
|
|
44
|
+
rescue NoMethodError => e
|
|
45
|
+
Rails.logger.debug { "==== Error! - #{doc.friendlier_id} ====" }
|
|
46
|
+
Rails.logger.debug e.inspect
|
|
47
|
+
next
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Write into tempfile
|
|
52
|
+
@tempfile = Tempfile.new(["export-#{Time.zone.today}", ".json"]).tap do |file|
|
|
53
|
+
file.write("[#{@json_array.join(",")}]")
|
|
54
|
+
end
|
|
55
|
+
@tempfile.rewind
|
|
56
|
+
|
|
57
|
+
# Create notification
|
|
58
|
+
# Message: "Download Type|Row Count|Button Label"
|
|
59
|
+
notification = ExportNotification.with(message: "JSON FILE |#{ActionController::Base.helpers.number_with_delimiter(documents.size)} rows|JSON")
|
|
60
|
+
|
|
61
|
+
# Deliver notification
|
|
62
|
+
notification.deliver(current_user)
|
|
63
|
+
|
|
64
|
+
# Attach JSON file (can only attach after persisted)
|
|
65
|
+
notification.record.file.attach(io: @tempfile, filename: "geomg-export-#{Time.zone.today}.json",
|
|
66
|
+
content_type: "application/json")
|
|
67
|
+
|
|
68
|
+
# Update UI
|
|
69
|
+
ActionCable.server.broadcast("export_channel", {
|
|
70
|
+
data: "Notification ready!",
|
|
71
|
+
actions: [
|
|
72
|
+
{
|
|
73
|
+
method: "RefreshNotifications",
|
|
74
|
+
payload: current_user.notifications.unread.count
|
|
75
|
+
}
|
|
76
|
+
]
|
|
77
|
+
})
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def crawl_query(request, query_params, doc_ids = [])
|
|
81
|
+
logger.debug("\n\n CRAWL Query: #{query_params}")
|
|
82
|
+
logger.debug("\n\n CRAWL Query Request: #{request}")
|
|
83
|
+
api_results = BlacklightApiIds.new(request, query_params)
|
|
84
|
+
logger.debug("API Results: #{api_results.results.inspect}")
|
|
85
|
+
|
|
86
|
+
doc_ids << api_results.results.pluck("id")
|
|
87
|
+
|
|
88
|
+
unless api_results.meta["pages"]["next_page"].nil?
|
|
89
|
+
crawl_query(request, query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
|
90
|
+
doc_ids)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
doc_ids
|
|
94
|
+
end
|
|
95
|
+
end
|
data/app/jobs/export_json_job.rb
CHANGED
|
@@ -3,22 +3,23 @@
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "pathname"
|
|
5
5
|
|
|
6
|
-
#
|
|
6
|
+
# ExportJsonJob
|
|
7
7
|
class ExportJsonJob < ApplicationJob
|
|
8
8
|
queue_as :default
|
|
9
9
|
|
|
10
|
-
def perform(current_user, query_params, export_service)
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
10
|
+
def perform(request, current_user, query_params, export_service)
|
|
11
|
+
logger.debug("\n\n Background Job: ♞")
|
|
12
|
+
logger.debug("Request: #{request.inspect}")
|
|
13
|
+
logger.debug("User: #{current_user.inspect}")
|
|
14
|
+
logger.debug("Query: #{query_params.inspect}")
|
|
15
|
+
logger.debug("Export Service: #{export_service.inspect}")
|
|
16
|
+
logger.debug("\n\n")
|
|
16
17
|
|
|
17
18
|
# Test broadcast
|
|
18
19
|
ActionCable.server.broadcast("export_channel", {data: "Hello from Export JSON Job!"})
|
|
19
20
|
|
|
20
21
|
# Query params into Doc ids
|
|
21
|
-
document_ids = query_params[:ids] || crawl_query(query_params)
|
|
22
|
+
document_ids = query_params[:ids] || crawl_query(request, query_params)
|
|
22
23
|
|
|
23
24
|
# Rails.logger.debug("Document Ids: #{document_ids}")
|
|
24
25
|
|
|
@@ -47,7 +48,7 @@ class ExportJsonJob < ApplicationJob
|
|
|
47
48
|
tree.dirname.mkpath
|
|
48
49
|
Rails.logger.debug tree.inspect
|
|
49
50
|
|
|
50
|
-
json_output = DocumentsController.render("_#{query_params[:format]}",
|
|
51
|
+
json_output = Admin::DocumentsController.render("_#{query_params[:format]}",
|
|
51
52
|
locals: {document: doc})
|
|
52
53
|
|
|
53
54
|
json_obj = JSON.parse(json_output)
|
|
@@ -103,15 +104,16 @@ class ExportJsonJob < ApplicationJob
|
|
|
103
104
|
})
|
|
104
105
|
end
|
|
105
106
|
|
|
106
|
-
def crawl_query(query_params, doc_ids = [])
|
|
107
|
+
def crawl_query(request, query_params, doc_ids = [])
|
|
107
108
|
logger.debug("\n\n CRAWL Query: #{query_params}")
|
|
108
|
-
|
|
109
|
+
logger.debug("\n\n CRAWL Query Request: #{request}")
|
|
110
|
+
api_results = BlacklightApiIds.new(request, query_params)
|
|
109
111
|
logger.debug("API Results: #{api_results.results.inspect}")
|
|
110
112
|
|
|
111
113
|
doc_ids << api_results.results.pluck("id")
|
|
112
114
|
|
|
113
115
|
unless api_results.meta["pages"]["next_page"].nil?
|
|
114
|
-
crawl_query(query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
|
116
|
+
crawl_query(request, query_params.merge!({page: api_results.meta["pages"]["next_page"]}),
|
|
115
117
|
doc_ids)
|
|
116
118
|
end
|
|
117
119
|
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Admin::Bookmark
|
|
4
|
+
module Admin
|
|
5
|
+
class Bookmark < ApplicationRecord
|
|
6
|
+
belongs_to :user, polymorphic: true
|
|
7
|
+
belongs_to :document, polymorphic: true
|
|
8
|
+
validates :user_id, presence: true
|
|
9
|
+
|
|
10
|
+
def document_id
|
|
11
|
+
document_id
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def document_type
|
|
15
|
+
Kithe::Model
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
class ApiSearchBuilder < Blacklight::SearchBuilder
|
|
2
4
|
include Blacklight::Solr::SearchBuilderBehavior
|
|
3
5
|
include BlacklightAdvancedSearch::AdvancedSearchBuilder
|
|
4
6
|
# include B1gDateRangeQueryConcern::RangeLimitBuilder
|
|
5
7
|
|
|
6
|
-
self.default_processor_chain += [
|
|
8
|
+
self.default_processor_chain += %i[add_advanced_parse_q_to_solr add_advanced_search_to_solr]
|
|
7
9
|
|
|
8
10
|
# Remove GBL Spatial, otherwise it will hide all suppressed docs
|
|
9
11
|
# from API search results
|
|
@@ -1,28 +1,28 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "httparty"
|
|
4
|
+
|
|
3
5
|
# BlacklightApi
|
|
4
6
|
class BlacklightApi
|
|
5
7
|
include HTTParty
|
|
6
8
|
default_timeout 300
|
|
7
9
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def initialize(**args)
|
|
10
|
+
def initialize(request, args)
|
|
11
11
|
defaults = {
|
|
12
12
|
q: "*",
|
|
13
13
|
page: 1,
|
|
14
14
|
sort: "score+desc%2C+dc_title_sort+asc",
|
|
15
15
|
rows: 20
|
|
16
16
|
}
|
|
17
|
-
|
|
18
|
-
@options = defaults.merge(
|
|
17
|
+
@request = request
|
|
18
|
+
@options = defaults.merge(args)
|
|
19
19
|
append_facets(@options[:f], @options)
|
|
20
20
|
append_daterange(@options[:f], @options)
|
|
21
21
|
@options.compact!
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
def fetch
|
|
25
|
-
@fetch ||= self.class.get("
|
|
25
|
+
@fetch ||= self.class.get("#{@request}#{BLACKLIGHT_JSON_API}", query: @options)
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
def results
|
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "httparty"
|
|
4
|
+
|
|
3
5
|
# BlacklightApiFacets
|
|
4
6
|
class BlacklightApiFacets
|
|
5
7
|
include HTTParty
|
|
6
|
-
|
|
8
|
+
|
|
9
|
+
def initialize(request)
|
|
10
|
+
@request = request
|
|
11
|
+
end
|
|
7
12
|
|
|
8
13
|
def fetch
|
|
9
|
-
@fetch ||= self.class.get("
|
|
14
|
+
@fetch ||= self.class.get("#{@request}#{BLACKLIGHT_JSON_API_FACETS}", query: {})
|
|
10
15
|
end
|
|
11
16
|
|
|
12
17
|
def facets
|
|
@@ -5,9 +5,7 @@ class BlacklightApiIds
|
|
|
5
5
|
include HTTParty
|
|
6
6
|
default_timeout 300
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def initialize(args = {})
|
|
8
|
+
def initialize(request, args = {})
|
|
11
9
|
defaults = {
|
|
12
10
|
q: "*",
|
|
13
11
|
page: 1,
|
|
@@ -15,6 +13,7 @@ class BlacklightApiIds
|
|
|
15
13
|
rows: 1000
|
|
16
14
|
}
|
|
17
15
|
|
|
16
|
+
@request = request
|
|
18
17
|
@options = defaults.merge(**args)
|
|
19
18
|
append_facets(@options[:f], @options)
|
|
20
19
|
append_daterange(@options[:f], @options)
|
|
@@ -24,7 +23,7 @@ class BlacklightApiIds
|
|
|
24
23
|
def fetch
|
|
25
24
|
Rails.logger.debug { "BlacklightApiIds > fetch > query: #{@options.inspect}" }
|
|
26
25
|
|
|
27
|
-
@fetch ||= self.class.get("
|
|
26
|
+
@fetch ||= self.class.get("#{@request}#{BLACKLIGHT_JSON_API_IDS}", query: @options)
|
|
28
27
|
end
|
|
29
28
|
|
|
30
29
|
def results
|
data/app/models/bulk_action.rb
CHANGED
|
@@ -69,7 +69,7 @@ class BulkAction < ApplicationRecord
|
|
|
69
69
|
def api_documents(uri)
|
|
70
70
|
qargs = Rack::Utils.parse_nested_query(uri.query)
|
|
71
71
|
query_params = {q: qargs["q"], f: qargs["f"], page: qargs["page"], rows: 1_000_000}
|
|
72
|
-
api_documents = BlacklightApiIds.new(**query_params)
|
|
72
|
+
api_documents = BlacklightApiIds.new(request, **query_params)
|
|
73
73
|
create_documents(api_documents.load_all)
|
|
74
74
|
end
|
|
75
75
|
|
|
@@ -14,7 +14,10 @@ class Document
|
|
|
14
14
|
valid_geom = true
|
|
15
15
|
|
|
16
16
|
# Sane for W,S,E,N?
|
|
17
|
-
|
|
17
|
+
unless record.send(GeoblacklightAdmin::Schema.instance.solr_fields[:bounding_box]).nil?
|
|
18
|
+
proper_bounding_box(record,
|
|
19
|
+
valid_geom)
|
|
20
|
+
end
|
|
18
21
|
|
|
19
22
|
valid_geom
|
|
20
23
|
end
|
|
@@ -57,10 +60,11 @@ class Document
|
|
|
57
60
|
# Reject ENVELOPE(-118.00.0000,-88.00.0000,51.00.0000,42.00.0000
|
|
58
61
|
# - Double period float-ish things?
|
|
59
62
|
geom.each do |val|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
63
|
+
next unless val.count(".") >= 2
|
|
64
|
+
|
|
65
|
+
valid_geom = false
|
|
66
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:bounding_box],
|
|
67
|
+
"invalid ENVELOPE(W,E,N,S) syntax - found multiple periods in a coordinate value.")
|
|
64
68
|
end
|
|
65
69
|
end
|
|
66
70
|
|
|
@@ -18,7 +18,8 @@ class Document
|
|
|
18
18
|
|
|
19
19
|
# Must split into two via '-'
|
|
20
20
|
if date_range.split("-").size != 2
|
|
21
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
|
21
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
|
22
|
+
"invalid date range present - missing YYYY-YYYY value")
|
|
22
23
|
valid_date_ranges = false
|
|
23
24
|
return valid_date_ranges
|
|
24
25
|
end
|
|
@@ -26,28 +27,30 @@ class Document
|
|
|
26
27
|
start_date, end_date = date_range.split("-")
|
|
27
28
|
|
|
28
29
|
unless valid_yyyy(start_date)
|
|
29
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
|
30
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
|
31
|
+
"invalid start date - bad YYYY-YYYY value, only integers or the wildcard '*' allowed")
|
|
30
32
|
valid_date_ranges = false
|
|
31
33
|
end
|
|
32
34
|
|
|
33
35
|
unless valid_yyyy(end_date)
|
|
34
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
|
36
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
|
37
|
+
"invalid end date - bad YYYY-YYYY value, only integers or the wildcard '*' allowed")
|
|
35
38
|
valid_date_ranges = false
|
|
36
39
|
end
|
|
37
40
|
|
|
38
41
|
# Allow only 1 wildcard
|
|
39
42
|
if start_date == "*" && end_date == "*"
|
|
40
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
|
43
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
|
44
|
+
"invalid only one wildcard can be used")
|
|
41
45
|
valid_date_ranges = false
|
|
42
46
|
end
|
|
43
47
|
|
|
44
48
|
# Avoid 1996-1946
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
end
|
|
49
|
+
next unless start_date[/\d+/] && end_date[/\d+/] && start_date.to_i > end_date.to_i
|
|
50
|
+
|
|
51
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:date_range],
|
|
52
|
+
"invalid start date must be less than the end date")
|
|
53
|
+
valid_date_ranges = false
|
|
51
54
|
end
|
|
52
55
|
valid_date_ranges
|
|
53
56
|
end
|
|
@@ -37,7 +37,8 @@ class Document
|
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
unless valid_geom
|
|
40
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:geometry],
|
|
40
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:geometry],
|
|
41
|
+
"Invalid envelope: #{error_message}")
|
|
41
42
|
end
|
|
42
43
|
|
|
43
44
|
valid_geom
|
|
@@ -60,7 +61,8 @@ class Document
|
|
|
60
61
|
# Guard against a whole world polygons
|
|
61
62
|
if geom == "POLYGON((-180 90, 180 90, 180 -90, -180 -90, -180 90))"
|
|
62
63
|
valid_geom = false
|
|
63
|
-
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:geometry],
|
|
64
|
+
record.errors.add(GeoblacklightAdmin::Schema.instance.solr_fields[:geometry],
|
|
65
|
+
"Invalid polygon: all points are coplanar input, Solr will not index")
|
|
64
66
|
end
|
|
65
67
|
|
|
66
68
|
valid_geom
|
data/app/models/document.rb
CHANGED
|
@@ -11,7 +11,8 @@ class Document < Kithe::Work
|
|
|
11
11
|
belongs_to :import, optional: true
|
|
12
12
|
|
|
13
13
|
# Statesman
|
|
14
|
-
has_many :document_transitions, foreign_key: "kithe_model_id", autosave: false, dependent: :destroy,
|
|
14
|
+
has_many :document_transitions, foreign_key: "kithe_model_id", autosave: false, dependent: :destroy,
|
|
15
|
+
inverse_of: :document
|
|
15
16
|
|
|
16
17
|
# DocumentAccesses
|
|
17
18
|
has_many :document_accesses, primary_key: "friendlier_id", foreign_key: "friendlier_id", autosave: false, dependent: :destroy,
|
|
@@ -69,7 +70,9 @@ class Document < Kithe::Work
|
|
|
69
70
|
# Index Transformations - *_json functions
|
|
70
71
|
def references_json
|
|
71
72
|
references = ActiveSupport::HashWithIndifferentAccess.new
|
|
72
|
-
send(GeoblacklightAdmin::Schema.instance.solr_fields[:reference]).each
|
|
73
|
+
send(GeoblacklightAdmin::Schema.instance.solr_fields[:reference]).each do |ref|
|
|
74
|
+
references[Document::Reference::REFERENCE_VALUES[ref.category.to_sym][:uri]] = ref.value
|
|
75
|
+
end
|
|
73
76
|
references = apply_downloads(references)
|
|
74
77
|
references.to_json
|
|
75
78
|
end
|
|
@@ -79,7 +82,10 @@ class Document < Kithe::Work
|
|
|
79
82
|
# Make sure downloads exist!
|
|
80
83
|
if document_downloads.present?
|
|
81
84
|
multiple_downloads = multiple_downloads_array
|
|
82
|
-
|
|
85
|
+
if dct_downloads.present?
|
|
86
|
+
multiple_downloads << {label: download_text(send(GeoblacklightAdmin::Schema.instance.solr_fields[:format])),
|
|
87
|
+
url: dct_downloads}
|
|
88
|
+
end
|
|
83
89
|
references[:"http://schema.org/downloadUrl"] = multiple_downloads
|
|
84
90
|
end
|
|
85
91
|
references
|
|
@@ -178,7 +184,9 @@ class Document < Kithe::Work
|
|
|
178
184
|
end
|
|
179
185
|
|
|
180
186
|
def dct_references_s_to_csv(key, destination)
|
|
181
|
-
send(destination).detect
|
|
187
|
+
send(destination).detect do |ref|
|
|
188
|
+
ref.category == GeoblacklightAdmin::Schema.instance.dct_references_mappings[key]
|
|
189
|
+
end.value
|
|
182
190
|
rescue NoMethodError
|
|
183
191
|
nil
|
|
184
192
|
end
|
|
@@ -219,9 +227,9 @@ class Document < Kithe::Work
|
|
|
219
227
|
end
|
|
220
228
|
|
|
221
229
|
def set_geometry
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
230
|
+
return unless locn_geometry.blank? && self&.dcat_bbox&.present?
|
|
231
|
+
|
|
232
|
+
self.locn_geometry = derive_polygon
|
|
225
233
|
end
|
|
226
234
|
|
|
227
235
|
# Convert GEOM for Solr Indexing
|
data/app/models/element.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
class Element < ApplicationRecord
|
|
2
4
|
serialize :html_attributes
|
|
3
5
|
|
|
@@ -21,13 +23,13 @@ class Element < ApplicationRecord
|
|
|
21
23
|
# Validations
|
|
22
24
|
validates :label, :solr_field, :field_type, presence: true
|
|
23
25
|
|
|
24
|
-
FIELD_TYPES = [
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
]
|
|
26
|
+
FIELD_TYPES = %w[
|
|
27
|
+
string
|
|
28
|
+
text
|
|
29
|
+
integer
|
|
30
|
+
boolean
|
|
31
|
+
datetime
|
|
32
|
+
].freeze
|
|
31
33
|
|
|
32
34
|
# Find by solr_field shortcut
|
|
33
35
|
def self.at(field)
|