decidim-cdtb 0.2.1 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/CHANGELOG.md +6 -1
- data/Gemfile +1 -0
- data/Gemfile.lock +3 -2
- data/README.md +84 -0
- data/config/initializers/rack_attack.rb +34 -11
- data/lib/decidim/cdtb/fixes/you_tube_embeds_fixer.rb +115 -0
- data/lib/decidim/cdtb/participatory_spaces/add_content_blocks.rb +117 -0
- data/lib/decidim/cdtb/tasks.rb +2 -0
- data/lib/decidim/cdtb/version.rb +1 -1
- data/lib/tasks/fix_youtube_embeds.rake +22 -0
- data/lib/tasks/logs.rake +107 -5
- data/lib/tasks/participatory_spaces.rake +49 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fa565b7eeda2d143b2fb496a4129288d4e054904f6edcf1c8da23712c212a025
|
4
|
+
data.tar.gz: f5862f5bef01ccb6bd19e172f378339cd69786924df835a8793825b8faf817e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c5461ffafd0c6d562fe1a197fba854a9c03856eec585a573f4298cc3d9d0c7f7f7f1faacf505d27d6404cefb1aa837e96e0686ffd1d4816daaec7e62f0d5272f
|
7
|
+
data.tar.gz: 8e88fd9d1bb77d669c73eba22d28a8adfaa0e22cae69b5d3a8eb2cca0caa55c1c126ed0ad029a636d44658b23d2a32d496fb1ea3dec30c8090b6d4206410dddf
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
|
3
|
-
## [0.
|
3
|
+
## [0.3.0] - 2024-12-24 (minor - L'artista de la pista)
|
4
|
+
|
5
|
+
- Fix YouTube embeds to Decidim v0.28 format in different places. Only YouTube is supported right now.
|
6
|
+
- Add task to add new content blocks in participatory processes
|
7
|
+
|
8
|
+
## [0.2.1] - 2024-09-25 (patch - Una idea sobre rodes)
|
4
9
|
|
5
10
|
- Fix validate_migrations CI which requires Postgres service
|
6
11
|
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
decidim-cdtb (0.
|
4
|
+
decidim-cdtb (0.3.0)
|
5
5
|
decidim (>= 0.27.0)
|
6
6
|
rails (>= 6)
|
7
7
|
ruby-progressbar
|
@@ -745,7 +745,7 @@ GEM
|
|
745
745
|
concurrent-ruby (~> 1.0)
|
746
746
|
uber (0.1.0)
|
747
747
|
unicode-display_width (2.5.0)
|
748
|
-
uri (0.13.
|
748
|
+
uri (0.13.1)
|
749
749
|
valid_email2 (2.3.1)
|
750
750
|
activemodel (>= 3.2)
|
751
751
|
mail (~> 2.5)
|
@@ -805,6 +805,7 @@ DEPENDENCIES
|
|
805
805
|
rspec (~> 3.0)
|
806
806
|
rubocop
|
807
807
|
sqlite3 (~> 1.4)
|
808
|
+
uri (>= 0.13.1)
|
808
809
|
|
809
810
|
BUNDLED WITH
|
810
811
|
2.3.6
|
data/README.md
CHANGED
@@ -51,6 +51,21 @@ Available rake tasks:
|
|
51
51
|
- `bin/rake cdtb:anonymize:system_admins` anonymizes system admins
|
52
52
|
- `bin/rake cdtb:anonymize:paper_trail` anonymizes paper trails
|
53
53
|
|
54
|
+
### Fix YouTube embeds for Decidim v0.28
|
55
|
+
|
56
|
+
Fixes YouTube embeds to Decidim v0.28 format in different places, which at the moment are:
|
57
|
+
|
58
|
+
- Decidim::Meetings::Meeting
|
59
|
+
- Decidim::Debates::Debate
|
60
|
+
- Decidim::Pages::Page
|
61
|
+
- Decidim::Assembly
|
62
|
+
- Decidim::ParticipatoryProcess
|
63
|
+
|
64
|
+
```
|
65
|
+
bin/rake cdtb:embeds:fix_youtube
|
66
|
+
```
|
67
|
+
|
68
|
+
|
54
69
|
### Migrate ActiveStorage service from S3 to local
|
55
70
|
|
56
71
|
To migrate from S3 to local storage, the identified steps will be:
|
@@ -135,6 +150,75 @@ To execute the task run:
|
|
135
150
|
bundle exec rake cdtb:users:remove[spam_users.csv, reporter_user@example.org]
|
136
151
|
```
|
137
152
|
|
153
|
+
### Participatory Spaces / Add content blocks
|
154
|
+
|
155
|
+
You can add content blocks to a participatory spaces with the content block name (manifest_name).
|
156
|
+
This rake task affects spaces in all organizations.
|
157
|
+
|
158
|
+
Content block manifest names list:
|
159
|
+
|
160
|
+
```
|
161
|
+
basic_only_text
|
162
|
+
image_text_cta
|
163
|
+
metadata
|
164
|
+
hero
|
165
|
+
participatory_processes
|
166
|
+
html_2
|
167
|
+
main_data
|
168
|
+
title
|
169
|
+
cta
|
170
|
+
highlighted_proposals
|
171
|
+
how_to_participate
|
172
|
+
html_1
|
173
|
+
related_documents
|
174
|
+
stats
|
175
|
+
html
|
176
|
+
slider
|
177
|
+
footer_sub_hero
|
178
|
+
global_menu
|
179
|
+
sub_hero
|
180
|
+
highlighted_content_banner
|
181
|
+
highlighted_processes
|
182
|
+
highlighted_assemblies
|
183
|
+
highlighted_regulations
|
184
|
+
upcoming_meetings
|
185
|
+
extra_data
|
186
|
+
highlighted_meetings
|
187
|
+
highlighted_results
|
188
|
+
metrics
|
189
|
+
related_assemblies
|
190
|
+
announcement
|
191
|
+
social_networks_metadata
|
192
|
+
related_processes
|
193
|
+
highlighted_posts
|
194
|
+
last_activity
|
195
|
+
```
|
196
|
+
|
197
|
+
Spaces supported:
|
198
|
+
|
199
|
+
- Decidim::ParticipatoryProcess
|
200
|
+
- Decidim::Assembly
|
201
|
+
|
202
|
+
To execute the task run:
|
203
|
+
|
204
|
+
```
|
205
|
+
bundle exec rake cdtb:participatory_spaces:add_content_blocks[manifest_name]
|
206
|
+
```
|
207
|
+
|
208
|
+
You can add multiple manifest names with this format:
|
209
|
+
|
210
|
+
```
|
211
|
+
bundle exec rake cdtb:participatory_spaces:add_content_blocks[['manifest_name_one manifest_name_two']]
|
212
|
+
```
|
213
|
+
|
214
|
+
Some examples
|
215
|
+
```
|
216
|
+
bundle exec rake cdtb:participatory_spaces:add_content_blocks[extra_data]
|
217
|
+
|
218
|
+
bundle exec rake cdtb:participatory_spaces:add_content_blocks[['extra_data related_documents']]
|
219
|
+
|
220
|
+
```
|
221
|
+
|
138
222
|
### Upgrades:
|
139
223
|
|
140
224
|
#### Upgrade modules
|
@@ -3,26 +3,49 @@
|
|
3
3
|
unless ENV["CDTB_RACK_ATTACK_DISABLED"].to_i.positive? || %w[development test].include?(Rails.env)
|
4
4
|
require "rack/attack"
|
5
5
|
|
6
|
-
|
7
|
-
period= ENV.fetch("RACK_ATTACK_THROTTLE_PERIOD", 60)
|
8
|
-
Rails.logger.info("Configuring Rack::Attack.throttle with limit: #{limit}, period: #{period}")
|
9
|
-
Rack::Attack.throttle("requests by (forwarded) ip", limit: limit.to_i, period: period.to_i) do |request|
|
10
|
-
# ignore requests to assets
|
11
|
-
next if request.path.start_with?("/rails/active_storage")
|
12
|
-
|
6
|
+
def extract_ip(request)
|
13
7
|
x_forwarded_for= request.get_header("HTTP_X_FORWARDED_FOR")
|
14
8
|
Rails.logger.info { ">>>>>>>>>>>>>>>>>>>> X-Forwarded-For: #{x_forwarded_for}" }
|
15
9
|
if x_forwarded_for.present?
|
16
|
-
|
17
|
-
|
10
|
+
x_forwarded_for.split(":").first
|
11
|
+
|
18
12
|
else
|
19
13
|
request.ip
|
20
14
|
end
|
21
15
|
end
|
22
16
|
|
17
|
+
limit= ENV.fetch("RACK_ATTACK_THROTTLE_LIMIT", 30)
|
18
|
+
period= ENV.fetch("RACK_ATTACK_THROTTLE_PERIOD", 60)
|
19
|
+
Rails.logger.info("Configuring Rack::Attack.throttle with limit: #{limit}, period: #{period}")
|
20
|
+
Rack::Attack.throttle("requests by ip", limit: limit.to_i, period: period.to_i) do |request|
|
21
|
+
# ignore requests to assets
|
22
|
+
next if request.path.start_with?("/rails/active_storage")
|
23
|
+
|
24
|
+
extract_ip(request)
|
25
|
+
end
|
26
|
+
|
27
|
+
limit= ENV.fetch("RACK_ATTACK_THROTTLE_RANGE_LIMIT", 10)
|
28
|
+
period= ENV.fetch("RACK_ATTACK_THROTTLE_RANGE_PERIOD", 20)
|
29
|
+
Rails.logger.info("Configuring Rack::Attack.throttle with limits for IP Ranges: #{limit}, period: #{period}")
|
30
|
+
Rack::Attack.throttle("requests by ip range", limit: limit.to_i, period: period.to_i) do |request|
|
31
|
+
# ignore requests to assets
|
32
|
+
next if request.path.start_with?("/rails/active_storage")
|
33
|
+
|
34
|
+
ip= extract_ip(request)
|
35
|
+
# rubocop: disable Lint/UselessAssignment
|
36
|
+
range_32bit= ip.split(".")[0, 2]
|
37
|
+
# rubocop: enable Lint/UselessAssignment
|
38
|
+
end
|
39
|
+
|
40
|
+
Rack::Attack.blocklist("block all /.well-known/traffic-advice") do |request|
|
41
|
+
request.path.start_with?("/.well-known/traffic-advice")
|
42
|
+
end
|
43
|
+
|
23
44
|
if ENV["RACK_ATTACK_BLOCKED_IPS"].present?
|
24
|
-
ENV["RACK_ATTACK_BLOCKED_IPS"].split(",")
|
25
|
-
|
45
|
+
blocked_ips_and_subnets= ENV["RACK_ATTACK_BLOCKED_IPS"].split(",")
|
46
|
+
Rack::Attack.blocklist("block all unaccepted IPs") do |request|
|
47
|
+
ip= extract_ip(request)
|
48
|
+
blocked_ips_and_subnets.any? { |ip_or_subnet| ip.start_with?(ip_or_subnet) }
|
26
49
|
end
|
27
50
|
end
|
28
51
|
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# require "decidim"
|
4
|
+
# require "decidim/assembly"
|
5
|
+
# require "decidim/participatory_process"
|
6
|
+
# require "decidim/meetings/meeting"
|
7
|
+
# require "decidim/debates/debate"
|
8
|
+
# require "decidim/pages/page"
|
9
|
+
|
10
|
+
module Decidim
|
11
|
+
module Cdtb
|
12
|
+
module Fixes
|
13
|
+
# Fixes YouTube embeds to Decidim v0.28 format in PROCESSED_MODELS.
|
14
|
+
# Only YouTube is supported right now.
|
15
|
+
class YouTubeEmbedsFixer < ::Decidim::Cdtb::Task
|
16
|
+
PROCESSED_MODELS= {
|
17
|
+
"Decidim::Meetings::Meeting" => [:description],
|
18
|
+
"Decidim::Debates::Debate" => %i[description instructions],
|
19
|
+
"Decidim::Pages::Page" => [:body],
|
20
|
+
"Decidim::Assembly" => %i[short_description description],
|
21
|
+
"Decidim::ParticipatoryProcess" => %i[short_description description]
|
22
|
+
}.freeze
|
23
|
+
|
24
|
+
def initialize
|
25
|
+
progress_bar= { title: self.class.name }
|
26
|
+
super("FIX YOUTUBE EMBEDS", progress_bar: progress_bar)
|
27
|
+
end
|
28
|
+
|
29
|
+
attr_reader :num_fixed
|
30
|
+
|
31
|
+
def prepare_execution(_ctx = nil)
|
32
|
+
@num_fixed= @num_items= 0
|
33
|
+
|
34
|
+
PROCESSED_MODELS.each_key do |model_class|
|
35
|
+
@num_items+= model_class.constantize.count
|
36
|
+
end
|
37
|
+
log_task_info("Checking #{@num_items} models...")
|
38
|
+
end
|
39
|
+
|
40
|
+
def total_items
|
41
|
+
@num_items
|
42
|
+
end
|
43
|
+
|
44
|
+
def do_execution(context)
|
45
|
+
progress_bar= context[:progress_bar]
|
46
|
+
|
47
|
+
PROCESSED_MODELS.each_pair do |model_class_name, attribs|
|
48
|
+
log_task_step("Processing #{model_class_name.pluralize}")
|
49
|
+
|
50
|
+
model_class= model_class_name.constantize
|
51
|
+
model_class.find_each do |model|
|
52
|
+
Rails.logger.debug("Processing #{model_class_name}[#{model.id}]")
|
53
|
+
|
54
|
+
attribs.each do |attribute|
|
55
|
+
fix_embed(model, attribute)
|
56
|
+
end
|
57
|
+
|
58
|
+
@num_fixed+= 1 if model.changed?
|
59
|
+
model.save!(validate: false)
|
60
|
+
progress_bar.increment
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def end_execution(_ctx)
|
66
|
+
log_task_step("#{@num_fixed} embeds fixed")
|
67
|
+
end
|
68
|
+
|
69
|
+
# --------------------------------------------------
|
70
|
+
private
|
71
|
+
|
72
|
+
# --------------------------------------------------
|
73
|
+
|
74
|
+
def fix_embed(model, attribute)
|
75
|
+
contents= model.send(attribute)
|
76
|
+
return if contents.blank?
|
77
|
+
|
78
|
+
contents.each_pair do |locale, content|
|
79
|
+
Rails.logger.debug "#{locale} => #{content}"
|
80
|
+
next if locale.to_s == "machine_translations"
|
81
|
+
next if content.blank?
|
82
|
+
|
83
|
+
fixes= fix_localized_embed(content)
|
84
|
+
contents[locale]= fixes.reverse.find { |fix| fix != false } if fixes.any?
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# rubocop: disable Metrics/AbcSize
|
89
|
+
def fix_localized_embed(content)
|
90
|
+
parsed= Nokogiri::HTML(content)
|
91
|
+
divs_w_embed= parsed.css("div[class=editor-content-videoEmbed]")
|
92
|
+
Rails.logger.debug "=> #{divs_w_embed.size} => #{content}"
|
93
|
+
|
94
|
+
divs_w_embed.map do |div|
|
95
|
+
regexp_match= div["data-video-embed"].match(%r{https://www.youtube.com/embed/(?<yt_id>\w+)})
|
96
|
+
next unless regexp_match
|
97
|
+
|
98
|
+
Rails.logger.debug("EMBED:::: #{div.class} => #{div}")
|
99
|
+
|
100
|
+
yt_id= regexp_match["yt_id"]
|
101
|
+
div["data-video-embed"]= "https://www.youtube.com/watch?v=#{yt_id}"
|
102
|
+
iframe= div.css("iframe").first
|
103
|
+
iframe["src"]= "https://www.youtube-nocookie.com/embed/#{yt_id}?cc_load_policy=1&modestbranding=1"
|
104
|
+
fixed_div= div
|
105
|
+
Rails.logger.debug("FIXED TO: #{fixed_div.to_html}")
|
106
|
+
new_content= parsed.css("body").children.to_html
|
107
|
+
Rails.logger.debug("FIXED TO: #{new_content}")
|
108
|
+
new_content
|
109
|
+
end
|
110
|
+
end
|
111
|
+
# rubocop: enable Metrics/AbcSize
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Decidim
|
4
|
+
module Cdtb
|
5
|
+
module ParticipatorySpaces
|
6
|
+
# Add content blocks to participatory spaces
|
7
|
+
class AddContentBlocks < ::Decidim::Cdtb::Task
|
8
|
+
def initialize(processed_models, content_block_names)
|
9
|
+
progress_bar= { title: self.class.name }
|
10
|
+
@processed_models = processed_models
|
11
|
+
@content_block_names = content_block_names
|
12
|
+
super("ADD CONTENT BLOCKS", progress_bar: progress_bar)
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :num_added
|
16
|
+
|
17
|
+
def prepare_execution(_ctx = nil)
|
18
|
+
@num_added= @num_items= 0
|
19
|
+
|
20
|
+
@processed_models.each do |model_name|
|
21
|
+
@num_items+= model_name.constantize.count
|
22
|
+
end
|
23
|
+
log_task_info("Adding content blocks in #{@num_items} spaces...")
|
24
|
+
end
|
25
|
+
|
26
|
+
def total_items
|
27
|
+
@num_items
|
28
|
+
end
|
29
|
+
|
30
|
+
# rubocop:disable Metrics/AbcSize
|
31
|
+
def do_execution(context)
|
32
|
+
progress_bar= context[:progress_bar]
|
33
|
+
|
34
|
+
@processed_models.each do |processed_model|
|
35
|
+
log_task_step("Processing #{processed_model.pluralize}")
|
36
|
+
|
37
|
+
spaces = processed_model.constantize
|
38
|
+
|
39
|
+
@content_block_names.each do |content_block_name|
|
40
|
+
log_task_step("Adding #{content_block_name} content block")
|
41
|
+
|
42
|
+
spaces.find_each do |space|
|
43
|
+
current_content_blocks = current_space_content_blocks(scope_name(space), space.organization, space.id)
|
44
|
+
|
45
|
+
new_content_block = create_content_block!(space, content_block_name, current_content_blocks)
|
46
|
+
if content_block_name == "extra_data" && space.instance_of?(Decidim::ParticipatoryProcess)
|
47
|
+
next if new_content_block.weight == 20
|
48
|
+
|
49
|
+
force_extra_data_content_block_weight!(content_block_name, current_content_blocks)
|
50
|
+
end
|
51
|
+
|
52
|
+
@num_added += 1
|
53
|
+
progress_bar.increment
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def end_execution(_ctx)
|
60
|
+
log_task_step("#{@num_added} content blocks added")
|
61
|
+
end
|
62
|
+
|
63
|
+
def create_content_block!(space, content_block_name, current_content_blocks)
|
64
|
+
exists_content_block = Decidim::ContentBlock.find_by(decidim_organization_id: space.organization.id,
|
65
|
+
scope_name: scope_name(space), manifest_name: content_block_name,
|
66
|
+
scoped_resource_id: space.id)
|
67
|
+
|
68
|
+
return exists_content_block if exists_content_block.present?
|
69
|
+
|
70
|
+
weight = (current_content_blocks.last.weight + 1) * 10
|
71
|
+
log_task_step("Adding #{content_block_name} to #{space.slug}[#{space.id}]")
|
72
|
+
Decidim::ContentBlock.create(
|
73
|
+
decidim_organization_id: space.organization.id,
|
74
|
+
weight: weight,
|
75
|
+
scope_name: scope_name(space),
|
76
|
+
scoped_resource_id: space.id,
|
77
|
+
manifest_name: content_block_name,
|
78
|
+
published_at: Time.current
|
79
|
+
)
|
80
|
+
end
|
81
|
+
# rubocop:enable Metrics/AbcSize
|
82
|
+
|
83
|
+
# +extra_data+ content block usually be down of hero image, therefore, it's weight is 20 and all others content blocks
|
84
|
+
# go one position down added 10
|
85
|
+
def force_extra_data_content_block_weight!(content_block_name, current_content_blocks)
|
86
|
+
extra_data_content_block = current_content_blocks.find_by(manifest_name: content_block_name)
|
87
|
+
extra_data_content_block.update(weight: 20)
|
88
|
+
|
89
|
+
current_content_blocks.each do |content_block|
|
90
|
+
# hero is usually the first content block
|
91
|
+
next if content_block == extra_data_content_block || content_block.manifest_name == "hero"
|
92
|
+
|
93
|
+
content_block.update(weight: content_block.weight + 10)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def current_space_content_blocks(scope_name, organization, scoped_resource_id)
|
98
|
+
Decidim::ContentBlock.for_scope(scope_name, organization: organization).where(scoped_resource_id: scoped_resource_id)
|
99
|
+
end
|
100
|
+
|
101
|
+
# --------------------------------------------------
|
102
|
+
private
|
103
|
+
|
104
|
+
# --------------------------------------------------
|
105
|
+
|
106
|
+
def manifest_for(resource)
|
107
|
+
return resource.manifest if resource.is_a? Decidim::Participable
|
108
|
+
return resource.resource_manifest if resource.is_a? Decidim::Resourceable
|
109
|
+
end
|
110
|
+
|
111
|
+
def scope_name(space)
|
112
|
+
manifest_for(space).content_blocks_scope_name
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
data/lib/decidim/cdtb/tasks.rb
CHANGED
@@ -3,8 +3,10 @@
|
|
3
3
|
require "decidim/cdtb/tasks_utils"
|
4
4
|
require "decidim/cdtb/task"
|
5
5
|
require "decidim/cdtb/fixes/nickname_fixer"
|
6
|
+
require "decidim/cdtb/fixes/you_tube_embeds_fixer"
|
6
7
|
require "decidim/cdtb/users/remover"
|
7
8
|
require "decidim/cdtb/multitenants/org_by_host_like"
|
9
|
+
require "decidim/cdtb/participatory_spaces/add_content_blocks"
|
8
10
|
require "decidim/cdtb/spam/user_spam_detector"
|
9
11
|
require "decidim/cdtb/storage/local_sharding"
|
10
12
|
require "decidim/cdtb/storage/set_local_on_blobs"
|
data/lib/decidim/cdtb/version.rb
CHANGED
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "decidim/version"
|
4
|
+
require "decidim/cdtb/tasks"
|
5
|
+
|
6
|
+
namespace :cdtb do
|
7
|
+
namespace :embeds do
|
8
|
+
desc <<~EODESC
|
9
|
+
Fix YouTube embeds to Decidim v0.28 format.
|
10
|
+
Only youtube is supported right now.
|
11
|
+
EODESC
|
12
|
+
task fix_youtube: [:environment] do
|
13
|
+
unless Decidim.version >= "0.28"
|
14
|
+
puts "This command is only compatible with Decidim v0.28 or higher"
|
15
|
+
exit(-1)
|
16
|
+
end
|
17
|
+
|
18
|
+
fixer= ::Decidim::Cdtb::Fixes::YouTubeEmbedsFixer.new
|
19
|
+
fixer.execute!
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/tasks/logs.rake
CHANGED
@@ -2,15 +2,117 @@
|
|
2
2
|
|
3
3
|
namespace :cdtb do
|
4
4
|
namespace :logs do
|
5
|
-
desc "Analize logs in Rails format. Counts the number of requests for each IP in the logs. Accepts a
|
5
|
+
desc "Analize logs in Rails format. Counts the number of requests for each IP in the logs. Accepts a log path param."
|
6
6
|
task :num_rq_per_ip, [:logfile] do |_task, args|
|
7
|
-
|
7
|
+
file_path= args.logfile || "log/production.log"
|
8
8
|
|
9
|
-
file_path= "log/#{logfile}"
|
10
9
|
first_cmd= "grep Started"
|
11
10
|
piped_cmds= [%(grep " for "), "cut -d ' ' -f13", "sort", "uniq -c", "sort"].join(" | ")
|
12
|
-
|
13
|
-
|
11
|
+
cmd= "#{first_cmd} #{file_path} | #{piped_cmds}"
|
12
|
+
|
13
|
+
puts "Running: `#{cmd}`"
|
14
|
+
puts `#{cmd}`
|
15
|
+
end
|
16
|
+
|
17
|
+
desc "Analize logs in Rails format. Counts the number of requests for each IP range in the logs. Accepts a log path param."
|
18
|
+
task :num_rq_per_ip_range, [:logfile] do |_task, args|
|
19
|
+
file_path= args.logfile || "log/production.log"
|
20
|
+
|
21
|
+
first_cmd= "grep Started"
|
22
|
+
piped_cmds= [
|
23
|
+
%(grep " for "),
|
24
|
+
# take the IP
|
25
|
+
"cut -d ' ' -f13",
|
26
|
+
# trim blank spaces
|
27
|
+
"awk '{$1=$1;print}'",
|
28
|
+
# range: only the first 2 blocks of the IP
|
29
|
+
"cut -d '.' -f1,2",
|
30
|
+
"sort",
|
31
|
+
"uniq -c",
|
32
|
+
"sort"
|
33
|
+
].join(" | ")
|
34
|
+
cmd= "#{first_cmd} #{file_path} | #{piped_cmds}"
|
35
|
+
|
36
|
+
puts "Running: `#{cmd}`"
|
37
|
+
puts `#{cmd}`
|
38
|
+
end
|
39
|
+
|
40
|
+
desc 'Returns the duration, the order in the logs, and the timestamp of each "Completed" trace.'
|
41
|
+
task :slow_rq, [:logfile] do |_task, args|
|
42
|
+
# Based on the follwoing example execution:
|
43
|
+
# grep -e "Completed 200 OK in" log/development.log | ruby logparser.rb | sort
|
44
|
+
|
45
|
+
file_path= args.logfile || "log/production.log"
|
46
|
+
|
47
|
+
idx= 0
|
48
|
+
File.open(file_path).each_line do |line|
|
49
|
+
idx += 1
|
50
|
+
next unless line.include?("Completed 200 OK in")
|
51
|
+
|
52
|
+
split= line.split("Completed 200 OK in")
|
53
|
+
second_part= split.last.strip
|
54
|
+
words= second_part.split
|
55
|
+
response_duration= words.first
|
56
|
+
puts "#{response_duration.rjust(7, "0")}, #{idx}, #{line[4, 22]}"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
desc "Most requested paths. Accepts a log path param, and an only_path to ignore the query part of the URL."
|
61
|
+
task :most_rq_paths, [:logfile, :only_path] do |_task, args|
|
62
|
+
file_path= args.logfile || "log/production.log"
|
63
|
+
|
64
|
+
first_cmd= "grep Started"
|
65
|
+
piped_cmds= ["cut -d ' ' -f11"]
|
66
|
+
piped_cmds << "cut -d '?' -f 1" if args.only_path == "only_path"
|
67
|
+
piped_cmds+= ["sort", "uniq -c", "sort"]
|
68
|
+
cmd= "#{first_cmd} #{file_path} | #{piped_cmds.join(" | ")}"
|
69
|
+
|
70
|
+
puts "Running: `#{cmd}`"
|
71
|
+
puts `#{cmd}`
|
72
|
+
end
|
73
|
+
|
74
|
+
desc "Analizes output from num_rq_per_ip. Finds the country from each IP in the log file. Accepts a log path param."
|
75
|
+
task :geolocate_ips, [:logfile] do |_task, args|
|
76
|
+
file_path= args.logfile || "log/num_rq_per_ip.log"
|
77
|
+
|
78
|
+
cmd= [
|
79
|
+
# strip spaces
|
80
|
+
"awk '{$1=$1;print}' #{file_path}",
|
81
|
+
"cut -d ' ' -f2",
|
82
|
+
"sort"
|
83
|
+
].join(" | ")
|
84
|
+
|
85
|
+
cmd_out= `#{cmd}`
|
86
|
+
cmd_out.each_line do |ip|
|
87
|
+
ip= ip.strip
|
88
|
+
locations= Geocoder.search(ip)
|
89
|
+
location_rs= locations.first
|
90
|
+
country= parse_nominatim_result(location_rs) || parse_ipinfoio_result(location_rs) || "N/F"
|
91
|
+
puts "#{ip},#{country}"
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def parse_nominatim_result(result)
|
96
|
+
result&.dig("address", "country")
|
97
|
+
end
|
98
|
+
|
99
|
+
def parse_ipinfoio_result(result)
|
100
|
+
result&.dig("country")
|
101
|
+
end
|
102
|
+
|
103
|
+
desc "Lists from the output of geolocated_ips. Finds the country from each IP in the csv file. Accepts a file path param."
|
104
|
+
task :bannable_ips, [:path, :countries] do |_task, args|
|
105
|
+
file_path= args.path || "tmp/geolocated_ips.csv"
|
106
|
+
countries= args.countries || %w[FR GB IT PT]
|
107
|
+
|
108
|
+
cmd= ["grep -v ES #{file_path}"]
|
109
|
+
countries.each do |country_iso|
|
110
|
+
cmd << "grep -v #{country_iso}"
|
111
|
+
end
|
112
|
+
cmd << "cut -d ',' -f1"
|
113
|
+
cmd << "sort"
|
114
|
+
|
115
|
+
puts `#{cmd.join(" | ")}`
|
14
116
|
end
|
15
117
|
end
|
16
118
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "decidim/version"
|
4
|
+
require "decidim/cdtb/tasks"
|
5
|
+
|
6
|
+
namespace :cdtb do
|
7
|
+
namespace :participatory_spaces do
|
8
|
+
desc <<~EODESC
|
9
|
+
Add content blocks to a participatory processes
|
10
|
+
EODESC
|
11
|
+
task :add_content_blocks, [:content_block_names] => :environment do |_task, args|
|
12
|
+
unless Decidim.version >= "0.28"
|
13
|
+
puts "This command is only compatible with Decidim v0.28 or higher"
|
14
|
+
exit(-1)
|
15
|
+
end
|
16
|
+
|
17
|
+
content_block_names = args[:content_block_names].split
|
18
|
+
|
19
|
+
puts "\n Select participatory spaces you want to add the content blocks: #{content_block_names}"
|
20
|
+
puts "\n 1. Decidim::ParticipatoryProcess"
|
21
|
+
puts "\n 2. Decidim::Assembly"
|
22
|
+
puts "\n 3. All"
|
23
|
+
|
24
|
+
selected_option = $stdin.gets.chomp
|
25
|
+
puts selected_option
|
26
|
+
|
27
|
+
case selected_option
|
28
|
+
when "1"
|
29
|
+
processed_models = [
|
30
|
+
"Decidim::ParticipatoryProcess"
|
31
|
+
].freeze
|
32
|
+
when "2"
|
33
|
+
processed_models = [
|
34
|
+
"Decidim::Assembly"
|
35
|
+
].freeze
|
36
|
+
when "3"
|
37
|
+
processed_models = [
|
38
|
+
"Decidim::ParticipatoryProcess",
|
39
|
+
"Decidim::Assembly"
|
40
|
+
].freeze
|
41
|
+
else
|
42
|
+
return "Please, select an option"
|
43
|
+
end
|
44
|
+
|
45
|
+
adder = ::Decidim::Cdtb::ParticipatorySpaces::AddContentBlocks.new(processed_models, content_block_names)
|
46
|
+
adder.execute!
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: decidim-cdtb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Oliver Valls
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2025-01-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: decidim
|
@@ -105,7 +105,9 @@ files:
|
|
105
105
|
- lib/decidim/cdtb.rb
|
106
106
|
- lib/decidim/cdtb/engine.rb
|
107
107
|
- lib/decidim/cdtb/fixes/nickname_fixer.rb
|
108
|
+
- lib/decidim/cdtb/fixes/you_tube_embeds_fixer.rb
|
108
109
|
- lib/decidim/cdtb/multitenants/org_by_host_like.rb
|
110
|
+
- lib/decidim/cdtb/participatory_spaces/add_content_blocks.rb
|
109
111
|
- lib/decidim/cdtb/spam/user_spam_detector.rb
|
110
112
|
- lib/decidim/cdtb/storage/local_sharding.rb
|
111
113
|
- lib/decidim/cdtb/storage/set_local_on_blobs.rb
|
@@ -122,8 +124,10 @@ files:
|
|
122
124
|
- lib/generators/cdtb/validate_migrations_ci_generator.rb
|
123
125
|
- lib/tasks/anonymize.rake
|
124
126
|
- lib/tasks/census.rake
|
127
|
+
- lib/tasks/fix_youtube_embeds.rake
|
125
128
|
- lib/tasks/logs.rake
|
126
129
|
- lib/tasks/multitenants.rake
|
130
|
+
- lib/tasks/participatory_spaces.rake
|
127
131
|
- lib/tasks/spam.rake
|
128
132
|
- lib/tasks/storage.rake
|
129
133
|
- lib/tasks/upgrade.rake
|