decidim-core 0.29.6 → 0.30.0.rc1

data/lib/tasks/decidim_taxonomies.rake

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "decidim/maintenance"
+
+namespace :decidim do
+  namespace :taxonomies do
+    desc "Creates a JSON file with the taxonomies structure imported from older models"
+    task :make_plan, [] => :environment do |_task, _args|
+      Decidim::Organization.find_each do |organization|
+        log(true).info "Creating a plan for organization #{organization.id} in #{plan_file_path(organization)}"
+        FileUtils.mkdir_p(Rails.root.join("tmp/taxonomies"))
+        json = planner(organization).to_json do |model|
+          log.info "...Exporting taxonomies for #{model.table_name}"
+        end
+        File.write(plan_file_path(organization), json)
+        log.info "Plan created, you can review or edit if needed before importing."
+        log.info "Import the plan with this command:"
+        log.info "bin/rails decidim:taxonomies:import_plan[#{plan_file_path(organization)}]"
+      end
+    end
+
+    desc "Imports taxonomies and filters structure from a JSON file"
+    task :import_plan, [:file] => :environment do |_task, args|
+      file = args[:file].to_s
+      abort "File not found! [#{file}]" unless File.exist?(file)
+
+      data = JSON.parse(File.read(file))
+      organization = Decidim::Organization.find_by(id: data.dig("organization", "id"))
+      abort "Organization not found! [#{data["organization"]}]" unless organization
+      log(true).info "Importing taxonomies and filters for organization #{organization.id}"
+
+      planner(organization).import(data) do |importer, model_name|
+        taxonomies = importer.roots
+        result = importer.result
+        log.info "...Importing #{taxonomies.count} root taxonomies from #{model_name}"
+        taxonomies.each do |name, taxonomy|
+          log.info "  - Root taxonomy: #{name}"
+          log.info "    1st level taxonomies: #{taxonomy["taxonomies"].count}"
+          log.info "    Filters: #{taxonomy["filters"].count}"
+          taxonomy["filters"].each do |filter|
+            log.info "      - Filter name: #{filter["name"]}"
+            log.info "        Internal name: #{filter["internal_name"] || "-"}"
+            log.info "        Space manifests: #{filter["participatory_space_manifests"]&.join(", ") || "-"}"
+            log.info "        Items: #{filter["items"].count}"
+            log.info "        Components: #{filter["components"].count}"
+          end
+        end
+        importer.import! do |type, detail|
+          log.info "    !#{type}: #{detail}"
+        end
+        log.info "    Created taxonomies: #{result[:taxonomies_created].count}"
+        result[:taxonomies_created].each do |name|
+          log.info "      - #{name}"
+        end
+        log.info "    Created filters: #{result[:filters_created].count}"
+        result[:filters_created].each do |name, items|
+          log.info "      - #{name}: #{items.count} items"
+        end
+        log.info "    Assigned resources: #{result[:taxonomies_assigned].count}"
+        result[:taxonomies_assigned].each do |name, items|
+          log.info "      - #{name}: #{items.count} resources"
+        end
+        log.info "    Assigned components: #{result[:components_assigned].count}"
+        result[:components_assigned].each do |name, items|
+          log.info "      - #{name}: #{items.count} components"
+        end
+        log.info "    Failed resources: #{result[:failed_resources].count}"
+        result[:failed_resources].each do |object_id|
+          log.info "      - #{object_id}"
+        end
+        log.info "    Failed components: #{result[:failed_components].count}"
+        result[:failed_components].each do |component_id|
+          log.info "      - #{component_id}"
+        end
+        Rails.root.join("tmp/taxonomies", "#{organization.host}_result.json").write(JSON.pretty_generate(result))
+        log.info "Result saved in tmp/taxonomies/#{organization.host}_result.json"
+      end
+      log.info "Taxonomies and filters imported successfully."
+    end
+
+    desc "Imports taxonomies and filters structure from all JSON files inside tmp/taxonomies"
+    task :import_all_plans, [] => :environment do |_task, _args|
+      Rails.root.glob("tmp/taxonomies/*_plan.json").each do |file|
+        log.info "Importing plan from #{file}"
+        Rake::Task["decidim:taxonomies:import_plan"].invoke(file)
+      end
+    end
+
+    desc "Adds taxonomies to metrics from old categories"
+    task :update_metrics, [:file] => :environment do |_task, args|
+      file = args[:file].to_s
+      abort "File not found! [#{file}]" unless File.exist?(file)
+
+      data = JSON.parse(File.read(file))
+      taxonomies = data["taxonomy_map"]
+      abort "No taxonomies found in the file" unless taxonomies && taxonomies&.any?
+
+      total = taxonomies.count
+      taxonomies.each_with_index do |(id, object_id), index|
+        next unless object_id.include?("ImportModels::Category")
+
+        category_id = object_id.split("/").last
+        percent = ((index + 1) * 100 / total).to_i
+
+        count = Decidim::Metric.where(decidim_category_id: category_id).count
+        log.info "...#{percent}% Updating #{count} metrics for category #{category_id} to taxonomy #{id}"
+        Decidim::Metric.where(decidim_category_id: category_id).update_all(decidim_taxonomy_id: id) # rubocop:disable Rails/SkipsModelValidations
+      end
+    end
+
+    desc "Processes all metrics for result files in tmp/taxonomies"
+    task :update_all_metrics, [] => :environment do |_task, _args|
+      Rails.root.glob("tmp/taxonomies/*_result.json").each do |file|
+        log.info "Processing metrics from #{file}"
+        Rake::Task["decidim:taxonomies:update_metrics"].invoke(file)
+      end
+    end
+
+    desc "Reset counters for taxonomies and taxonomy filters"
+    task :reset_counters, [] => :environment do |_task, _args|
+      Decidim::Taxonomy.find_each do |taxonomy|
+        taxonomy.reset_all_counters
+        log.info "Counters reset for taxonomy #{taxonomy.id}:"
+        log.info "  Children: #{taxonomy.children_count}"
+        log.info "  Taxonomizations: #{taxonomy.taxonomizations_count}"
+        log.info "  Filters: #{taxonomy.filters_count}"
+        log.info "  Filter items: #{taxonomy.filter_items_count}"
+      end
+      Decidim::TaxonomyFilter.find_each do |filter|
+        filter.reset_all_counters
+        log.info "Counters reset for taxonomy filter #{filter.id}:"
+        log.info "  Filter items: #{filter.filter_items_count}"
+        log.info "  Components: #{filter.components_count}"
+      end
+    end
+
+    def planner(organization)
+      models = ENV.fetch("IMPORTS", "ParticipatoryProcessType AssemblyType Scope Area Category").split.map do |model_name|
+        "Decidim::Maintenance::ImportModels::#{model_name}".constantize
+      end
+      Decidim::Maintenance::TaxonomyPlan.new(organization, models)
+    end
+
+    def plan_file_path(organization)
+      Rails.root.join("tmp/taxonomies", "#{organization.host}_plan.json")
+    end
+
+    def log(reset = false) # rubocop:disable Style/OptionalBooleanParameter
+      @log = nil if reset
+      @log ||= Logger.new($stdout, formatter: proc { |_severity, _time, _progname, msg|
+        "#{msg}\n"
+      })
+    end
+  end
+end

data/lib/tasks/upgrade/{decidim_fix_categorization.rake → clean.rake}

@@ -8,6 +8,7 @@ namespace :decidim do
         :"decidim:upgrade:clean:searchable_resources",
         :"decidim:upgrade:clean:notifications",
         :"decidim:upgrade:clean:follows",
+        :"decidim:upgrade:clean:categories",
         :"decidim:upgrade:clean:action_logs",
         :"decidim:upgrade:clean:clean_deleted_users",
         :"decidim:upgrade:clean:fix_blocked_user_notification"
@@ -15,14 +16,13 @@ namespace :decidim do
 
       desc "Remove data from deleted users"
       task clean_deleted_users: :environment do
-        logger = Logger.new($stdout)
         logger.info("=== Removing extra data from deleted users")
         Decidim::User.where.not(deleted_at: nil).update_all(personal_url: "", about: "", notifications_sending_frequency: "none") # rubocop:disable Rails/SkipsModelValidations
       end
 
       desc "Removes any action logs belonging to invalid resources"
       task :action_logs, [] => :environment do
-        puts "=== Deleting Action logs\n"
+        logger.info("=== Deleting Action logs")
         invalid = 0
         Decidim::ActionLog.find_each do |log|
           log.participatory_space if log.participatory_space_type.present?
@@ -42,12 +42,12 @@ namespace :decidim do
           log.delete
           invalid += 1
         end
-        puts "===== Deleted #{invalid} invalid action logs\n"
+        logger.info("===== Deleted #{invalid} invalid action logs")
       end
 
       desc "Removes any follows belonging to invalid resources"
       task :follows, [] => :environment do
-        puts "=== Deleting Follows\n"
+        logger.info("=== Deleting Follows")
         invalid = 0
         Decidim::Follow.find_each do |follow|
           follow.followable
@@ -63,12 +63,12 @@ namespace :decidim do
           follow.delete
           invalid += 1
         end
-        puts "===== Deleted #{invalid} invalid follows\n"
+        logger.info("===== Deleted #{invalid} invalid follows")
       end
 
       desc "Removes any notifications belonging to invalid resources"
       task :notifications, [] => :environment do
-        puts "=== Deleting Notification\n"
+        logger.info("=== Deleting Notification")
         invalid = 0
         Decidim::Notification.find_each do |notification|
           # Check if the resource class still exists
@@ -79,13 +79,13 @@ namespace :decidim do
           notification.destroy
           invalid += 1
         end
-        puts "===== Deleted #{invalid} invalid notifications\n"
+        logger.info("===== Deleted #{invalid} invalid notifications")
       end
 
       desc "Removes any resources from search index that do not exist"
       task :searchable_resources, [] => :environment do
-        puts "=== Deleting Searchable results\n"
-        puts "==== Deleting invalid spaces \n"
+        logger.info("=== Deleting Searchable results")
+        logger.info("==== Deleting invalid spaces ")
         invalid = 0
         Decidim::SearchableResource.where.not(decidim_participatory_space_type: nil).find_each do |search|
           search.decidim_participatory_space
@@ -93,9 +93,9 @@ namespace :decidim do
           search.destroy!
           invalid += 1
         end
-        puts "===== Deleted #{invalid} invalid spaces\n"
+        logger.info("===== Deleted #{invalid} invalid spaces")
 
-        puts "==== Deleting invalid resources from search index \n"
+        logger.info("==== Deleting invalid resources from search index ")
         invalid = 0
         Decidim::SearchableResource.find_each do |search|
           next unless search.resource.respond_to?(:component)
@@ -107,24 +107,31 @@ namespace :decidim do
           search.destroy!
           invalid += 1
         end
-        puts "===== Deleted #{invalid} invalid resources\n"
+        logger.info("===== Deleted #{invalid} invalid resources")
       end
 
-      desc "Removes orphan categorizations"
-      task fix_orphan_categorizations: :environment do
-        logger = Logger.new($stdout)
-        logger.info("Removing orphan categorizations...")
+      desc "Removes any categorizations belonging to invalid resources"
+      task categories: :environment do
+        logger.info("=== Removing orphan categorizations...")
+        invalid = 0
 
         Decidim::Categorization.find_each do |categorization|
-          categorization.destroy if categorization.categorizable.nil?
+          next unless categorization.categorizable.nil?
+
+          invalid += 1
+          categorization.destroy
+        rescue NameError
+          categorization.destroy!
+          invalid += 1
         end
+        logger.info("===== Deleted #{invalid} invalid resources")
       end
 
       desc "Update all blocked users notifications_sending_frequency setting"
       task fix_blocked_user_notification: :environment do
         logger.info("=== Updating all blocked users notifications_sending_frequency ...")
         blocked_users = 0
-        Decidim::User.blocked.where.not("notifications_sending_frequency = ?", "none").find_each do |blocked_user|
+        Decidim::User.blocked.where.not(notifications_sending_frequency: :none).find_each do |blocked_user|
           unless blocked_user.notifications_sending_frequency == "none"
             blocked_user.update(notifications_sending_frequency: "none")
             blocked_users += 1

data/lib/tasks/upgrade/decidim_active_storage_migration_tasks.rake

@@ -7,7 +7,7 @@ namespace :decidim do
       task :migrate_inline_images_to_active_storage, [:admin_email] => :environment do |_t, args|
         user = Decidim::User.find_by(email: args[:admin_email])
 
-        raise "Invalid admin. Please, provide the email of an admin with permissions to create editor images" unless user&.admin? && user&.admin_terms_accepted?
+        raise "Invalid admin. Please, provide the email of an admin with permissions to create editor images" unless user&.admin? && user.admin_terms_accepted?
 
         inline_images_available_attributes.each do |model, attributes|
           puts "=== Updating model #{model.name} (attributes: #{attributes.join(", ")})..."

data/lib/tasks/upgrade/decidim_fix_nickname_uniqueness.rake

@@ -2,40 +2,37 @@
 
 namespace :decidim do
   namespace :upgrade do
-    desc "Modifies nickname of the user to lower case"
-    task :fix_nickname_casing => :environment do
-      logger.info("Fixing user nicknames case...")
+    desc "Modifies nicknames with random numbers when exists similar ones case-insensitively"
+    task fix_nickname_uniqueness: :environment do
+      logger = Logger.new($stdout)
+      logger.info("Updating conflicting user nicknames...")
 
+      # list of users already changed in the process
       has_changed = []
-      Decidim::UserBaseEntity.not_deleted.find_each do |user|
-        user.nickname.downcase!
 
-        begin
-          if user.nickname_changed?
-            user.save!
-            has_changed << user.id
+      Decidim::User.not_deleted.find_each do |user|
+        next if has_changed.include? user.id
+
+        Decidim::User.where(organization: user.organization)
+                     .where("nickname ILIKE ?", user.nickname.downcase)
+                     .where.not(id: has_changed + [user.id])
+                     .not_deleted
+                     .order(:created_at)
+                     .each do |similar_user|
+          # change her nickname to the lowercased one with numbers if needed
+          begin
+            update_user_nickname(similar_user, Decidim::UserBaseEntity.nicknamize(similar_user.nickname, organization: similar_user.organization))
+          rescue ActiveRecord::RecordInvalid => e
+            logger.warn("User ID (#{similar_user.id}) : #{e}")
           end
-        rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
-          update_user_nickname(user, Decidim::UserBaseEntity.nicknamize(user.nickname, user.decidim_organization_id))
-          has_changed << user.id
-        rescue ActiveRecord::RecordInvalid # rubocop:disable Lint/DuplicateRescueException
-          logger.warn("User ID (#{user.id}) : #{e}")
+          has_changed.append(similar_user.id)
         end
       end
       logger.info("Process terminated, #{has_changed.count} users nickname have been updated.")
     end
 
-    desc "Modifies nicknames with random numbers when exists similar ones case-insensitively"
-    task :fix_nickname_uniqueness => :environment do
-      Rake::Task["decidim:upgrade:fix_nickname_casing"].execute
-    end
-
     private
 
-    def logger
-      @logger ||= Logger.new($stdout)
-    end
-
     def send_notification_to(user)
       Decidim::EventsManager.publish(
         event: "decidim.events.nickname_event",

data/lib/tasks/upgrade/migrations.rake

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+namespace :decidim do
+  namespace :upgrade do
+    desc "Replace legacy migration content"
+    task migrations: :environment do
+      available_modules = {
+        "accountability" => "Accountability",
+        "admin" => "Admin",
+        "ai" => "Ai",
+        "api" => "Api",
+        "assemblies" => "Assemblies",
+        "blogs" => "Blogs",
+        "budgets" => "Budgets",
+        "comments" => "Comments",
+        "conferences" => "Conferences",
+        "core" => "Core",
+        "debates" => "Debates",
+        "design" => "Design",
+        "dev" => "Dev",
+        "forms" => "Forms",
+        "initiatives" => "Initiatives",
+        "meetings" => "Meetings",
+        "pages" => "Pages",
+        "participatory_processes" => "ParticipatoryProcesses",
+        "proposals" => "Proposals",
+        "sortitions" => "Sortitions",
+        "surveys" => "Surveys",
+        "system" => "System",
+        "templates" => "Templates",
+        "verifications" => "Verifications"
+      }
+
+      available_modules.each do |key, value|
+        next unless Decidim.module_installed?(key.to_sym)
+
+        migration_files = Dir["Decidim::#{value}::Engine".constantize.root.join("db/migrate/*.rb")]
+        next if migration_files.blank?
+
+        migration_files.each do |file|
+          next if file.include?("active_storage")
+
+          replace_content(file)
+        end
+      end
+    end
+
+    private
+
+    def replace_content(file_path)
+      migration_file = File.basename(file_path)
+      version = migration_file.split("_").first
+      migration_file_base = migration_file.gsub!(/^\d+_/, "").gsub!(/\.rb$/, "")
+
+      target_file = Rails.root.glob("db/migrate/**_#{migration_file_base}.*.rb").first
+      return if target_file.blank?
+
+      scope = target_file.to_s.split(".")[-2]
+
+      source = File.binread(file_path)
+
+      # This has been extracted from activerecord file lib/active_record/migration.rb
+      inserted_comment = "# This migration comes from #{scope} (originally #{version})\n"
+      magic_comments = +""
+      loop do
+        source.sub!(/\A(?:#.*\b(?:en)?coding:\s*\S+|#\s*frozen_string_literal:\s*(?:true|false)).*\n/) do |magic_comment|
+          magic_comments << magic_comment
+          ""
+        end || break
+      end
+
+      if !magic_comments.empty? && source.start_with?("\n")
+        magic_comments << "\n"
+        source = source[1..-1]
+      end
+
+      new_source = "#{magic_comments}#{inserted_comment}#{source}"
+      old_source = File.binread(target_file)
+
+      return if old_source == new_source
+
+      logger.warn("[Patch migration] Replacing content of #{File.basename(target_file)}")
+
+      File.binwrite(target_file, new_source)
+    end
+
+    def logger
+      @logger ||= Logger.new($stdout)
+    end
+  end
+end