decidim-core 0.28.2 → 0.28.3

data/lib/decidim/asset_router/storage.rb

@@ -6,6 +6,23 @@ module Decidim
     # saved through ActiveStorage. This handles the different cases for routing
     # to the remote routes when using an assets CDN or to local routes when
     # using the local disk storage driver.
+    #
+    # Note that when the assets are stored in a remote storage service, such as
+    # Amazon S3, Google Cloud Storage or Azure Storage, this generates the asset
+    # URL directly to the storage service itself bypassing the Rails server and
+    # saving CPU time from serving the asset redirect requests. This causes a
+    # significant performance improvement on pages that display a lot of images.
+    # It will also produce a less significant performance improvement when using
+    # the local disk storage because in this situation, the images are served
+    # using one request instead of two when served directly from the storage
+    # service rather than through the asset redirect URL.
+    #
+    # When implementing changes to the logic, please keep the remote storage
+    # options and performance implications in mind because the specs for this
+    # utility do not cover the remote storage options because the extra
+    # configuration needed to test, the service itself needed for testing and
+    # the extra dependency overhead for adding these remote storage gems when
+    # they are not needed.
     class Storage
       # Initializes the router.
       #
@@ -13,25 +30,36 @@ module Decidim
       #   to
       def initialize(asset)
         @asset = asset
+        @blob =
+          case asset
+          when ActiveStorage::Blob
+            asset
+          else
+            asset&.blob
+          end
       end
 
       # Generates the correct URL to the asset with the provided options.
       #
       # @param options The options for the URL that are the normal route options
       #   Rails route helpers accept
+      # @return [String] The URL of the asset
       def url(**options)
-        if asset.is_a? ActiveStorage::Attached
-          routes.rails_blob_url(asset.blob, **default_options.merge(options))
-        elsif asset.is_a? ActiveStorage::Blob
-          routes.rails_blob_url(asset, **default_options.merge(options))
-        else
+        case asset
+        when ActiveStorage::Attached
+          ensure_current_host(asset.record, **options)
+          blob_url(**options)
+        when ActiveStorage::Blob
+          blob_url(**options)
+        else # ActiveStorage::VariantWithRecord, ActiveStorage::Variant
+          ensure_current_host(nil, **options)
           representation_url(**options)
         end
       end
 
       private
 
-      attr_reader :asset
+      attr_reader :asset, :blob
 
       # Provides the route helpers depending on whether the URL is generated to
       # the local host or an external CDN (remote).
@@ -80,24 +108,199 @@ module Decidim
         }.compact
       end
 
-      # Converts the variation URLs last part to the correct file extension in
-      # case the variation has a different format than the original image.
+      # Most of the times the current host should be set through the controller
+      # already when the logic below is unnecessary. This logic is needed e.g.
+      # for serializers where the request context is not available.
+      #
+      # @param record The record for which to check the organization
+      # @param opts Options for building the URL
+      # @return [void]
+      def ensure_current_host(record, **opts)
+        return if asset_url_available?
+
+        options = remote? ? remote_storage_options : routes.default_url_options
+        options = options.merge(opts)
+
+        if opts[:host].blank? && record.present?
+          organization = organization_for(record)
+          options[:host] = organization.host if organization
+        end
+
+        uri =
+          if options[:protocol] == "https" || options[:scheme] == "https"
+            URI::HTTPS.build(options)
+          else
+            URI::HTTP.build(options)
+          end
+
+        ActiveStorage::Current.host = uri.to_s
+      end
+
+      # Determines the organization for the passed record.
       #
-      # @return [String] The converted representation URL
+      # @param record The record for which to fetch the organization
+      # @return [Decidim::Organization, nil] The organization for the record or
+      #   `nil` if the organization cannot be determined
+      def organization_for(record)
+        if record.is_a?(Decidim::Organization)
+          record
+        elsif record.respond_to?(:organization)
+          record.organization
+        end
+      end
+
+      # Returns the URL for the given blob object.
+      #
+      # @param blob The blob object
+      # @param options Options for building the URL
+      # @return [String, nil] The URL to the blob object or `nil` if the blob is
+      #   not defined.
+      def blob_url(**options)
+        return unless blob
+
+        if options[:only_path] || remote? || !asset_url_available?
+          routes.rails_blob_url(blob, **default_options.merge(options))
+        else
+          blob.url(**options)
+        end
+      end
+
+      # Returns a representation URL for the asset either directly through the
+      # storage service or through the Rails representation URL in case the
+      # path URL is requested or if the asset variant has not been processed yet
+      # and is not therefore yet stored at the storage service.
+      #
+      # @return [String] The representation URL for the image variant
       def representation_url(**options)
+        return rails_representation_url(**options) if options[:only_path] || remote?
+
+        representation_url = variant_url(**options)
+        return representation_url if representation_url.present?
+
+        # In case the representation has not been processed yet, it may not have
+        # a representation URL yet and it therefore needs to be served through
+        # the local representation URL for the first time (or until it has been
+        # processed).
+        if options[:host]
+          rails_representation_url(**options)
+        else
+          representation_url(**options.merge(only_path: true))
+        end
+      end
+
+      # Returns the local Rails representation URL meaning that the asset will
+      # be served through the service itself. This may be necessary if the asset
+      # variant (e.g. a thumbnail) has not been processed yet because the
+      # variant representation has not been requested before.
+      #
+      # Due to performance reasons it is advised to avoid requesting the assets
+      # through the Rails representation URLs when possible because that causes
+      # a lot of requests to the Rails backend and slowness to the service under
+      # heavy loads.
+      #
+      # Converts the variation URLs last part to the correct file extension in
+      # case the variation has a different format than the original image. The
+      # conversion needs to be only done for the Rails representation URLs
+      # because once the image is stored at the storage service, it already has
+      # the correct file extension.
+      #
+      # @param options The options for building the URL
+      # @return [String, nil] The converted representation URL or `nil` if the
+      #   asset is not defined.
+      def rails_representation_url(**options)
+        return unless asset
+
         representation_url = routes.rails_representation_url(asset, **default_options.merge(options))
+
         variation = asset.try(:variation)
         return representation_url unless variation
 
         format = variation.try(:format)
         return representation_url unless format
+        return unless blob
 
-        original_ext = File.extname(asset.blob.filename.to_s)
+        original_ext = File.extname(blob.filename.to_s)
         return representation_url if original_ext == ".#{format}"
 
-        basename = File.basename(asset.blob.filename.to_s, original_ext)
+        basename = File.basename(blob.filename.to_s, original_ext)
         representation_url.sub(/#{basename}\.#{original_ext.tr(".", "")}$/, "#{basename}.#{format}")
       end
+
+      # Fetches the image variant's URL at the storage service if the variant
+      # has already been processed and is stored at the storage service. If the
+      # variant has not been processed yet, returns `nil` in which case the
+      # variant has to be served through the service's own representation URL
+      # causing it to be processed and stored at the storage service.
+      #
+      # @param options The options for building the URL
+      # @return [String, nil] The variant URL at the storage service or `nil` if
+      #   the variant has not been processed yet and does not yet exist at the
+      #   storage service or `nil` when the asset is not defined
+      def variant_url(**options)
+        return unless asset
+        return unless asset_url_available?
+        return unless asset_exist?
+
+        case asset
+        when ActiveStorage::VariantWithRecord
+          # This is used when `ActiveStorage.track_variants` is enabled through
+          # `config.active_storage.track_variants`. In case the variant has not
+          # been processed yet, the `#url` method would return nil.
+          #
+          # Note that if the `asset.processed?` returns `true`, the variant
+          # record has been created in the database but it does not mean that
+          # it has been uploaded to the storage service yet. Likely a bug in
+          # ActiveStorage but to be sure that the asset is uploaded to the
+          # storage service, we also check that.
+          asset.url(**options) if asset.processed?
+        else # ActiveStorage::Variant
+          # Check whether the variant exists at the storage service before
+          # returning its URL. Otherwise the URL would be returned even when the
+          # variant is not yet processed causing 404 errors for the images on
+          # the page.
+          #
+          # Note that the `ActiveStorage::Variant#url` method only accepts
+          # certain keyword arguments where as the other objects allow any
+          # keyword arguments.
+          possible_kwargs = asset.method(:url).parameters.select { |p| p[0] == :key }.map { |p| p[1] }
+          asset.url(**options.slice(*possible_kwargs))
+        end
+      end
+
+      # Determines if the asset exists at the storage service.
+      #
+      # @return [Boolean] A boolean answering the question "does this asset
+      # exist at the storage service?".
+      def asset_exist?
+        return false if asset.key.blank?
+
+        blob.service.exist?(asset.key)
+      end
+
+      # Determines if the current host is required to build the asset URL.
+      #
+      # @return [Boolean] A boolean indicating if the current host is required
+      #   to build the asset URL.
+      def current_host_required?
+        return false unless blob
+        return false unless defined?(ActiveStorage::Service::DiskService)
+
+        blob.service.is_a?(ActiveStorage::Service::DiskService)
+      end
+
+      # Determines if the asset URL can be generated.
+      #
+      # @return [Boolean] A boolean indicating if the asset URL can be
+      #   generated.
+      def asset_url_available?
+        # If the service is an external service, the URL can be generated
+        # regardless of the current host being set.
+        return true unless current_host_required?
+
+        # For the disk service, the URL can be only generated if the current
+        # host has been set.
+        ActiveStorage::Current.host.present?
+      end
     end
   end
 end

data/lib/decidim/core/engine.rb

@@ -237,6 +237,14 @@ module Decidim
         app.config.action_mailer.deliver_later_queue_name = :mailers
       end
 
+      initializer "decidim_core.signed_global_id", after: "global_id" do |app|
+        next if app.config.global_id.fetch(:expires_in, nil).present?
+
+        config.after_initialize do
+          SignedGlobalID.expires_in = nil
+        end
+      end
+
       initializer "decidim_core.middleware" do |app|
         if app.config.public_file_server.enabled
           headers = app.config.public_file_server.headers || {}

data/lib/decidim/core/test/shared_examples/attachable_interface_examples.rb

@@ -10,7 +10,7 @@ shared_examples_for "attachable interface" do
 
     it "includes the attachment urls" do
       attachment_urls = response["attachments"].map { |attachment| attachment["url"] }
-      expect(attachment_urls).to include(*attachments.map(&:url))
+      expect(attachment_urls).to include_blob_urls(*attachments.map(&:file).map(&:blob))
     end
   end
 end

data/lib/decidim/core/test/shared_examples/follows_examples.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
 
-shared_examples "follows" do
+# When using these shared examples, make sure there are no prior requests within
+# the same group of examples where this is included. Otherwise you may end up
+# in race conditions that cause these to fail as explained at:
+# https://github.com/decidim/decidim/pull/6161
+shared_examples "followable content for users" do
   before do
+    switch_to_host(organization.host)
     login_as user, scope: :user
   end
 
@@ -34,9 +39,9 @@ shared_examples "follows" do
   end
 end
 
-shared_examples "follows with a component" do
+shared_examples "followable content for users with a component" do
   include_context "with a component"
-  include_examples "follows"
+  include_examples "followable content for users"
 
   context "when the user is following the followable's participatory space" do
     before do

data/lib/decidim/core/test/shared_examples/paginated_resource_examples.rb

@@ -1,22 +1,22 @@
 # frozen_string_literal: true
 
 shared_examples "a paginated resource" do
-  let(:collection_size) { 30 }
+  let(:collection_size) { 50 }
 
   before do
     visit_component
   end
 
-  it "lists 10 resources per page by default" do
-    expect(page).to have_css(resource_selector, count: 10)
-    expect(page).to have_css("[data-pages] [data-page]", count: 3)
+  it "lists 25 resources per page by default" do
+    expect(page).to have_css(resource_selector, count: 25)
+    expect(page).to have_css("[data-pages] [data-page]", count: 2)
   end
 
   it "results per page can be changed from the selector" do
     expect(page).to have_css("[data-pagination]")
 
     within "[data-pagination]" do
-      page.find("summary", text: "10").click
+      page.find("summary", text: "25").click
       click_link "50"
     end
 

data/lib/decidim/core/version.rb

@@ -4,7 +4,7 @@ module Decidim
   # This holds the decidim-core version.
   module Core
     def self.version
-      "0.28.2"
+      "0.28.3"
     end
   end
 end

data/lib/decidim/seven_zip_wrapper.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+module Decidim
+  class SevenZipWrapper
+    class << self
+      def compress_and_encrypt(filename:, password:, input_directory:)
+        run("cd #{escape(input_directory)} && 7z a -tzip -p#{escape(password)} -mem=AES256 #{escape(filename)} .")
+      end
+
+      def extract_and_decrypt(filename:, password:, output_directory:)
+        run("7z x -tzip #{escape(filename)} -o#{escape(output_directory)} -p#{escape(password)}")
+      end
+
+      private
+
+      def run(command)
+        success = system(command)
+
+        raise "Command failed: #{command}" unless success
+      end
+
+      def escape(string)
+        Shellwords.escape(string)
+      end
+    end
+  end
+end

data/lib/tasks/upgrade/decidim_fix_categorization.rake

@@ -2,7 +2,107 @@
 
 namespace :decidim do
   namespace :upgrade do
-    desc "Remove orphan categorizations"
+    namespace :clean do
+      desc "Removes all the invalid records from search, notifications, follows and action_logs"
+      task invalid_records: [
+        :"decidim:upgrade:clean:searchable_resources",
+        :"decidim:upgrade:clean:notifications",
+        :"decidim:upgrade:clean:follows",
+        :"decidim:upgrade:clean:action_logs"
+      ]
+
+      desc "Removes any action logs belonging to invalid resources"
+      task :action_logs, [] => :environment do
+        puts "=== Deleting Action logs\n"
+        invalid = 0
+        Decidim::ActionLog.find_each do |log|
+          log.participatory_space if log.participatory_space_type.present?
+          log.resource if log.resource_type.present?
+
+          if log.resource_type == "Decidim::Component" && log.resource.blank?
+            log.delete
+            invalid += 1
+          end
+
+          next if log.decidim_component_id.blank?
+          next if log.component.present?
+
+          log.delete
+          invalid += 1
+        rescue NameError
+          log.delete
+          invalid += 1
+        end
+        puts "===== Deleted #{invalid} invalid action logs\n"
+      end
+
+      desc "Removes any follows belonging to invalid resources"
+      task :follows, [] => :environment do
+        puts "=== Deleting Follows\n"
+        invalid = 0
+        Decidim::Follow.find_each do |follow|
+          follow.followable
+
+          next unless follow.followable.respond_to?(:component)
+          next if follow.followable.component.present?
+
+          # We attempt to remove any of the follows that refer to spaces or components that disappeared
+          follow.destroy
+          invalid += 1
+        rescue NameError
+          # We use delete as we do not want to call the hooks
+          follow.delete
+          invalid += 1
+        end
+        puts "===== Deleted #{invalid} invalid follows\n"
+      end
+
+      desc "Removes any notifications belonging to invalid resources"
+      task :notifications, [] => :environment do
+        puts "=== Deleting Notification\n"
+        invalid = 0
+        Decidim::Notification.find_each do |notification|
+          # Check if the resource class still exists
+          notification.resource
+          # Check if the event class still exists
+          notification.event_class_instance
+        rescue NameError
+          notification.destroy
+          invalid += 1
+        end
+        puts "===== Deleted #{invalid} invalid notifications\n"
+      end
+
+      desc "Removes any resources from search index that do not exist"
+      task :searchable_resources, [] => :environment do
+        puts "=== Deleting Searchable results\n"
+        puts "==== Deleting invalid spaces \n"
+        invalid = 0
+        Decidim::SearchableResource.where.not(decidim_participatory_space_type: nil).find_each do |search|
+          search.decidim_participatory_space
+        rescue NameError
+          search.destroy!
+          invalid += 1
+        end
+        puts "===== Deleted #{invalid} invalid spaces\n"
+
+        puts "==== Deleting invalid resources from search index \n"
+        invalid = 0
+        Decidim::SearchableResource.find_each do |search|
+          next unless search.resource.respond_to?(:component)
+          next if search.resource.component.present?
+
+          search.destroy
+          invalid += 1
+        rescue NameError
+          search.destroy!
+          invalid += 1
+        end
+        puts "===== Deleted #{invalid} invalid resources\n"
+      end
+    end
+
+    desc "Removes orphan categorizations"
     task fix_orphan_categorizations: :environment do
       logger = Logger.new($stdout)
       logger.info("Removing orphan categorizations...")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: decidim-core
 version: !ruby/object:Gem::Version
-  version: 0.28.2
+  version: 0.28.3
 platform: ruby
 authors:
 - Josep Jaume Rey Peroy
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-16 00:00:00.000000000 Z
+date: 2024-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: active_link_to
@@ -688,20 +688,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: seven_zip_ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.3'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: shakapacker
   requirement: !ruby/object:Gem::Requirement
@@ -764,28 +750,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.28.2
+        version: 0.28.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.28.2
+        version: 0.28.3
 - !ruby/object:Gem::Dependency
   name: decidim-dev
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.28.2
+        version: 0.28.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.28.2
+        version: 0.28.3
 description: Adds core features so other engines can hook into the framework.
 email:
 - josepjaume@gmail.com
@@ -1880,7 +1866,6 @@ files:
 - app/services/decidim/static_map_generator.rb
 - app/services/decidim/tokenizer.rb
 - app/services/decidim/traceability.rb
-- app/services/decidim/zip_stream/writer.rb
 - app/uploaders/decidim/application_uploader.rb
 - app/uploaders/decidim/attachment_uploader.rb
 - app/uploaders/decidim/avatar_uploader.rb
@@ -2814,6 +2799,7 @@ files:
 - lib/decidim/searchable.rb
 - lib/decidim/seeds.rb
 - lib/decidim/settings_manifest.rb
+- lib/decidim/seven_zip_wrapper.rb
 - lib/decidim/shareable_with_token.rb
 - lib/decidim/snippets.rb
 - lib/decidim/social_share_service_manifest.rb

data/app/services/decidim/zip_stream/writer.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module Decidim
-  module ZipStream
-    module Writer
-      def add_user_data_to_zip_stream(out, user_data)
-        user_data.each do |element|
-          filename_file = element.last.filename(element.first.parameterize)
-
-          out.put_next_entry(filename_file)
-          if element.last.read.presence
-            out.write element.last.read
-          else
-            out.write "No data"
-          end
-        end
-      end
-
-      def add_attachments_to_zip_stream(out, export_attachments)
-        export_attachments.each do |attachment_block|
-          next if attachment_block.last.nil?
-
-          folder_name = attachment_block.first.parameterize
-          attachment_block.last.each do |attachment|
-            next unless attachment.attached?
-
-            blobs = attachment.is_a?(ActiveStorage::Attached::One) ? [attachment.blob] : attachment.blobs
-            blobs.each do |blob|
-              out.put_next_entry("#{folder_name}/#{blob.filename}")
-              blob.open do |f|
-                out << f.read
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end