decidim-ai 0.30.0.rc1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3dfbdf71329ed49a7ff135c96047b20b05dad7746bb61885c823f7b759cec3db
+  data.tar.gz: 28742be322499ab899e685dc990d5ef0aa8fdc9e8e6c4fc878f22b976c1e1245
+SHA512:
+  metadata.gz: f112e89075cb0bb1ee98f4df949ea4138a11dce85a12d8c7cb804ef116eb9f17117693fe467d05cc78990223826341297d9c0ca895c1b5fb9b66069521ee8963
+  data.tar.gz: 852350e6ebec0220b63829da254d9187c5ced7639d57fffa14df30cf2776ad11ba0a76b668fc4a7656f27497ec2ae57aa3d7fc463725fdb5cc705fbcf159b8dc

data/README.md

@@ -0,0 +1,47 @@
+# Decidim::Ai
+
+The Decidim::Ai is a library that aims to provide Artificial Intelligence tools for Decidim. This plugin has been initially developed aiming to analyze the content and provide spam classification using Naive Bayes algorithm.
+All AI related functionality provided by Decidim should be included in this same module.
+
+For more documentation on the AI tools API, please refer to [documentation](https://docs.decidim.org/en/develop/develop/ai_tools.html)
+
+## Installation
+
+In order to install use this module, you need at least Decidim 0.30 to be installed.
+
+To install this module, run in your console:
+
+```bash
+bundle add decidim-ai
+```
+
+After that, add an initializer file as presented in the [documentation](https://docs.decidim.org/en/develop/services/aitools.html#_configuration)
+
+Then, you need to run the below command, so that the reporting user is created.
+
+```ruby
+bin/rails decidim:ai:spam:create_reporting_user
+```
+
+Then you can use the below command to train the engine with the module dataset:
+
+```ruby
+bin/rails decidim:ai:spam:load_module_dataset
+```
+
+Add the queue name to `config/sidekiq.yml` file:
+
+```yaml
+:queues:
+- ["default", 1]
+- ["spam_analysis", 1]
+# The other yaml entries
+```
+
+## Contributing
+
+See [Decidim](https://github.com/decidim/decidim).
+
+## License
+
+See [Decidim](https://github.com/decidim/decidim).

data/Rakefile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "decidim/dev/common_rake"

data/app/jobs/decidim/ai/application_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    class ApplicationJob < Decidim::ApplicationJob
+    end
+  end
+end

data/app/jobs/decidim/ai/spam_detection/application_job.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      class ApplicationJob < Decidim::Ai::ApplicationJob
+        queue_as :spam_analysis
+
+        protected
+
+        def classifier
+          @classifier ||= Decidim::Ai::SpamDetection.resource_classifier
+        end
+      end
+    end
+  end
+end

data/app/jobs/decidim/ai/spam_detection/generic_spam_analyzer_job.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      class GenericSpamAnalyzerJob < ApplicationJob
+        include Decidim::TranslatableAttributes
+
+        def perform(reportable, author, locale, fields)
+          @author = author
+          overall_score = I18n.with_locale(locale) do
+            fields.map do |field|
+              classifier.classify(translated_attribute(reportable.send(field)))
+              classifier.score
+            end
+          end
+
+          overall_score = overall_score.inject(0.0, :+) / overall_score.size
+
+          return unless overall_score >= Decidim::Ai::SpamDetection.resource_score_threshold
+
+          Decidim::CreateReport.call(form, reportable)
+        end
+
+        private
+
+        def form
+          @form ||= Decidim::ReportForm.new(reason: "spam", details: classifier.classification_log).with_context(current_user: reporting_user)
+        end
+
+        def reporting_user
+          @reporting_user ||= Decidim::User.find_by!(email: Decidim::Ai::SpamDetection.reporting_user_email)
+        end
+      end
+    end
+  end
+end

data/app/jobs/decidim/ai/spam_detection/user_spam_analyzer_job.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      class UserSpamAnalyzerJob < GenericSpamAnalyzerJob
+        def perform(reportable)
+          @author = reportable
+
+          classifier.classify(reportable.about)
+
+          return unless classifier.score >= Decidim::Ai::SpamDetection.user_score_threshold
+
+          Decidim::CreateUserReport.call(form, reportable)
+        end
+
+        protected
+
+        def classifier
+          @classifier ||= Decidim::Ai::SpamDetection.user_classifier
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/engine.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    class Engine < ::Rails::Engine
+      isolate_namespace Decidim::Ai
+
+      paths["db/migrate"] = nil
+
+      initializer "decidim_ai.resource_classifiers" do |_app|
+        Decidim::Ai::SpamDetection.resource_analyzers.each do |analyzer|
+          Decidim::Ai::SpamDetection.resource_registry.register_analyzer(**analyzer)
+        end
+      end
+
+      initializer "decidim_ai.user_classifiers" do |_app|
+        Decidim::Ai::SpamDetection.user_analyzers.each do |analyzer|
+          Decidim::Ai::SpamDetection.user_registry.register_analyzer(**analyzer)
+        end
+      end
+
+      initializer "decidim_ai.events.subscribe_profile" do
+        config.to_prepare do
+          Decidim::EventsManager.subscribe("decidim.update_account:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::UserSpamAnalyzerJob.perform_later(data[:resource])
+          end
+          Decidim::EventsManager.subscribe("decidim.update_user_group:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::UserSpamAnalyzerJob.perform_later(data[:resource])
+          end
+          Decidim::EventsManager.subscribe("decidim.create_user_group:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::UserSpamAnalyzerJob.perform_later(data[:resource])
+          end
+        end
+      end
+
+      initializer "decidim_ai.events.subscribe_comments" do
+        config.to_prepare do
+          ActiveSupport::Notifications.subscribe("decidim.comments.create_comment:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:body])
+          end
+          ActiveSupport::Notifications.subscribe("decidim.comments.update_comment:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:body])
+          end
+        end
+      end
+
+      initializer "decidim_ai.events.subscribe_meeting" do
+        config.to_prepare do
+          ActiveSupport::Notifications.subscribe("decidim.meetings.create_meeting:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale),
+                                                                             [:description, :title, :location_hints, :registration_terms])
+          end
+          ActiveSupport::Notifications.subscribe("decidim.meetings.update_meeting:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale),
+                                                                             [:description, :title, :location_hints, :registration_terms])
+          end
+        end
+      end
+
+      initializer "decidim_ai.events.subscribe_debate" do
+        config.to_prepare do
+          ActiveSupport::Notifications.subscribe("decidim.debates.create_debate:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:description, :title])
+          end
+          ActiveSupport::Notifications.subscribe("decidim.debates.update_debate:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:description, :title])
+          end
+        end
+      end
+
+      initializer "decidim_ai.events.subscribe_initiatives" do
+        config.to_prepare do
+          ActiveSupport::Notifications.subscribe("decidim.initiatives.create_initiative:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:description, :title])
+          end
+          ActiveSupport::Notifications.subscribe("decidim.initiatives.update_initiative:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:description, :title])
+          end
+        end
+      end
+
+      initializer "decidim_ai.events.subscribe_proposals" do
+        config.to_prepare do
+          ActiveSupport::Notifications.subscribe("decidim.proposals.create_proposal:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:body, :title])
+          end
+          ActiveSupport::Notifications.subscribe("decidim.proposals.update_proposal:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:body, :title])
+          end
+          ActiveSupport::Notifications.subscribe("decidim.proposals.create_collaborative_draft:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:body, :title])
+          end
+          ActiveSupport::Notifications.subscribe("decidim.proposals.update_collaborative_draft:after") do |_event_name, data|
+            Decidim::Ai::SpamDetection::GenericSpamAnalyzerJob.perform_later(data[:resource], data.dig(:extra, :event_author), data.dig(:extra, :locale), [:body, :title])
+          end
+        end
+      end
+
+      def load_seed
+        nil
+      end
+    end
+  end
+end

data/lib/decidim/ai/language/formatter.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module Language
+      class Formatter
+        include ActionView::Helpers::SanitizeHelper
+
+        # for the moment, we just use strip_tags to clean-up the text. At a later stage, we may need to introduce
+        # stemmers, ngrams or other kind of text normalization, as well any language specific criteria
+        def cleanup(text)
+          strip_tags(text)
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/language/language.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module Language
+      autoload :Formatter, "decidim/ai/language/formatter"
+      include ActiveSupport::Configurable
+
+      # Text cleanup service
+      #
+      # If you want to implement your own text formatter, you can use a class having the following contract
+      #
+      # class Formatter
+      #   def cleanup(text)
+      #     # your code
+      #   end
+      # end
+      config_accessor :formatter do
+        "Decidim::Ai::Language::Formatter"
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/importer/database.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Importer
+        class Database
+          def self.call
+            Decidim::Ai::SpamDetection.resource_models.values.each do |model|
+              model.constantize.new.batch_train
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/importer/file.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Importer
+        class File
+          def self.call(file, service)
+            ext = ::File.extname(file)[1..-1]
+            reader_class = Decidim::Admin::Import::Readers.search_by_file_extension(ext)
+
+            reader_class.new(file).read_rows do |row|
+              next unless [:spam, :ham].include?(row[0].to_sym)
+              next if row[1].blank?
+
+              service.train(row[0].to_sym, row[1])
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/resource/base.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Resource
+        class Base
+          include Decidim::TranslatableAttributes
+
+          def fields; end
+
+          def batch_train
+            query.find_each(batch_size: 100) do |resource|
+              classification = resource_hidden?(resource) ? :spam : :ham
+
+              fields.each do |field_name|
+                raise "#{resource.class.name} does not implement #{field_name} as defined in `#{self.class.name}`" unless resource.respond_to?(field_name.to_sym)
+
+                train classification, translated_attribute(resource.send(field_name.to_sym))
+              end
+            end
+          end
+
+          def train(category, text)
+            raise error_message("Decidim::Ai::SpamDetection.resource_detection_service", __method__) unless classifier.respond_to?(:train)
+
+            classifier.train(category, text)
+          end
+
+          def untrain(category, text)
+            raise error_message("Decidim::Ai::SpamDetection.resource_detection_service", __method__) unless classifier.respond_to?(:untrain)
+
+            classifier.untrain(category, text)
+          end
+
+          protected
+
+          def error_message(klass, method_name)
+            "Invalid Classifier class! The class defined under `#{klass}` does not follow the contract regarding ##{method_name} method"
+          end
+
+          def resource_hidden?(resource)
+            resource.class.included_modules.include?(Decidim::Reportable) && resource.hidden?
+          end
+
+          def classifier
+            @classifier ||= Decidim::Ai::SpamDetection.resource_classifier
+          end
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/resource/collaborative_draft.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Resource
+        class CollaborativeDraft < Base
+          def fields = [:body, :title]
+
+          protected
+
+          def query = Decidim::Proposals::CollaborativeDraft.includes(:moderation)
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/resource/comment.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Resource
+        class Comment < Base
+          def fields = [:body]
+
+          protected
+
+          def query = Decidim::Comments::Comment.includes(:moderation)
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/resource/debate.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Resource
+        class Debate < Base
+          def fields = [:description, :title]
+
+          protected
+
+          def query = Decidim::Debates::Debate.includes(:moderation)
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/resource/initiative.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Resource
+        class Initiative < Base
+          def fields = [:description, :title]
+
+          protected
+
+          def query = Decidim::Initiative
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/resource/meeting.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Resource
+        class Meeting < Base
+          def fields = [:description, :title, :location_hints, :registration_terms, :closing_report]
+
+          protected
+
+          def query = Decidim::Meetings::Meeting.includes(:moderation)
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/resource/proposal.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Resource
+        class Proposal < Base
+          def fields = [:body, :title]
+
+          protected
+
+          def query = Decidim::Proposals::Proposal.includes(:moderation)
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/resource/user_base_entity.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Resource
+        class UserBaseEntity < Base
+          def fields = [:about]
+
+          def train(category, text)
+            raise error_message("Decidim::Ai::SpamDetection.user_detection_service", __method__) unless classifier.respond_to?(:train)
+
+            classifier.train(category, text)
+          end
+
+          def untrain(category, text)
+            raise error_message("Decidim::Ai::SpamDetection.user_detection_service", __method__) unless classifier.respond_to?(:untrain)
+
+            classifier.untrain(category, text)
+          end
+
+          protected
+
+          def query = Decidim::UserBaseEntity
+
+          def resource_hidden?(resource) = resource.class.included_modules.include?(Decidim::UserReportable) && resource.blocked?
+
+          def classifier
+            @classifier ||= Decidim::Ai::SpamDetection.user_classifier
+          end
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/service.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      class Service
+        def initialize(registry:)
+          @registry = registry
+        end
+
+        def reset
+          @registry.each do |strategy|
+            next unless strategy.respond_to?(:reset)
+
+            strategy.reset
+          end
+        end
+
+        def train(category, text)
+          text = formatter.cleanup(text)
+          return if text.blank?
+
+          @registry.each do |strategy|
+            strategy.train(category, text)
+          end
+        end
+
+        def classify(text)
+          text = formatter.cleanup(text)
+          return if text.blank?
+
+          @registry.each do |strategy|
+            strategy.classify(text)
+          end
+        end
+
+        def untrain(category, text)
+          text = formatter.cleanup(text)
+          return if text.blank?
+
+          @registry.each do |strategy|
+            strategy.untrain(category, text)
+          end
+        end
+
+        def score
+          @registry.collect(&:score).inject(0.0, :+) / @registry.size
+        end
+
+        def classification_log
+          @classification_log = []
+          @registry.each do |strategy|
+            @classification_log << strategy.log
+          end
+          @classification_log.join("\n")
+        end
+
+        protected
+
+        def formatter
+          @formatter ||= Decidim::Ai::Language.formatter.safe_constantize&.new
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/spam_detection.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      include ActiveSupport::Configurable
+
+      autoload :Service, "decidim/ai/spam_detection/service"
+
+      module Resource
+        autoload :Base, "decidim/ai/spam_detection/resource/base"
+        autoload :Comment, "decidim/ai/spam_detection/resource/comment"
+        autoload :Debate, "decidim/ai/spam_detection/resource/debate"
+        autoload :Initiative, "decidim/ai/spam_detection/resource/initiative"
+        autoload :Proposal, "decidim/ai/spam_detection/resource/proposal"
+        autoload :CollaborativeDraft, "decidim/ai/spam_detection/resource/collaborative_draft"
+        autoload :Meeting, "decidim/ai/spam_detection/resource/meeting"
+        autoload :UserBaseEntity, "decidim/ai/spam_detection/resource/user_base_entity"
+      end
+
+      module Importer
+        autoload :File, "decidim/ai/spam_detection/importer/file"
+        autoload :Database, "decidim/ai/spam_detection/importer/database"
+      end
+
+      module Strategy
+        autoload :Base, "decidim/ai/spam_detection/strategy/base"
+        autoload :Bayes, "decidim/ai/spam_detection/strategy/bayes"
+      end
+
+      # This is the email address used by the spam engine to
+      # properly identify the user that will report users and content
+      config_accessor :reporting_user_email do
+        "decidim-reporting-user@example.org"
+      end
+
+      # You can configure the spam threshold for the spam detection service.
+      # The threshold is a float value between 0 and 1.
+      # The default value is 0.75
+      # Any value below the threshold will be considered spam.
+      config_accessor :resource_score_threshold do
+        0.75
+      end
+
+      # Registered analyzers.
+      # You can register your own analyzer by adding a new entry to this array.
+      # The entry must be a hash with the following keys:
+      # - name: the name of the analyzer
+      # - strategy: the class of the strategy to use
+      # - options: a hash with the options to pass to the strategy
+      # Example:
+      # config.resource_analyzers = {
+      #   name: :bayes,
+      #   strategy: Decidim::Ai::SpamContent::BayesStrategy,
+      #   options: {
+      #     adapter: :redis,
+      #     params: {
+      #       url: ENV["DECIDIM_SPAM_DETECTION_BACKEND_RESOURCE_URL"]
+      #     }
+      #   }
+      # }
+      config_accessor :resource_analyzers do
+        [
+          {
+            name: :bayes,
+            strategy: Decidim::Ai::SpamDetection::Strategy::Bayes,
+            options: {
+              adapter: ENV.fetch("DECIDIM_SPAM_DETECTION_BACKEND_RESOURCE", "redis"),
+              params: { url: ENV.fetch("DECIDIM_SPAM_DETECTION_BACKEND_RESOURCE_URL", "redis://localhost:6379/2") }
+            }
+          }
+        ]
+      end
+
+      # This config_accessor allows the implementers to change the class being used by the classifier,
+      # in order to change the finder method. or even define own resource visibility criteria.
+      # This is the place where new resources can be registered following the pattern
+      # Resource => Handler
+      config_accessor :resource_models do
+        @models ||= begin
+          models = {}
+          models["Decidim::Comments::Comment"] = "Decidim::Ai::SpamDetection::Resource::Comment" if Decidim.module_installed?("comments")
+          models["Decidim::Debates::Debate"] = "Decidim::Ai::SpamDetection::Resource::Debate" if Decidim.module_installed?("debates")
+          models["Decidim::Initiative"] = "Decidim::Ai::SpamDetection::Resource::Initiative" if Decidim.module_installed?("initiatives")
+          models["Decidim::Meetings::Meeting"] = "Decidim::Ai::SpamDetection::Resource::Meeting" if Decidim.module_installed?("meetings")
+          models["Decidim::Proposals::Proposal"] = "Decidim::Ai::SpamDetection::Resource::Proposal" if Decidim.module_installed?("proposals")
+          models["Decidim::Proposals::CollaborativeDraft"] = "Decidim::Ai::SpamDetection::Resource::CollaborativeDraft" if Decidim.module_installed?("proposals")
+          models
+        end
+      end
+
+      # Spam detection service class.
+      # If you want to use a different spam detection service, you can use a class service having the following contract
+      config_accessor :resource_detection_service do
+        "Decidim::Ai::SpamDetection::Service"
+      end
+
+      # You can configure the spam threshold for the spam detection service.
+      # The threshold is a float value between 0 and 1.
+      # The default value is 0.75
+      # Any value below the threshold will be considered spam.
+      config_accessor :user_score_threshold do
+        0.75
+      end
+
+      # Registered analyzers.
+      # You can register your own analyzer by adding a new entry to this array.
+      # The entry must be a hash with the following keys:
+      # - name: the name of the analyzer
+      # - strategy: the class of the strategy to use
+      # - options: a hash with the options to pass to the strategy
+      # Example:
+      # config.user_analyzers = {
+      #   name: :bayes,
+      #   strategy: Decidim::Ai::SpamContent::BayesStrategy,
+      #   options: {
+      #     adapter: :redis,
+      #     params: {
+      #       url: ENV["DECIDIM_SPAM_DETECTION_BACKEND_USER_REDIS_URL"]
+      #     }
+      #   }
+      # }
+      config_accessor :user_analyzers do
+        [
+          {
+            name: :bayes,
+            strategy: Decidim::Ai::SpamDetection::Strategy::Bayes,
+            options: {
+              adapter: ENV.fetch("DECIDIM_SPAM_DETECTION_BACKEND_USER", "redis"),
+              params: { url: ENV.fetch("DECIDIM_SPAM_DETECTION_BACKEND_USER_REDIS_URL", "redis://localhost:6379/3") }
+            }
+          }
+        ]
+      end
+
+      # This config_accessor allows the implementers to change the class being used by the classifier,
+      # in order to change the finder method or what a hidden user really is.
+      # The same applies for UserGroups.
+      config_accessor :user_models do
+        @user_models ||= begin
+          user_models = {}
+
+          user_models["Decidim::UserGroup"] = "Decidim::Ai::SpamDetection::Resource::UserBaseEntity"
+          user_models["Decidim::User"] = "Decidim::Ai::SpamDetection::Resource::UserBaseEntity"
+          user_models
+        end
+      end
+
+      # Spam detection service class.
+      # If you want to use a different spam detection service, you can use a class service having the following contract
+      config_accessor :user_detection_service do
+        "Decidim::Ai::SpamDetection::Service"
+      end
+
+      # this is the generic resource classifier class. If you need to change your own class, please change the
+      # configuration of `Decidim::Ai::SpamDetection.detection_service` variable.
+      def self.resource_classifier
+        @resource_classifier = Decidim::Ai::SpamDetection.resource_detection_service.safe_constantize&.new(
+          registry: Decidim::Ai::SpamDetection.resource_registry
+        )
+      end
+
+      # The registry instance that stores the list of strategies needed to process the resources
+      # In essence is an enumerator class that responds to `register_analyzer(**params)` and `for(name)` methods
+      def self.resource_registry
+        @resource_registry ||= Decidim::Ai::StrategyRegistry.new
+      end
+
+      # this is the generic user classifier class. If you need to change your own class, please change the
+      # configuration of `Decidim::Ai::SpamDetection.detection_service` variable
+      def self.user_classifier
+        @user_classifier = Decidim::Ai::SpamDetection.user_detection_service.safe_constantize&.new(
+          registry: Decidim::Ai::SpamDetection.user_registry
+        )
+      end
+
+      # The registry instance that stores the list of strategies needed to process the user objects
+      # In essence is an enumerator class that responds to `register_analyzer(**params)` and `for(name)` methods
+      def self.user_registry
+        @user_registry ||= Decidim::Ai::StrategyRegistry.new
+      end
+
+      # This method is being called to ensure that user with email configured in
+      # `Decidim::Ai::SpamDetection.reporting_user_email` variable exists in the database.
+      def self.create_reporting_user!
+        Decidim::Organization.find_each do |organization|
+          user = organization.users.find_or_initialize_by(email: Decidim::Ai::SpamDetection.reporting_user_email)
+          next if user.persisted?
+
+          password = SecureRandom.hex(10)
+          user.password = password
+          user.password_confirmation = password
+
+          user.deleted_at = Time.current
+          user.tos_agreement = true
+          user.name = ""
+          user.skip_confirmation!
+          user.save!
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/strategy/base.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Strategy
+        class Base
+          attr_reader :name
+
+          def initialize(options = {})
+            @name = options.delete(:name)
+            @options = options
+          end
+
+          def classify(_content); end
+
+          def train(_classification, _content); end
+
+          def untrain(_classification, _content); end
+
+          def log; end
+
+          def score = 0.0
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/spam_detection/strategy/bayes.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "classifier-reborn"
+
+module Decidim
+  module Ai
+    module SpamDetection
+      module Strategy
+        class Bayes < Base
+          def initialize(options = {})
+            super
+            @options = { adapter: :memory, categories: [:ham, :spam], params: {} }.deep_merge(options)
+
+            @available_categories = options[:categories]
+            @backend = ClassifierReborn::Bayes.new(*available_categories, backend: configured_backend)
+          end
+
+          def log
+            return unless category
+
+            "The Classification engine marked this as #{category}"
+          end
+
+          # Calling this method without any trained categories will throw an error
+          def untrain(category, content)
+            return unless backend.categories.collect(&:downcase).collect(&:to_sym).include?(category)
+
+            backend.untrain(category, content)
+          end
+
+          delegate :train, :reset, to: :backend
+
+          def classify(content)
+            @category, @internal_score = backend.classify_with_score(content)
+            category
+          end
+
+          # The Bayes strategy returns a score between that can be lower than -1
+          # As per ClassifierReborn documentation, closest to 0 is being picked as the dominant category
+          #
+          # From original documentation:
+          #   Returns the scores in each category the provided +text+. E.g.,
+          #     b.classifications "I hate bad words and you"
+          #       =>  {"Uninteresting"=>-12.6997928013932, "Interesting"=>-18.4206807439524}
+          #   The largest of these scores (the one closest to 0) is the one picked out by #classify
+          def score
+            category.presence == "spam" ? 1 : 0
+          end
+
+          private
+
+          attr_reader :backend, :options, :available_categories, :category, :internal_score
+
+          def configured_backend
+            if options[:adapter].to_s == "memory"
+              system_log "[decidim-ai] #{self.class.name} - Running the Memory backend as it was requested. This is not recommended for production environment."
+              ClassifierReborn::BayesMemoryBackend.new
+            elsif options.dig(:params, :url) && options.dig(:params, :url).empty?
+              system_log "[decidim-ai] #{self.class.name} - Running the Memory backend as there are no redis credentials. This is not recommended for production environment."
+              ClassifierReborn::BayesMemoryBackend.new
+            else
+              system_log "[decidim-ai] #{self.class.name} - Running the Redis backend"
+              ClassifierReborn::BayesRedisBackend.new options[:params]
+            end
+          end
+
+          def system_log(message)
+            Rails.logger.info message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/decidim/ai/strategy_registry.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    class StrategyRegistry
+      class StrategyAlreadyRegistered < StandardError; end
+
+      delegate :clear, :collect, :each, :size, to: :strategies
+      attr_reader :strategies
+
+      def initialize
+        @strategies = []
+      end
+
+      def register_analyzer(name:, strategy:, options: {})
+        if self.for(name).present?
+          raise(
+            StrategyAlreadyRegistered,
+            "There is a strategy already registered with the name `:#{name}`"
+          )
+        end
+
+        options = { name: }.merge(options)
+        strategies << strategy.new(options)
+      end
+
+      def for(name)
+        strategies.select { |k, _v| k.name == name }.first
+      end
+    end
+  end
+end

data/lib/decidim/ai/test/factories.rb

@@ -0,0 +1 @@
+# frozen_string_literal: true

data/lib/decidim/ai/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Ai
+    def self.version
+      "0.30.0.rc1"
+    end
+  end
+end

data/lib/decidim/ai.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "decidim/ai/engine"
+
+module Decidim
+  module Ai
+    autoload :StrategyRegistry, "decidim/ai/strategy_registry"
+    autoload :SpamDetection, "decidim/ai/spam_detection/spam_detection"
+    autoload :Language, "decidim/ai/language/language"
+
+    include ActiveSupport::Configurable
+  end
+end

data/lib/tasks/decidim_ai.rake

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+namespace :decidim do
+  namespace :ai do
+    namespace :spam do
+      desc "Create reporting user"
+      task create_reporting_user: :environment do
+        Decidim::Ai::SpamDetection.create_reporting_user!
+      end
+
+      desc "Load application dataset file"
+      task :load_application_dataset, [:file] => :environment do |_, args|
+        Decidim::Ai::SpamDetection::Importer::File.call(args[:file], Decidim::Ai::SpamDetection.user_classifier)
+        Decidim::Ai::SpamDetection::Importer::File.call(args[:file], Decidim::Ai::SpamDetection.resource_classifier)
+      end
+
+      desc "Train model using application database"
+      task train_application_database: :environment do
+        Decidim::Ai::SpamDetection::Importer::Database.call
+      end
+
+      desc "Reset all training model"
+      task reset: :environment do
+        Decidim::Ai::SpamDetection.user_classifier.reset
+        Decidim::Ai::SpamDetection.resource_classifier.reset
+      end
+
+      private
+
+      def plugin_path
+        Gem.loaded_specs["decidim-ai"].full_gem_path
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: decidim-ai
+version: !ruby/object:Gem::Version
+  version: 0.30.0.rc1
+platform: ruby
+authors:
+- Alexandru-Emil Lupu
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-02-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: classifier-reborn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+- !ruby/object:Gem::Dependency
+  name: decidim-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+- !ruby/object:Gem::Dependency
+  name: decidim-debates
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+- !ruby/object:Gem::Dependency
+  name: decidim-initiatives
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+- !ruby/object:Gem::Dependency
+  name: decidim-meetings
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+- !ruby/object:Gem::Dependency
+  name: decidim-proposals
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.30.0.rc1
+description: A module that aims to provide Artificial Intelligence tools for Decidim.
+email:
+- contact@alecslupu.ro
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- app/jobs/decidim/ai/application_job.rb
+- app/jobs/decidim/ai/spam_detection/application_job.rb
+- app/jobs/decidim/ai/spam_detection/generic_spam_analyzer_job.rb
+- app/jobs/decidim/ai/spam_detection/user_spam_analyzer_job.rb
+- lib/decidim/ai.rb
+- lib/decidim/ai/engine.rb
+- lib/decidim/ai/language/formatter.rb
+- lib/decidim/ai/language/language.rb
+- lib/decidim/ai/spam_detection/importer/database.rb
+- lib/decidim/ai/spam_detection/importer/file.rb
+- lib/decidim/ai/spam_detection/resource/base.rb
+- lib/decidim/ai/spam_detection/resource/collaborative_draft.rb
+- lib/decidim/ai/spam_detection/resource/comment.rb
+- lib/decidim/ai/spam_detection/resource/debate.rb
+- lib/decidim/ai/spam_detection/resource/initiative.rb
+- lib/decidim/ai/spam_detection/resource/meeting.rb
+- lib/decidim/ai/spam_detection/resource/proposal.rb
+- lib/decidim/ai/spam_detection/resource/user_base_entity.rb
+- lib/decidim/ai/spam_detection/service.rb
+- lib/decidim/ai/spam_detection/spam_detection.rb
+- lib/decidim/ai/spam_detection/strategy/base.rb
+- lib/decidim/ai/spam_detection/strategy/bayes.rb
+- lib/decidim/ai/strategy_registry.rb
+- lib/decidim/ai/test/factories.rb
+- lib/decidim/ai/version.rb
+- lib/tasks/decidim_ai.rake
+homepage: https://decidim.org
+licenses:
+- AGPL-3.0-or-later
+metadata:
+  bug_tracker_uri: https://github.com/decidim/decidim/issues
+  documentation_uri: https://docs.decidim.org/
+  funding_uri: https://opencollective.com/decidim
+  homepage_uri: https://decidim.org
+  source_code_uri: https://github.com/decidim/decidim
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: 3.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key:
+specification_version: 4
+summary: A Decidim module with AI tools
+test_files: []