decidim-spam_detection 0.1.5

data/README.md

@@ -0,0 +1,75 @@
+# Decidim::SpamDetection
+[![codecov](https://codecov.io/gh/OpenSourcePolitics/decidim-spam_detection/branch/master/graph/badge.svg?token=eJu34XLlVu)](https://codecov.io/gh/OpenSourcePolitics/decidim-spam_detection)
+![Tests](https://github.com/opensourcepolitics/decidim-spam_detection/actions/workflows/tests.yml/badge.svg)
+![Tests](https://github.com/opensourcepolitics/decidim-spam_detection/actions/workflows/lint.yml/badge.svg)
+
+## Usage
+
+SpamDetection is a detection bot made by OpenSourcePolitics. It works with a [spam detection service](https://github.com/OpenSourcePolitics/spam_detection) 
+which marks the user with a spam probability score, between 0.7 
+and 0.99 it is probable, and above 0.99 it is very sure.
+
+By default, the bot does not blocks the user, it only reports them.
+All reports and blocks are made like regular Decidim ones.
+
+### Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "decidim-spam_detection", git: "https://github.com/OpenSourcePolitics/decidim-spam_detection.git"
+```
+
+And then execute:
+
+```bash
+bundle exec rake decidim:spam_detection:mark_users
+```
+
+if you are using sidekiq scheduler you can use the following configuration:
+```
+:queues:
+- user_report
+- block_user
+- scheduled
+
+:schedule:
+    DetectSpamUsers:
+    cron: '0 0 8 * * *' # Run at 08:00
+    class: Decidim::SpamDetection::MarkUsersJob
+    queue: scheduled
+```
+
+### Further configuration
+list of env var, default value and their usage:
+```
+SPAM_DETECTION_API_AUTH_TOKEN
+    default_value: dummy
+    usage: Token auth for authentication used by external service, ask us for more details
+SPAM_DETECTION_API_URL 
+    default_value: "http://localhost:8080/api"
+    usage: URL of the external service
+SPAM_DETECTION_NAME 
+    default_value: "spam detection bot"
+    usage: Name used by the spam detection bot
+SPAM_DETECTION_NICKNAME 
+    default_value: "Spam_detection_bot"
+    usage: Nickname used by the spam detection bot
+SPAM_DETECTION_EMAIL 
+    default_value: "spam_detection_bot@opensourcepolitcs.eu"
+    usage: Email used by the spam detection bot
+PERFORM_BLOCK_USER 
+    default_value: false
+    usage: Determine if the bot can perform blocking, default mode is just report
+```
+
+## API usage
+We can provide the detection service, please check us out at [contact@opensourcepolitics.eu](mailto:contact@opensourcepolitics.eu)
+
+## Contributing
+
+See [Decidim](https://github.com/decidim/decidim).
+
+## License
+
+This engine is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.

data/Rakefile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "decidim/dev/common_rake"
+
+desc "Generates a dummy app for testing"
+task test_app: "decidim:generate_external_test_app"
+
+desc "Generates a development app."
+task development_app: "decidim:generate_external_development_app"
+
+task :push_tag_and_release do
+  system("git tag v#{Decidim::SpamDetection.version}")
+  system("git push --tags")
+  system("gh release create v#{Decidim::SpamDetection.version}")
+end

data/app/commands/decidim/admin/unblock_user.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Admin
+    class UnblockUser < Rectify::Command
+      # Public: Initializes the command.
+      #
+      # blocked_user - the user that is unblocked
+      # current_user - the user performing the action
+      def initialize(blocked_user, current_user)
+        @blocked_user = blocked_user
+        @current_user = current_user
+      end
+
+      # Executes the command. Broadcasts these events:
+      #
+      # - :ok when everything is valid, together with the resource.
+      # - :invalid if the resource is not reported
+      #
+      # Returns nothing.
+      def call
+        return broadcast(:invalid) unless @blocked_user.blocked?
+
+        unblock!
+        add_spam_detection_metadata!
+        broadcast(:ok, @blocked_user)
+      end
+
+      private
+
+      def unblock!
+        Decidim.traceability.perform_action!(
+          "unblock",
+          @blocked_user,
+          @current_user,
+          extra: {
+            reportable_type: @blocked_user.class.name
+          }
+        ) do
+          @blocked_user.blocked = false
+          @blocked_user.blocked_at = nil
+          @blocked_user.block_id = nil
+          @blocked_user.name = @blocked_user.user_name
+          @blocked_user.save!
+        end
+      end
+
+      def add_spam_detection_metadata!
+        return if @blocked_user.extended_data.dig("spam_detection", "blocked_at").blank?
+
+        @blocked_user.update!(extended_data: @blocked_user.extended_data.dup.deep_merge("spam_detection" => { "unblocked_at": Time.current }))
+      end
+    end
+  end
+end

data/app/commands/decidim/admin/unreport_user.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Admin
+    class UnreportUser < Rectify::Command
+      # Public: Initializes the command.
+      #
+      # reportable - A Decidim::User - The user reported
+      # current_user - the user performing the action
+      def initialize(reportable, current_user)
+        @reportable = reportable
+        @current_user = current_user
+      end
+
+      # Executes the command. Broadcasts these events:
+      #
+      # - :ok when everything is valid, together with the resource.
+      # - :invalid if the resource is not reported
+      #
+      # Returns nothing.
+      def call
+        return broadcast(:invalid) unless @reportable.reported?
+
+        unreport!
+        add_spam_detection_metadata!
+        broadcast(:ok, @reportable)
+      end
+
+      private
+
+      def unreport!
+        Decidim.traceability.perform_action!(
+          "unreport",
+          @reportable.user_moderation,
+          @current_user,
+          extra: {
+            reportable_type: @reportable.class.name,
+            username: @reportable.name,
+            user_id: @reportable.id
+          }
+        ) do
+          @reportable.user_moderation.destroy!
+        end
+      end
+
+      def add_spam_detection_metadata!
+        return if @reportable.extended_data.dig("spam_detection", "reported_at").blank?
+
+        @reportable.update!(extended_data: @reportable.extended_data.dup.deep_merge("spam_detection" => { "unreported_at": Time.current }))
+      end
+    end
+  end
+end

data/app/jobs/decidim/spam_detection/mark_users_job.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Decidim
+  module SpamDetection
+    class MarkUsersJob < ApplicationJob
+      queue_as :default
+
+      def perform
+        mark_users_service.call
+      end
+
+      private
+
+      def mark_users_service
+        @mark_users_service ||= Decidim::SpamDetection::MarkUsersService
+      end
+    end
+  end
+end

data/app/services/decidim/spam_detection/mark_users_service.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "uri"
+require "net/http"
+
+module Decidim
+  module SpamDetection
+    class MarkUsersService
+      PUBLICY_SEARCHABLE_COLUMNS = [
+        :id,
+        :decidim_organization_id,
+        :sign_in_count,
+        :personal_url,
+        :about,
+        :avatar,
+        :extended_data,
+        :followers_count,
+        :following_count,
+        :invitations_count,
+        :failed_attempts,
+        :admin
+      ].freeze
+
+      def initialize
+        @users = Decidim::User.left_outer_joins(:user_moderation)
+                              .where(decidim_user_moderations: { decidim_user_id: nil })
+                              .where(admin: false, blocked: false, deleted_at: nil)
+                              .where("(extended_data #> '{spam_detection, unreported_at}') is null")
+                              .where("(extended_data #> '{spam_detection, unblocked_at}') is null")
+        @results = []
+      end
+
+      def self.call
+        new.ask_and_mark
+      end
+
+      def ask_and_mark
+        spam_probability_array = Decidim::SpamDetection::ApiProxy.request(cleaned_users)
+
+        mark_spam_users(merge_response_with_users(spam_probability_array))
+      end
+
+      def mark_spam_users(probability_array)
+        probability_array.each do |probability_hash|
+          @results << Decidim::SpamDetection::SpamUserCommandAdapter.call(probability_hash).result
+        end
+      end
+
+      def cleaned_users
+        @cleaned_users ||= @users.select(PUBLICY_SEARCHABLE_COLUMNS)
+                                 .map { |u| u.serializable_hash(force_except: true) }
+      end
+
+      def merge_response_with_users(response)
+        response.map { |resp| resp.merge("original_user" => @users.find(resp["id"])) }
+      end
+
+      def status
+        @results.tally
+      end
+    end
+  end
+end

data/config/assets.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+base_path = File.expand_path("..", __dir__)
+
+Decidim::Webpacker.register_path("#{base_path}/app/packs")
+Decidim::Webpacker.register_entrypoints(
+  decidim_spam_detection: "#{base_path}/app/packs/entrypoints/decidim_spam_detection.js"
+)
+Decidim::Webpacker.register_stylesheet_import("stylesheets/decidim/spam_detection/spam_detection")

data/config/i18n-tasks.yml

@@ -0,0 +1,10 @@
+---
+
+base_locale: en
+locales: [en]
+
+ignore_unused:
+  - "decidim.components.spam_detection.name"
+
+ignore_missing:
+  - decidim.participatory_processes.scopes.global

data/config/locales/en.yml

@@ -0,0 +1,6 @@
+---
+en:
+  decidim:
+    components:
+      spam_detection:
+        name: SpamDetection

data/lib/decidim/spam_detection/abstract_spam_user_command.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "uri"
+require "net/http"
+
+module Decidim
+  module SpamDetection
+    class AbstractSpamUserCommand
+      SPAM_USER = {
+        name: ENV.fetch("SPAM_DETECTION_NAME", "spam detection bot"),
+        nickname: ENV.fetch("SPAM_DETECTION_NICKNAME", "Spam_detection_bot"),
+        email: ENV.fetch("SPAM_DETECTION_EMAIL", "spam_detection_bot@opensourcepolitcs.eu")
+      }.freeze
+
+      include Decidim::FormFactory
+
+      def initialize(user, probability)
+        @user = user
+        @probability = probability
+        @moderator = moderation_user
+      end
+
+      def call
+        raise NotImplementedError
+      end
+
+      def moderation_user
+        moderation_admin_params = {
+          name: SPAM_USER[:name],
+          nickname: SPAM_USER[:nickname],
+          email: SPAM_USER[:email],
+          admin: true,
+          organization: @user.organization
+        }
+
+        moderation_admin = Decidim::User.find_by(moderation_admin_params)
+
+        return moderation_admin unless moderation_admin.nil?
+
+        create_moderation_admin(moderation_admin_params)
+      end
+
+      def create_moderation_admin(params)
+        password = ::Devise.friendly_token(::Devise.password_length.last)
+        additional_params = {
+          password: password,
+          password_confirmation: password,
+          tos_agreement: true,
+          email_on_notification: false,
+          email_on_moderations: false
+        }
+        moderation_admin = Decidim::User.new(params.merge(additional_params))
+        moderation_admin.skip_confirmation!
+        moderation_admin.save
+        moderation_admin
+      end
+
+      def add_spam_detection_metadata!(metadata)
+        @user.update!(extended_data: @user.extended_data
+                                        .dup
+                                        .deep_merge("spam_detection" => metadata))
+      end
+    end
+  end
+end

data/lib/decidim/spam_detection/admin.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Decidim
+  module SpamDetection
+    # This module contains all the domain logic associated to Decidim's SpamDetection
+    # component admin panel.
+    module Admin
+    end
+  end
+end

data/lib/decidim/spam_detection/admin_engine.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Decidim
+  module SpamDetection
+    # This is the engine that runs on the public interface of `SpamDetection`.
+    class AdminEngine < ::Rails::Engine
+      isolate_namespace Decidim::SpamDetection::Admin
+
+      paths["db/migrate"] = nil
+      paths["lib/tasks"] = nil
+
+      routes do
+        # Add admin engine routes here
+        # resources :spam_detection do
+        #   collection do
+        #     resources :exports, only: [:create]
+        #   end
+        # end
+        # root to: "spam_detection#index"
+      end
+
+      def load_seed
+        nil
+      end
+    end
+  end
+end

data/lib/decidim/spam_detection/api_proxy.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "uri"
+require "net/http"
+
+module Decidim
+  module SpamDetection
+    class ApiProxy
+      URL = URI(ENV.fetch("SPAM_DETECTION_API_URL", "http://localhost:8080/api"))
+      AUTH_TOKEN = ENV.fetch("SPAM_DETECTION_API_AUTH_TOKEN", "dummy")
+
+      def initialize(data_array, batch_size)
+        @data_array = data_array
+        @batch_size = batch_size
+        @retries = [3, 5, 10]
+      end
+
+      def self.request(data_array, batch_size = 1000)
+        new(data_array, batch_size).send_request_in_batch
+      end
+
+      def send_request_in_batch
+        responses = []
+        @data_array.each_slice(@batch_size) do |subdata_array|
+          responses << JSON.parse(send_request_to_api(subdata_array))
+        end
+
+        responses.flatten
+      end
+
+      def send_request_to_api(data)
+        http = Net::HTTP.new(URL.host, URL.port)
+        request = Net::HTTP::Post.new(URL)
+        request["Content-Type"] = "application/json"
+        request["AUTH_TOKEN"] = AUTH_TOKEN
+        request.body = JSON.dump(data)
+        http.use_ssl = true if self.class.use_ssl?(URL)
+        response = http.request(request)
+        response.read_body
+      rescue Net::ReadTimeout
+        raise Net::ReadTimeout if @retries.empty?
+
+        sleep @retries.first
+        @retries.shift
+        retry
+      end
+
+      def self.use_ssl?(url)
+        url.scheme == "https"
+      end
+    end
+  end
+end

data/lib/decidim/spam_detection/block_spam_user_command.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "uri"
+require "net/http"
+
+module Decidim
+  module SpamDetection
+    class BlockSpamUserCommand < Decidim::SpamDetection::AbstractSpamUserCommand
+      prepend Decidim::SpamDetection::Command
+
+      def call
+        form = form(Decidim::Admin::BlockUserForm).from_params(
+          justification: "The user was blocked because of a high spam probability by Decidim spam detection bot"
+        )
+
+        moderator = @moderator
+        user = @user
+
+        form.define_singleton_method(:user) { user }
+        form.define_singleton_method(:current_user) { moderator }
+        form.define_singleton_method(:blocking_user) { moderator }
+
+        Decidim::Admin::BlockUser.call(form)
+
+        add_spam_detection_metadata!({
+                                       "blocked_at" => Time.current,
+                                       "spam_probability" => @probability
+                                     })
+
+        @user.create_user_moderation
+        Rails.logger.info("User with id #{@user["id"]} was blocked for spam")
+
+        :ok
+      end
+    end
+  end
+end

data/lib/decidim/spam_detection/command.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "decidim/spam_detection/command_errors"
+
+module Decidim
+  module SpamDetection
+    module Command
+      attr_reader :result
+
+      module ClassMethods
+        def call(*args, **kwargs)
+          new(*args, **kwargs).call
+        end
+      end
+
+      def self.prepended(base)
+        base.extend ClassMethods
+      end
+
+      def call
+        raise NotImplementedError unless defined?(super)
+
+        @called = true
+        @result = super
+
+        self
+      end
+
+      def success?
+        called? && !failure?
+      end
+
+      def failure?
+        called? && errors.any?
+      end
+
+      def errors
+        return super if defined?(super)
+
+        @errors ||= Decidim::SpamDetection::CommandErrors.new
+      end
+
+      private
+
+      def called?
+        @called ||= false
+      end
+    end
+  end
+end

data/lib/decidim/spam_detection/command_errors.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Decidim
+  module SpamDetection
+    class CommandErrors < Hash
+      def add(key, value, _opts = {})
+        self[key] ||= []
+        self[key] << value
+        self[key].uniq!
+      end
+
+      def add_multiple_errors(errors_hash)
+        errors_hash.each do |key, values|
+          values.each { |value| add key, value }
+        end
+      end
+
+      def each
+        each_key do |field|
+          self[field].each { |message| yield field, message }
+        end
+      end
+
+      def full_messages
+        map { |attribute, message| full_message(attribute, message) }
+      end
+
+      private
+
+      def full_message(attribute, message)
+        return message if attribute == :base
+
+        "#{attribute.to_s.tr(".", "_").capitalize} #{message}"
+      end
+    end
+  end
+end

data/lib/decidim/spam_detection/engine.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "rails"
+require "decidim/core"
+
+module Decidim
+  module SpamDetection
+    # This is the engine that runs on the public interface of spam_detection.
+    class Engine < ::Rails::Engine
+      isolate_namespace Decidim::SpamDetection
+    end
+  end
+end

data/lib/decidim/spam_detection/report_spam_user_command.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "uri"
+require "net/http"
+
+module Decidim
+  module SpamDetection
+    class ReportSpamUserCommand < Decidim::SpamDetection::AbstractSpamUserCommand
+      prepend Decidim::SpamDetection::Command
+
+      def call
+        form = form(Decidim::ReportForm).from_params(
+          reason: "spam",
+          details: "The user was marked as spam by Decidim spam detection bot"
+        )
+
+        current_organization = @user.organization
+        moderator = @moderator
+        user = @user
+
+        report = Decidim::CreateUserReport.new(form, user, moderator)
+        report.define_singleton_method(:current_organization) { current_organization }
+        report.define_singleton_method(:current_user) { moderator }
+        report.define_singleton_method(:reportable) { user }
+        report.call
+
+        add_spam_detection_metadata!({
+                                       "reported_at" => Time.current,
+                                       "spam_probability" => @probability
+                                     })
+
+        Rails.logger.info("User with id #{user.id} was reported for spam")
+
+        :ok
+      end
+    end
+  end
+end

data/lib/decidim/spam_detection/spam_user_command_adapter.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Decidim
+  module SpamDetection
+    class SpamUserCommandAdapter
+      prepend Decidim::SpamDetection::Command
+      SPAM_LEVEL = { very_sure: 0.99, probable: 0.7 }.freeze
+
+      def self.perform_block_user?
+        ENV.fetch("PERFORM_BLOCK_USER", false)
+      end
+
+      def initialize(probability_hash)
+        @probability = probability_hash["spam_probability"]
+        @user = probability_hash["original_user"]
+      end
+
+      def call
+        if @probability > SPAM_LEVEL[:very_sure] && self.class.perform_block_user?
+          Decidim::SpamDetection::BlockSpamUserCommand.call(@user, @probability)
+
+          :blocked_user
+        elsif @probability > SPAM_LEVEL[:probable]
+          Decidim::SpamDetection::ReportSpamUserCommand.call(@user, @probability)
+
+          :reported_user
+        else
+          :nothing
+        end
+      end
+    end
+  end
+end

data/lib/decidim/spam_detection/test/factories.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "decidim/core/test/factories"