bas 1.7.1 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63490ab5165fd3aa1793a7521272869addb0ebd691d65e7e300b757af770b6fe
-  data.tar.gz: 834e5ff6ebe43795ddfd94fa19f7d3435cd6ae99fc24902abdec2a8c3215ce3d
+  metadata.gz: 87acac8ff5c199705abe5b3e41d72834bf104298aa2533740dd40a637f7d7ca2
+  data.tar.gz: d8bc727403b7eaaf2211116fb0e697a0c7b709022b43528ac1d4afffb25c7167
 SHA512:
-  metadata.gz: 97a51249e553a947453084004b2c4701f801bd43a9b14c439721fe1f3e2262813ac541ce0326262997757368eddfc46e08390938f116eef3eb75ecdb80d5e41d
-  data.tar.gz: 2d0d10c08ae2c5a1810e88c624e42d11dbf16ecacc45184185f0ff55caf27a0819b37baeb645991d3eac1ace9595dd1fedba1c4c7939460d824acfaa0fcc1d86
+  metadata.gz: e2fa969db931a22fa52e973ff7438e68f3cace2a25dd625394230c13b2939f0aa0190f3068ce69a16346bc9d7cc53aa52c7fde54f9cf76c97801f37f9c4b20c3
+  data.tar.gz: bd4443f7a5ce9c638936cbb73062148bf77c7fe1721eba3470f602d6cfcbb5b7d0c6dca815e3e79425875a921224cfc0fee5ebf16922fe50426524273d354195

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+# 1.7.2 (04.07.2025)
+- [Improve Elasticsearch mapping creation](https://github.com/kommitters/bas/pull/140)
+- [Fix in Elasticsearch shared storage update_stage method](https://github.com/kommitters/bas/pull/139)
+- [feat: Add Elasticsearch Shared Storage](https://github.com/kommitters/bas/issues/138)
+
 # 1.7.1 (11.02.2025)
 - [Fix absolute_path in order to execute the scripts](https://github.com/kommitters/bas/pull/133)
 

data/Gemfile

@@ -15,8 +15,8 @@ gem "simplecov-lcov", "~> 0.8.0"
 gem "vcr"
 gem "webmock"
 
+gem "elasticsearch", "~> 8.0"
 gem "httparty"
-
 gem "pg", "~> 1.5", ">= 1.5.4"
 
 group :test do
@@ -29,4 +29,5 @@ group :test do
   gem "net-smtp", "~> 0.4.0.1"
   gem "octokit", "~> 8.1.0"
   gem "openssl", "~> 3.2"
+  gem "timecop"
 end

data/lib/bas/orchestrator/manager.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "concurrent-ruby"
+require "date"
 
 module Bas
   module Orchestrator
@@ -10,6 +11,7 @@ module Bas
     # This class initializes a thread pool and processes scheduled scripts based on
     # time intervals, specific days, or exact times.
     #
+    # rubocop:disable Metrics/ClassLength
     class Manager
       def initialize(schedules)
         @last_executions = Hash.new(0.0)
@@ -30,9 +32,7 @@ module Bas
         loop do
           @actual_time = Time.new
 
-          execute_interval(script) if interval?(script)
-          execute_day(script) if day?(script) && time?(script)
-          execute_time(script) if time?(script) && !day?(script)
+          procces_schedule(script)
 
           sleep 0.1
         rescue StandardError => e
@@ -40,6 +40,13 @@ module Bas
         end
       end
 
+      def procces_schedule(script)
+        execute_interval(script) if interval?(script)
+        execute_day(script) if day?(script) && time?(script)
+        execute_time(script) if time?(script) && !day?(script)
+        execute_custom_rule(script, @actual_time) if custom_rule?(script)
+      end
+
       def execute_interval(script)
         return unless time_in_milliseconds - @last_executions[script[:path]] >= script[:interval]
 
@@ -59,6 +66,57 @@ module Bas
         @last_executions[script[:path]] = current_time
       end
 
+      # rubocop:disable Metrics/MethodLength
+      def execute_custom_rule(script, current_moment)
+        rule = script[:custom_rule]
+        case rule[:type]
+        when "last_day_of_week_in_month"
+          execute_last_day_of_week_in_month(script, rule, current_moment)
+        when "last_day_of_month"
+          execute_last_day_of_month(script, rule, current_moment)
+        when "last_day_of_week"
+          execute_last_day_of_week(script, rule, current_moment)
+        when "last_day_of_year"
+          execute_last_day_of_year(script, rule, current_moment)
+        else
+          puts "Unknown custom rule type: #{rule[:type]} for script '#{script[:path]}'"
+        end
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def execute_last_day_of_week_in_month(script, rule, current_moment)
+        return unless rule[:time]&.include?(current_moment.strftime("%H:%M"))
+        return unless today_is_last_day_of_week_in_month?(current_moment, rule[:day_of_week])
+
+        execute_once_per_time(script, current_moment.strftime("%H:%M"))
+      end
+
+      def execute_last_day_of_month(script, rule, current_moment)
+        return unless rule[:time]&.include?(current_moment.strftime("%H:%M"))
+        return unless current_moment.to_date == end_of_month(current_moment.to_date)
+
+        execute_once_per_time(script, current_moment.strftime("%H:%M"))
+      end
+
+      def execute_last_day_of_week(script, rule, current_moment)
+        return unless rule[:time]&.include?(current_moment.strftime("%H:%M"))
+
+        target_wday = Date::DAYNAMES.index { |name| name.casecmp(rule[:day_of_week]).zero? }
+        return if target_wday.nil?
+
+        today = current_moment.to_date
+        return unless today.wday == target_wday && last_occurrence_in_week?(today, target_wday)
+
+        execute_once_per_time(script, current_moment.strftime("%H:%M"))
+      end
+
+      def execute_last_day_of_year(script, rule, current_moment)
+        return unless rule[:time]&.include?(current_moment.strftime("%H:%M"))
+        return unless current_moment.month == 12 && current_moment.day == 31
+
+        execute_once_per_time(script, current_moment.strftime("%H:%M"))
+      end
+
       def interval?(script)
         script[:interval]
       end
@@ -71,6 +129,10 @@ module Bas
         script[:day]
       end
 
+      def custom_rule?(script)
+        script[:custom_rule] && script[:custom_rule][:type]
+      end
+
       def time_in_milliseconds
         @actual_time.to_f * 1000
       end
@@ -83,6 +145,41 @@ module Bas
         @actual_time.strftime("%A")
       end
 
+      def today_is_last_day_of_week_in_month?(time_obj, target_day_name)
+        date = time_obj.to_date
+        target_wday = Date::DAYNAMES.index { |name| name.casecmp(target_day_name).zero? }
+
+        return false if target_wday.nil?
+        return false unless date.wday == target_wday
+
+        last_target_day_of_month?(date)
+      end
+
+      def last_occurrence_in_week?(date, target_wday)
+        current_date = date
+        next_occurrence = current_date + 7
+        next_occurrence.wday == target_wday && last_target_day_of_month?(current_date)
+      end
+
+      def last_target_day_of_month?(date)
+        current_date = date
+        next_occurrence = current_date + 7
+
+        next_occurrence.month != current_date.month
+      end
+
+      def end_of_month(date)
+        next_month = date.month == 12 ? Date.new(date.year + 1, 1, 1) : Date.new(date.year, date.month + 1, 1)
+        next_month - 1
+      end
+
+      def execute_once_per_time(script, execution_time)
+        return if @last_executions[script[:path]] == execution_time
+
+        execute(script)
+        @last_executions[script[:path]] = execution_time
+      end
+
       def execute(script)
         puts "Executing #{script[:path]} at #{current_time}"
         absolute_path = File.expand_path(script[:path], __dir__)
@@ -90,5 +187,6 @@ module Bas
         system("ruby", absolute_path)
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/bas/shared_storage/elasticsearch.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "types/read"
+require_relative "../utils/elasticsearch/request"
+require_relative "../version"
+
+module Bas
+  module SharedStorage
+    ##
+    # The SharedStorage::Elasticsearch class serves as a shared storage implementation to read and write on
+    # a shared storage defined as a elasticsearch database
+    #
+    class Elasticsearch < Bas::SharedStorage::Base
+      def read
+        params = {
+          connection: read_options[:connection], index: read_options[:index],
+          query: read_body, method: :search
+        }
+
+        result = Utils::Elasticsearch::Request.execute(params)
+        @read_response = build_read_response(result)
+      end
+
+      def write(data)
+        params = {
+          connection: write_options[:connection], index: write_options[:index],
+          body: write_body(data), method: :index
+        }
+
+        create_mapping
+        @write_response = Utils::Elasticsearch::Request.execute(params).body
+      end
+
+      def set_in_process
+        return if read_options[:avoid_process].eql?(true) || read_response.id.nil?
+
+        update_stage(read_response.id, "in process")
+      end
+
+      def set_processed
+        return if read_options[:avoid_process].eql?(true) || read_response.id.nil?
+
+        update_stage(read_response.id, "processed")
+      end
+
+      private
+
+      # rubocop:disable Metrics/MethodLength
+      def create_mapping
+        short_text_properties = {
+          type: "text",
+          fields: {
+            keyword: { type: "keyword", ignore_above: 256 }
+          }
+        }
+
+        params = {
+          connection: write_options[:connection],
+          index: write_options[:index],
+          body: {
+            mappings: {
+              properties: {
+                data: { type: "object" },
+                tag: short_text_properties,
+                archived: { type: "boolean" },
+                inserted_at: { type: "date", format: "yyyy-MM-dd HH:mm:ss Z" },
+                stage: short_text_properties,
+                status: short_text_properties,
+                error_message: { type: "object" },
+                version: short_text_properties
+              }
+            }
+          },
+          method: :create_mapping
+        }
+
+        Utils::Elasticsearch::Request.execute(params)
+      end
+
+      def read_body
+        return read_options[:query] if read_options[:query].is_a?(Hash)
+
+        {
+          query: {
+            bool: {
+              must: [
+                { match: { status: "success" } }, { match: { tag: read_options[:tag] } },
+                { match: { archived: false } }, { match: { stage: "unprocessed" } }
+              ]
+            }
+          },
+          sort: [{ inserted_at: { order: "asc" } }]
+        }
+      end
+
+      def write_body(data)
+        if data[:success]
+          return {
+            data: data[:success], tag: write_options[:tag],
+            archived: false, inserted_at: Time.now.strftime("%Y-%m-%d %H:%M:%S %z"),
+            stage: "unprocessed", status: "success", error_message: nil, version: Bas::VERSION
+          }
+        end
+
+        {
+          data: nil, tag: write_options[:tag], archived: false,
+          inserted_at: Time.now.strftime("%Y-%m-%d %H:%M:%S %z"), stage: "unprocessed",
+          status: "failed", error_message: data[:error], version: Bas::VERSION
+        }
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def build_read_response(result)
+        first_hit = result["hits"]["hits"].empty? ? {} : result["hits"]["hits"].first
+        Bas::SharedStorage::Types::Read.new(
+          first_hit["_id"], first_hit["_source"]["data"].to_json, first_hit.dig("_source", "inserted_at")
+        )
+      end
+
+      def update_stage(id, stage)
+        params = {
+          connection: read_options[:connection], index: read_options[:index], method: :update,
+          body: {
+            query: { ids: { values: [id] } },
+            script: { source: "ctx._source.stage = params.new_value", params: { new_value: stage } }
+          }
+        }
+
+        response = Utils::Elasticsearch::Request.execute(params)
+        return unless response["updated"].zero?
+
+        raise StandardError, "Document #{id} not found, so it was not updated"
+      end
+    end
+  end
+end

data/lib/bas/utils/elasticsearch/request.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "elasticsearch"
+
+module Utils
+  module Elasticsearch
+    ##
+    # This module is a ElasticsearchDB utility to make requests to a Elasticsearch database.
+    #
+    module Request
+      # Implements the request process logic to the ElasticsearchDB index.
+      #
+      # <br>
+      # <b>Params:</b>
+      # * <tt>connection</tt> Connection parameters to the database: `host`, `port`, `index`, `user`, `password`.
+      # * <b>query</b>:
+      #   * <tt>String</tt>: String with the Elasticsearch query to be executed.
+      #   * <tt>Hash</tt>: Hash with the Elasticsearch query to be executed.
+      # * <tt>method</tt>: Method to be executed.
+      #                    Allowed methods are: `:search`, `:index`, `:update`, `:create_mapping`.
+      #
+      # <br>
+      # <b>returns</b> <tt>Elasticsearch::Response</tt>
+      #
+      class << self
+        def execute(params)
+          client = ::Elasticsearch::Client.new(
+            host: params[:connection][:host],
+            port: params[:connection][:port],
+            user: params[:connection][:user],
+            password: params[:connection][:password],
+            api_versioning: false,
+            transport_options: build_ssl_options(params[:connection])
+          )
+
+          perform_request(params, client)
+        end
+
+        private
+
+        def build_ssl_options(connection_params)
+          ssl_options = {}
+          ssl_options[:ca_file] = connection_params[:ca_file] if connection_params[:ca_file]
+          ssl_options[:verify] = connection_params[:ssl_verify] if connection_params.key?(:ssl_verify)
+          { ssl: ssl_options }
+        end
+
+        def perform_request(params, client)
+          case params[:method]
+          when :index
+            index_document(params, client)
+          when :search
+            search(params, client)
+          when :update
+            update_documents(params, client)
+          when :create_mapping
+            create_mapping(params, client)
+          end
+        end
+
+        def search(params, client)
+          search_params = { index: params[:index] }
+          search_params[:size] = 1 # return only one document
+          if params[:query].is_a?(Hash)
+            search_params[:body] = params[:query]
+          else
+            search_params[:q] = params[:query]
+          end
+
+          client.search(**search_params)
+        end
+
+        def index_document(params, client)
+          client.index(index: params[:index], body: params[:body])
+        end
+
+        def update_documents(params, client)
+          client.update_by_query(index: params[:index], body: params[:body], wait_for_completion: true, refresh: true)
+        end
+
+        def create_mapping(params, client)
+          return if client.indices.exists?(index: params[:index])
+
+          client.indices.create(index: params[:index], body: params[:body])
+        end
+      end
+    end
+  end
+end

data/lib/bas/version.rb

@@ -2,5 +2,5 @@
 
 module Bas
   # Gem version
-  VERSION = "1.7.1"
+  VERSION = "1.7.2"
 end

metadata

@@ -1,14 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: bas
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.7.2
 platform: ruby
 authors:
 - kommitters Open Source
 bindir: exe
 cert_chain: []
-date: 2025-02-11 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '8.0'
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
@@ -65,11 +79,13 @@ files:
 - lib/bas/orchestrator/manager.rb
 - lib/bas/shared_storage/base.rb
 - lib/bas/shared_storage/default.rb
+- lib/bas/shared_storage/elasticsearch.rb
 - lib/bas/shared_storage/postgres.rb
 - lib/bas/shared_storage/types/read.rb
 - lib/bas/utils/digital_ocean/request.rb
 - lib/bas/utils/discord/integration.rb
 - lib/bas/utils/discord/request.rb
+- lib/bas/utils/elasticsearch/request.rb
 - lib/bas/utils/exceptions/function_not_implemented.rb
 - lib/bas/utils/exceptions/invalid_process_response.rb
 - lib/bas/utils/github/octokit_client.rb
@@ -106,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: BAS - Business automation suite
 test_files: []