betterplace-bi 0.7.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 119422d8d08ceabf4379d677d556c4dc2b6bcb32f0ae4f7b6304164bc7fabd59
+  data.tar.gz: 2cccf0803769aec1947716a60ea2d1dec45074785347b6a28058a92c99f7d3e9
+SHA512:
+  metadata.gz: 34c3d5aa9e9f363011bdf77fa53e5b4bfaaac9728a29f916d14e0b1b7769e0364c03e8c46ee1572e50316cfe09fe5e9ff18e9375fe7fb9de3078a3f20fee2373
+  data.tar.gz: 3b96febf3b0a6aa22ef134980e4a574603ea8d2020661af16abac9016a6354c84dc9d65a216664cfcc576e8fb790caf68feeb8e58a79e0af4cd5d31cde68430a

data/.gitignore

@@ -0,0 +1,10 @@
+.*.sw[pon]
+.AppleDouble
+.DS_Store
+.byebug_history
+.rvmrc
+Gemfile.lock
+coverage
+errors.lst
+pkg
+tags

data/.semaphore/semaphore.yml

@@ -0,0 +1,26 @@
+version: v1.0
+name: Betterplace-BI pipeline
+agent:
+  machine:
+    type: e1-standard-2
+    os_image: ubuntu2004
+
+blocks:
+  - name: "Unit Tests"
+    task:
+      env_vars:
+        # Matches the configuration used in sem-service
+        - name: RAILS_ENV
+          value: test
+      jobs:
+      - name: RSpec
+        commands:
+          - checkout
+
+          # Setup ruby
+          - sem-version ruby $(awk '/^ruby/ { print $2 }' .tool-versions)
+
+          # Setup gems
+          - bundle config set path 'vendor/bundle'
+          - bundle install
+          - bundle exec rake spec

data/.tool-versions

@@ -0,0 +1,2 @@
+ruby 3.2.0
+bundler 2.2.32

data/.utilsrc

@@ -0,0 +1,26 @@
+# vim: set ft=ruby:
+
+search do
+  prune_dirs /\A(\.svn|\.git|CVS|tmp|tags|coverage|pkg)\z/
+  skip_files /(\A\.|\.sw[pon]\z|\.(log|fnm|jpg|jpeg|png|pdf|svg)\z|tags|~\z)/i
+end
+
+discover do
+  prune_dirs /\A(\.svn|\.git|CVS|tmp|tags|coverage|pkg)\z/
+  skip_files /(\A\.|\.sw[pon]\z|\.log\z|~\z)/
+  binary false
+end
+
+strip_spaces do
+  prune_dirs /\A(\..*|CVS|pkg)\z/
+  skip_files /(\A\.|\.sw[pon]\z|\.log\z|~\z)/
+end
+
+probe do
+  test_framework :rspec
+  #include_dirs 'features'
+end
+
+ssh_tunnel do
+  terminal_multiplexer :tmux
+end

data/Gemfile

@@ -0,0 +1,5 @@
+# vim: set filetype=ruby et sw=2 ts=2:
+
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,76 @@
+# Betterplace-BI
+
+## Description
+
+This library transfers BI data from an application to a BIME-server.
+
+## Configuration
+
+Via a `config/bi.yml` file in YAML like so:
+
+```
+---
+production:
+  update: yes
+  auth:
+    username: bi
+    password: secret
+  clear: http://localhost:1234/api/v1/all/:id
+  endpoints:
+    BI::TestModelValue:
+      POST: http://localhost:1234/api/v1/tests
+      DELETE: http://localhost:1234/api/v1/test/:id
+  bime_dir:
+    <%= Pathname.pwd.join('../bime') %>
+```
+
+## Usage
+
+For every Rails model `TestModel` define a class named
+`BI::TestModelValue` like this:
+
+```
+class BI::TestModelValue
+  include BI::SharedValue
+
+  def self.update_class
+    ::TestModel
+  end
+
+  field type: 'string', db: 'gorm:"type:uuid;primary_key"'
+  def id
+    @object.id
+  end
+
+  field type: '*int64', db: 'gorm:"type:bigint"'
+  def number
+    10 ** 10
+  end
+
+  field type: '*time.Time', db: bp_timestamp # ISO timestamp string, e. g. "2018-11-05T15:40:14.362219+01:00"
+  def created_at
+    iso_timestamp(@object.created_at)
+  end
+end
+```
+
+Compute the values to be transfered by accessing @object and define GO types
+and GORM struct tags for these values.
+
+Also include `BI::Updater` in `TestModel` like so:
+
+```
+class TestModel < ApplicationRecord
+  include BI::Updater
+
+  has_one :dependent
+
+  def trigger_dependent_updates
+    dependent.schedule_bi_update_job(true)
+  end
+end
+```
+
+## License
+
+This software is licensed under the Apache 2.0 license.

data/Rakefile

@@ -0,0 +1,37 @@
+# vim: set filetype=ruby et sw=2 ts=2:
+
+require 'gem_hadar'
+
+GemHadar do
+  name        'betterplace-bi'
+  path_name   'bi'
+  path_module 'BI'
+  author      'Florian Frank'
+  email       'flori@ping.de'
+  homepage    "https://github.com/betterplace/#{name}"
+  summary     'Business Intelligence library'
+  description 'This library contains functionality to send/store BI and AB-testing data'
+  test_dir    'spec'
+  ignore      '.*.sw[pon]', 'pkg', 'Gemfile.lock', 'coverage', '.rvmrc',
+    '.AppleDouble', '.DS_Store', '.byebug_history', 'errors.lst', 'tags'
+
+  readme      'README.md'
+  title       "#{name.camelize} -- BI library"
+  licenses    << 'Apache-2.0'
+
+  dependency             'json'
+  dependency             'tins'
+  dependency             'complex_config'
+  dependency             'globalid'
+  dependency             'excon'
+  dependency             'rails'
+  dependency             'sinatra'
+  dependency             'betterlog'
+  dependency             'mize'
+  development_dependency 'rake'
+  development_dependency 'simplecov'
+  development_dependency 'rspec'
+  development_dependency 'byebug'
+end
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.7.0

data/betterplace-bi.gemspec

@@ -0,0 +1,39 @@
+# -*- encoding: utf-8 -*-
+# stub: betterplace-bi 0.7.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "betterplace-bi".freeze
+  s.version = "0.7.0".freeze
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib".freeze]
+  s.authors = ["Florian Frank".freeze]
+  s.date = "2024-04-08"
+  s.description = "This library contains functionality to send/store BI and AB-testing data".freeze
+  s.email = "flori@ping.de".freeze
+  s.extra_rdoc_files = ["README.md".freeze, "lib/betterplace-bi.rb".freeze, "lib/bi.rb".freeze, "lib/bi/ab_test_helper.rb".freeze, "lib/bi/api.rb".freeze, "lib/bi/commands.rb".freeze, "lib/bi/commands/base.rb".freeze, "lib/bi/commands/collector.rb".freeze, "lib/bi/commands/connection.rb".freeze, "lib/bi/commands/delete.rb".freeze, "lib/bi/commands/serializer.rb".freeze, "lib/bi/commands/update.rb".freeze, "lib/bi/commands_job.rb".freeze, "lib/bi/event.rb".freeze, "lib/bi/planning_value_parser.rb".freeze, "lib/bi/planning_value_validations.rb".freeze, "lib/bi/railtie.rb".freeze, "lib/bi/request_analyzer.rb".freeze, "lib/bi/session_id.rb".freeze, "lib/bi/shared_value.rb".freeze, "lib/bi/tracking.rb".freeze, "lib/bi/type_generator.rb".freeze, "lib/bi/update_error.rb".freeze, "lib/bi/updater.rb".freeze, "lib/bi/version.rb".freeze]
+  s.files = [".gitignore".freeze, ".semaphore/semaphore.yml".freeze, ".tool-versions".freeze, ".utilsrc".freeze, "Gemfile".freeze, "README.md".freeze, "Rakefile".freeze, "VERSION".freeze, "betterplace-bi.gemspec".freeze, "lib/betterplace-bi.rb".freeze, "lib/bi.rb".freeze, "lib/bi/ab_test_helper.rb".freeze, "lib/bi/api.rb".freeze, "lib/bi/commands.rb".freeze, "lib/bi/commands/base.rb".freeze, "lib/bi/commands/collector.rb".freeze, "lib/bi/commands/connection.rb".freeze, "lib/bi/commands/delete.rb".freeze, "lib/bi/commands/serializer.rb".freeze, "lib/bi/commands/update.rb".freeze, "lib/bi/commands_job.rb".freeze, "lib/bi/event.rb".freeze, "lib/bi/planning_value_parser.rb".freeze, "lib/bi/planning_value_validations.rb".freeze, "lib/bi/railtie.rb".freeze, "lib/bi/request_analyzer.rb".freeze, "lib/bi/session_id.rb".freeze, "lib/bi/shared_value.rb".freeze, "lib/bi/tracking.rb".freeze, "lib/bi/type_generator.rb".freeze, "lib/bi/update_error.rb".freeze, "lib/bi/updater.rb".freeze, "lib/bi/version.rb".freeze, "lib/tasks/bime.rake".freeze, "spec/bi/ab_test_helper_spec.rb".freeze, "spec/bi/commands/collector_spec.rb".freeze, "spec/bi/commands/connection_spec.rb".freeze, "spec/bi/commands_job_spec.rb".freeze, "spec/bi/commands_spec.rb".freeze, "spec/bi/event_spec.rb".freeze, "spec/bi/planning_value_parser_spec.rb".freeze, "spec/bi/request_analyzer_spec.rb".freeze, "spec/bi/shared_value_spec.rb".freeze, "spec/bi/tracking_spec.rb".freeze, "spec/bi/type_generator_spec.rb".freeze, "spec/bi/updater_spec.rb".freeze, "spec/bime_dir/.keep".freeze, "spec/config/bi.yml".freeze, "spec/spec_helper.rb".freeze, "spec/support/models.rb".freeze]
+  s.homepage = "https://github.com/betterplace/betterplace-bi".freeze
+  s.licenses = ["Apache-2.0".freeze]
+  s.rdoc_options = ["--title".freeze, "Betterplace-bi -- BI library".freeze, "--main".freeze, "README.md".freeze]
+  s.rubygems_version = "3.4.21".freeze
+  s.summary = "Business Intelligence library".freeze
+  s.test_files = ["spec/bi/ab_test_helper_spec.rb".freeze, "spec/bi/commands/collector_spec.rb".freeze, "spec/bi/commands/connection_spec.rb".freeze, "spec/bi/commands_job_spec.rb".freeze, "spec/bi/commands_spec.rb".freeze, "spec/bi/event_spec.rb".freeze, "spec/bi/planning_value_parser_spec.rb".freeze, "spec/bi/request_analyzer_spec.rb".freeze, "spec/bi/shared_value_spec.rb".freeze, "spec/bi/tracking_spec.rb".freeze, "spec/bi/type_generator_spec.rb".freeze, "spec/bi/updater_spec.rb".freeze, "spec/spec_helper.rb".freeze, "spec/support/models.rb".freeze]
+
+  s.specification_version = 4
+
+  s.add_development_dependency(%q<gem_hadar>.freeze, ["~> 1.11.0".freeze])
+  s.add_development_dependency(%q<rake>.freeze, [">= 0".freeze])
+  s.add_development_dependency(%q<simplecov>.freeze, [">= 0".freeze])
+  s.add_development_dependency(%q<rspec>.freeze, [">= 0".freeze])
+  s.add_development_dependency(%q<byebug>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<json>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<tins>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<complex_config>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<globalid>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<excon>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<rails>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<sinatra>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<betterlog>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<mize>.freeze, [">= 0".freeze])
+end

data/lib/betterplace-bi.rb

@@ -0,0 +1 @@
+require 'bi'

data/lib/bi/ab_test_helper.rb

@@ -0,0 +1,81 @@
+module BI
+  module ABTestHelper
+    OVERRIDE_PARAM_NAME = "ab_test"
+
+    # Parameter +name+ is the name of the AB-test, the first element of
+    # +alternatives+ is the control. +request+ is a ActionDispatch::Request
+    # object.
+    def ab_test(name, *alternatives, request: self.request, &block)
+      alternatives.flatten!
+      alternatives.empty? and return
+      session_id, mode = nil, nil
+      chosen =
+        case
+        when override = ab_test_override?(name, *alternatives, request: request)
+          mode = :override
+          override
+        when ab_test_ignore_user?(request: request)
+          mode = :ignore
+          alternatives.first
+        when session = request.ask_and_send(:session)
+          session.loaded? or session.send(:load!)
+          session_id = BI::SessionID.session_id(session)
+          n = if session_id.empty?
+                mode = :random
+                rand(1 << 128)
+              else
+                mode = :session
+                Digest::MD5.hexdigest([ name, session_id ] * ?:).to_i(16)
+              end
+          alternatives[n % alternatives.size]
+        else
+          mode = :default
+          n = rand(1 << 128)
+          alternatives[n % alternatives.size]
+        end
+      Log.info(
+        "session.id=#{session_id} => #{chosen} was chosen for #{name} (#{mode})",
+        meta: { module: 'bi' }
+      )
+      block&.(chosen) || chosen
+    end
+
+    def ab_test_ignore_user?(request: self.request)
+      request and BI::RequestAnalyzer.new(request).ignore?
+    end
+
+    # It's possible to pick the control with the URL parameter
+    # 'ab_test=control' or ab_test=0, the first alternative by
+    # ab_test=alternative or ab_test=1, further alternatives by 2, 3,… It's
+    # also possible to pick one by using the version's name like ab_test=foo.
+    # If there is more than one AB-Test running, this parameter can be
+    # ab_test[name]=… to only switch the AB-Test
+    # named +name+ to the control or alternatives. This way it's possible to
+    # view all the possible permutations of controls and alternatives if you're
+    # crazy enough.
+    def ab_test_override?(name, *alternatives, request: self.request)
+      alternatives.flatten!
+      overriden = request.params[OVERRIDE_PARAM_NAME] rescue nil
+      case
+      when overriden.respond_to?(:to_str)
+        ab_test_override_alternative(overriden.to_str, *alternatives)
+      when overriden.respond_to?(:to_hash)
+        ab_test_override_alternative(overriden.to_hash[name], *alternatives)
+      end
+    end
+
+    def ab_test_override_alternative(override, *alternatives)
+      alternatives.flatten!
+      case override
+      when 'control'
+        alternatives.first
+      when 'alternative'
+        alternatives.second
+      when /\A\d+\z/
+        alternatives[$&.to_i] || alternatives.first
+      when *alternatives
+        override
+      end
+    end
+  end
+end

data/lib/bi/api.rb

@@ -0,0 +1,115 @@
+require 'sinatra/base'
+
+module BI
+  module API
+
+    module Helper
+      include BI::ABTestHelper
+
+      def self.included(modul)
+        modul.extend self
+      end
+
+      def hostname
+        @hostname ||= Socket.gethostname rescue 'n/a'
+      end
+
+      def status_code(symbol)
+        Rack::Utils::SYMBOL_TO_STATUS_CODE[symbol]
+      end
+
+      def rails_request
+        ActionDispatch::Request.new(env)
+      end
+
+      def rails_session
+        rails_request.session
+      end
+
+      def render_result(result, code: :ok, headers: {})
+        headers = headers.merge(
+          'Content-Type' => 'application/json',
+        )
+        code = status_code(code)
+        [ code, headers, result  ]
+      end
+
+      def render_not_found
+        result = JSON(
+          hostname:  hostname,
+          message:   'not found',
+        )
+        render_result(result, code: :not_found)
+      end
+
+      def render_exception(exception, code: :internal_server_error)
+        result = JSON(
+          hostname: hostname,
+          error:     exception.class.name,
+          message:   exception.message,
+        )
+        render_result(result, code: code)
+      end
+
+      def ab_test_result(channel, request)
+        config   = cc.ab_tests[channel.to_s] or return
+        analyzer = BI::RequestAnalyzer.new(request)
+        current  = ab_test(config.name, config.alternatives, request: rails_request)
+        version  = "#{config.name}/#{current}"
+        {
+          channel:     channel,
+          version:     version,
+          session:     BI::SessionID.session_id(request.session),
+          device_type: analyzer.device_type,
+          user_agent:  analyzer.user_agent,
+        }
+      end
+    end
+
+    class App < Sinatra::Base
+      include BI::API::Helper
+
+      before do
+        rr = rails_request
+        Log.info "Received #{rr.method} to #{rr.path}",
+          type: 'bi_request',
+          meta: {
+            module:  'bi',
+            url:     rr.url,
+            method:  rr.method,
+            params:  rr.params,
+            referer: rr.referer,
+          }
+      end
+
+      get '/' do
+        render_result JSON(
+          endpoints: {
+            '/bi'                  => { methods: [ 'GET' ] },
+            '/bi/ab_test/:channel' => { methods: [ 'GET', 'POST' ] },
+          }
+        )
+      end
+
+      get '/ab_test/:channel' do
+        channel     = params.fetch('channel')
+        result      = ab_test_result(channel, rails_request) or return
+        render_result JSON(result)
+      end
+
+      post '/ab_test/:channel' do
+        channel    = params.fetch('channel')
+        result     = ab_test_result(channel, rails_request) or render_not_found
+        event_data = JSON(
+          request.body.tap(&:rewind).read
+        ).symbolize_keys_recursive
+        event_data.update result
+        BI::Event.new.write(**event_data)
+        render_result JSON(result)
+      rescue => e
+        Log.error e, meta: { module: 'bi' }
+        render_exception(e, code: :unprocessable_entity)
+      end
+    end
+  end
+end

data/lib/bi/commands/base.rb

@@ -0,0 +1,11 @@
+module BI
+  module Commands
+    class Base
+      attr_reader :data
+
+      def initialize(data)
+        @data = data
+      end
+    end
+  end
+end

data/lib/bi/commands/collector.rb

@@ -0,0 +1,32 @@
+module BI
+  module Commands
+    class Collector
+      def initialize
+        @first_object = nil
+        @commands = []
+      end
+
+      def add(object, value_class)
+        @first_object ||= object
+        if object&.destroyed?
+          if url = value_class.url(method: 'DELETE', id: object.id)
+            @commands << BI::Commands::Delete.new(url: url)
+          end
+        else
+          value = value_class.new(object)
+          @commands << BI::Commands::Update.new(value: value)
+        end
+        self
+      end
+
+      def schedule_job?(object)
+        @first_object == object
+      end
+
+      def schedule_job
+        BI::CommandsJob.new(*@commands).enqueue
+        self
+      end
+    end
+  end
+end

data/lib/bi/commands/connection.rb

@@ -0,0 +1,25 @@
+module BI
+  module Commands
+    module Connection
+      module_function
+
+      def create_connection
+        Log.info("Connecting BI Server at %s now" % cc.bi.server)
+        Excon.new(
+          cc.bi.server,
+          user:     cc.bi.auth.username,
+          password: cc.bi.auth.password,
+          persistent: true,
+        )
+      end
+
+      def connect_to_server
+        connection = create_connection
+        yield connection
+        self
+      ensure
+        connection&.reset
+      end
+    end
+  end
+end

data/lib/bi/commands/delete.rb

@@ -0,0 +1,33 @@
+require 'bi/commands/base'
+
+module BI
+  module Commands
+    class Delete < BI::Commands::Base
+
+      def url
+        data[:url]
+      end
+
+      def perform_via(connection)
+        Log.info("Sending BI Delete to #{url}")
+        response = connection.delete(
+          path: url,
+          headers:  {
+            'Content-Type' => 'application/json',
+            'Accept'       => 'application/json',
+          }
+        )
+        case response.status
+        when 0...300
+          Log.info "Successfully sent delete to #{url}."
+        when 404
+          Log.warn "Duplicate deletion sent to #{url}: status=#{response.status}\n#{response.body}"
+          # Already deleted
+        else
+          Log.warn "Failed to sent delete to #{url}: status=#{response.status}\n#{response.body}"
+          raise BI::UpdateError, "status=#{response.status}: #{response.body}"
+        end
+      end
+    end
+  end
+end

data/lib/bi/commands/serializer.rb

@@ -0,0 +1,40 @@
+module BI
+  module Commands
+    class Serializer < ActiveJob::Serializers::ObjectSerializer
+      # Checks if an argument should be serialized by this serializer.
+      def serialize?(argument)
+        argument.is_a? ::BI::Commands::Base
+      end
+
+      GLOBALID_KEY = "_bi_globalid"
+
+      # Converts an object to a simpler representative using supported object
+      # types. The recommended representative is a Hash with a specific key. Keys
+      # can be of basic types only. You should call `super` to add the custom
+      # serializer type to the hash.
+      def serialize(command)
+        hash = { class: command.class.name }
+        hash |= command.data.to_h.transform_values { |v|
+          if v.is_a?(GlobalID::Identification)
+            v = { GLOBALID_KEY => v.to_global_id.to_s }
+          end
+          v
+        }
+        hash.stringify_keys!
+        super(hash)
+      end
+
+      # Converts serialized value into a proper object.
+      def deserialize(hash)
+        hash.delete("class").constantize.new(
+          hash.transform_values { |v|
+            if h = v.ask_and_send(:to_hash) and h.include?(GLOBALID_KEY)
+              v = GlobalID::Locator.locate h[GLOBALID_KEY]
+            end
+            v
+          }.symbolize_keys
+        )
+      end
+    end
+  end
+end

data/lib/bi/commands/update.rb

@@ -0,0 +1,29 @@
+require 'bi/commands/base'
+
+module BI
+  module Commands
+    class Update < BI::Commands::Base
+      def value
+        data[:value]
+      end
+
+      def perform_via(connection)
+        Log.info("Sending BI Update value for %s" % value)
+        response = connection.post(
+          path: value.url,
+          headers: {
+            'Content-Type' => 'application/json',
+            'Accept'       => 'application/json',
+          },
+          body: value.to_json,
+        )
+        if response.status < 300
+          Log.info "Successfully sent update for #{value}."
+        else
+          Log.warn "Failed to sent update for #{value}: status=#{response.status}\n#{response.body}"
+          raise BI::UpdateError, "status=#{response.status}: #{response.body}"
+        end
+      end
+    end
+  end
+end

data/lib/bi/commands.rb

@@ -0,0 +1,10 @@
+module BI
+  module Commands
+  end
+end
+
+require 'bi/commands/connection'
+require 'bi/commands/update'
+require 'bi/commands/delete'
+require 'bi/commands/collector'
+require 'bi/commands/serializer' if defined?(ActiveJob::Serializers)

data/lib/bi/commands_job.rb

@@ -0,0 +1,24 @@
+require 'active_job/arguments'
+
+module BI
+  class CommandsJob < ::ActiveJob::Base
+    include BI::Commands::Connection
+
+    queue_as :bime
+
+    retry_on Excon::Error::Socket, # handle temporary dns resolution problems
+             Excon::Error::Timeout # handle timeouts
+
+    retry_on(ActiveJob::DeserializationError, wait: :polynomially_longer, attempts: 4) {}
+
+    def perform(*commands)
+      commands.empty? and return
+      connect_to_server do |connection|
+        commands.each do |command|
+          command.perform_via(connection)
+        end
+      end
+      self
+    end
+  end
+end