dialog_ruby 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00b5be156c8481f0ecf9a93d185b4c3d070f53d05dae47addf2d5dd002d838b9
-  data.tar.gz: c6fa9db24028794a0b705de5a7334adebbda338180fa902af2d6fc45d5b2c376
+  metadata.gz: edf13cf660e814e5a448bea845d6078bee8a9ea15c8b5e89b432f5f56b616f07
+  data.tar.gz: 27414e8dfb0d6eba3282a2f695b12643d8b107ee84d1138f4cdb798291133939
 SHA512:
-  metadata.gz: 2c282cc5e863660f002fe4c03a6da2812e655b07248f238dc4280a4b6430a221076a6329b0548c5243304247013f04095a2ac3c2d0ee9ad46ee1c33b682335cd
-  data.tar.gz: c300f6cbd8767592793a2b2e30ec8d3e281287ae2e0f25750b3fc0e4ee84f4ff3dd3d2e85871256ce637110703bc085e7ed0edfa08e8c0b0c3dae4f84a952a34
+  metadata.gz: 938c3e4a721e3b939bda7db593602462325773d2100270c4418d8162b353df88b7d1d63bfde4872fcd15a32fc5da33eed8a5148bd3fbd6efebd24646726cb4a3
+  data.tar.gz: 27aa3098126b9ec3046be95869992bedb04581ca2a915bf4cc8092894807fe843fdacbc3358a4ab0f52f8d679155e5109abc9c589702444b00a7b105a8f0ae85

data/Dockerfile

@@ -0,0 +1,13 @@
+FROM opensuse:leap
+LABEL maintainer="Eugene Istomin <e.istomin@innosense.org>"
+
+RUN zypper ar -cf http://download.opensuse.org/repositories/openSUSE:/Leap:/15.0/standard standard::leap15.0
+RUN zypper ar -cf http://download.opensuse.org/update/leap/15.0/oss update_oss::leap15.0
+RUN zypper ar -cf http://download.opensuse.org/update/openSUSE-stable update_oss::stable
+
+RUN zypper --gpg-auto-import-keys --non-interactive in --replacefiles -f ruby2.5-devel ruby2.5-rubygem-sqlite3
+RUN gem specific_install -u -l https://gitlab.com/Innosense/MVP/dialog_ruby.git
+
+ENTRYPOINT ["/bin/echo"]
+CMD ["hi"]
+ 

data/{lib/storage/redis → _dev}/redisPubSub.rb

@@ -33,7 +33,7 @@ module Dialog
       def subGetUnsub(args,redisChannelPrefix,redisSubTimeout)
         args[:x][:redis][:redisChannelpostfix].nil? ? args[:x][:redis][:redisChannelpostfix] = "" : true      
         redisChannel = "#{redisChannelPrefix}__#{args[:x][:redis][:subPostfix]}_#{args[:x][:redis][:redisChannelpostfix]}"      
-        redis ||= Redis.new(path: Bot.configuration.redis_url)
+        #redis ||=  Redis.new
         redis.subscribe_with_timeout(redisSubTimeout, redisChannel) do |on, subscriptions|    
           Dialog.logger.debug  "Subscribed to #{redisChannel} (#{subscriptions} subscriptions)"                  
           on.message do |channel, msg|

data/background/_init.rb

@@ -0,0 +1,43 @@
+########    #######    ########    #######    ########    ########
+##     / / / /    License    \ \ \ \ 
+##    Copyleft culture, Copyright (C) is prohibited here
+##    This work is licensed under a CC BY-SA 4.0
+##    Creative Commons Attribution-ShareAlike 4.0 License
+##    Refer to the http://creativecommons.org/licenses/by-sa/4.0/
+########    #######    ########    #######    ########    ########
+##    / / / /    Code Climate    \ \ \ \ 
+##    Language = ruby
+##    Indent = space;    2 chars;
+########    #######    ########    #######    ########    ########
+
+module Dialog
+  module Background
+    class << self
+    
+      require 'active_support/time'
+      require 'drb'            
+      require_relative './../lib/require.rb'
+      Dialog._require_min
+      Dialog._require_background    
+      Dir.glob(File.join("#{File.dirname(__FILE__)}/../lib/**", "*.rb")).each{|f| require f}
+      
+        Sidekiq.configure_server do |config|
+          config.redis = { path: Dialog.config.db.redis_url, db: Dialog.config.db.redisDBBackground }
+          config.average_scheduled_poll_interval = 5   ## DEV
+        end
+
+        Sidekiq.configure_client do |config|
+          config.redis = { path: Dialog.config.db.redis_url, db: Dialog.config.db.redisDBBackground }
+        end            
+      
+      ## DEV
+      rs = Sidekiq::RetrySet.new
+      rs.size
+      rs.clear    
+      ##
+      
+      Dir.glob(File.join("#{File.dirname(__FILE__)}/**", "*.rb")).each{|f| require f}
+    
+    end
+  end
+end

data/background/batch.rb

@@ -0,0 +1,69 @@
+########    #######    ########    #######    ########    ########
+##     / / / /    License    \ \ \ \ 
+##    Copyleft culture, Copyright (C) is prohibited here
+##    This work is licensed under a CC BY-SA 4.0
+##    Creative Commons Attribution-ShareAlike 4.0 License
+##    Refer to the http://creativecommons.org/licenses/by-sa/4.0/
+########    #######    ########    #######    ########    ########
+##    / / / /    Code Climate    \ \ \ \ 
+##    Language = ruby
+##    Indent = space;    2 chars;
+########    #######    ########    #######    ########    ########
+    
+module Dialog
+  module Background
+    
+    class BotApi
+      include Sidekiq::Worker
+      include Sidekiq::Status::Worker
+      sidekiq_options queue: 'default'
+
+      def perform(instance,args)
+        batch = Sidekiq::Batch.new
+        batch.on(:success, 'Dialog::Background::Callback#success')
+        batch.on(:complete,'Dialog::Background::Callback#complete')
+
+        batch.jobs do
+          BotApiSend.perform_in(5.seconds.from_now,instance,args.to_h)
+        end
+        
+      end
+    end
+
+
+    class Callback
+      include Sidekiq::Worker
+      include Sidekiq::Status::Worker
+      sidekiq_options queue: 'default'
+
+      def success(status, options)
+        puts "You called a callback 'success', status: #{status}, options: #{options}"
+      end
+      
+      def complete(status, options)
+        puts "You called a callback 'complete', status: #{status}, options: #{options}"
+        #if status.failures != 0
+      end  
+    end
+
+
+
+    class BotApiSend
+      include Sidekiq::Worker
+      include Sidekiq::Status::Worker
+      sidekiq_options queue: 'default'
+
+      def perform(instance, mmsg)
+        bot = DRbObject.new_with_uri("drbunix:./_data/bot/#{instance}/drb.sock")
+        puts "instance: #{instance}, mmsg: #{mmsg}"
+        smsg = {}
+        pp bot.api.send(:getMe)
+        chatId = mmsg["fqndata"]["o"]["chat"]["id"]
+        pp bot.api.send(:sendMessage, {'chat_id' => chatId, 'text' => "This is the background proc, man: reply to you message from '#{DateTime.strptime(mmsg["fqndata"]["o"]["msg"]["date"].to_s, '%s')}'"}) 
+      end
+    end
+
+
+
+  end
+end

data/background/redis.rb

@@ -0,0 +1,34 @@
+########    #######    ########    #######    ########    ########
+##     / / / /    License    \ \ \ \ 
+##    Copyleft culture, Copyright (C) is prohibited here
+##    This work is licensed under a CC BY-SA 4.0
+##    Creative Commons Attribution-ShareAlike 4.0 License
+##    Refer to the http://creativecommons.org/licenses/by-sa/4.0/
+########    #######    ########    #######    ########    ########
+##    / / / /    Code Climate    \ \ \ \ 
+##    Language = ruby
+##    Indent = space;    2 chars;
+########    #######    ########    #######    ########    ########
+
+module Dialog
+  module Background
+    class << self
+
+      Sidekiq::Cron::Job.create(name: 'RedisDumper - every 1 min', cron: '*/1 * * * *', class: 'Dialog::Background::RedisDumper')
+      pp Sidekiq::Cron::Job.all      
+      
+    end
+    
+    class RedisDumper
+      include Sidekiq::Worker
+      include Sidekiq::Status::Worker
+      sidekiq_options queue: 'default'
+      
+      def perform
+        puts Dialog::Statistics.redisDump2json
+      end
+      
+    end
+  
+  end
+end

data/dialog_ruby.gemspec

@@ -20,21 +20,20 @@ Gem::Specification.new do |s|
   s.executables   = s.files.grep(%r{^exe/}) { |f| File.basename(f) }  
   s.require_paths = ['lib'.freeze]
   
-  #Innosense
-  s.add_runtime_dependency  'dialect_lookup',       '~> 0.0.4'
-  s.add_runtime_dependency  'dialog_bot_ruby',      '~> 0.8.7'
-  
-  
   #External
   s.add_runtime_dependency  'tdlib-ruby',           '~> 1.0'
+  s.add_runtime_dependency  'activesupport',        '~> 5.2'  
   s.add_runtime_dependency  'i18n',                 '~> 1.1'
   s.add_runtime_dependency  'redis',                '~> 4.0'
-  s.add_runtime_dependency  'concurrent-ruby',      '~> 1.0.5'
+  s.add_runtime_dependency  'concurrent-ruby',      '~> 1.0'
   s.add_runtime_dependency  'oj',                   '~> 3.6'
-  s.add_runtime_dependency  'sidekiq-cron',         '~> 1.0'
   s.add_runtime_dependency  'sidekiq',              '~> 5.2'
+  s.add_runtime_dependency  'sidekiq-batch',        '~> 0.1' 
+  s.add_runtime_dependency  'sidekiq-status',       '~> 1.1'
+  s.add_runtime_dependency  'sidekiq-cron',         '~> 1.0'
   s.add_runtime_dependency  'sidekiq-rate-limiter', '~> 0.1'
   s.add_runtime_dependency  'dry-configurable',     '~> 0.7'
+  s.add_runtime_dependency  'redis-dump',           '~> 0.4'
   
 end
 

data/exec/conf/redis_global.conf

@@ -1,16 +1,18 @@
 daemonize no
-port 0
+port 4000
 unixsocketperm 700
 loglevel notice
 syslog-enabled yes
 syslog-facility local1
-databases 2
+databases 4
 save 600 1
 save 300 5
 save 180 10
 save 60 50
 save 10 100
 
+maxmemory-policy noeviction
+
 stop-writes-on-bgsave-error yes
 rdbcompression yes
 rdbchecksum yes

data/exec/dialog.sh

@@ -15,38 +15,40 @@ set -e
 ## PRE - ENV ##
 # . ./engine/exec.sh
 
-if [[ -z $RunMethod ]]; then
-    DialogMsg="Please specify run method"
-    echo $DialogMsg; select RunMethod in console systemd;  do  break ; done
+if [[ -z $runInstance ]]; then
+    DialogMsg="Please specify instance"
+    echo $DialogMsg; select runInstance in `ls ./_env/*|grep -v tmpl|sed -e 's/.*.\///'`;  do  break ; done
+    runMode=`echo $runInstance|cut -d '.' -f 2`
+    runInstance=`echo $runInstance|cut -d '.' -f 1`
 fi
 
-if [[ -z $BotFullname ]]; then
-    DialogMsg="Please specify BotFullname"
-    echo $DialogMsg; select BotFullname in `ls ./_env/*|grep -v tmpl|sed -e 's/.*.\///'`;  do  break ; done
+if [[ -z $runScope ]]; then
+    DialogMsg="Please specify scope"
+    echo $DialogMsg; select runScope in init systemd_start console_start;  do  break ; done
+    showCommand=1
 fi
 
-case $RunMethod in 
-'sytemd')
+case $runScope in 
+'console_start')
+  runModes="bot tdlib check dialect" 
 ;;
+*)
+  runModes="bot tdlib bot_dev"
+;;
+esac
 
-'console')
-
-  if [[ -z $runScope ]]; then
-      DialogMsg="Please specify scope"
-      echo $DialogMsg; select runScope in init start stop;  do  break ; done
-  fi
-  
-  if [[ -z $runMode ]]; then
-      DialogMsg="Please specify mode: "
-      echo $DialogMsg; select runMode in check bot dev dev_offline;  do  break ; done
-  fi  
+if [[ -z $runMode ]]; then
+    DialogMsg="Please specify mode: "
+    echo $DialogMsg; select runMode in $runModes;  do  break ; done
+fi   
   
-  export $(grep -v '^#' ./_env/$BotFullname | xargs)
+if [[ $showCommand == 1 ]]; then
+  GREEN='\033[0;32m'
+  NC='\033[0m' # No Color
+  runCommand="runInstance=$runInstance runScope=$runScope runMode=$runMode"
+  echo -e "Start using:\n${GREEN}$runCommand bash ./dialog.sh ${NC}"  
+else
+  export $(grep -v '^#' ./_env/$runInstance.* | xargs)
   ruby ./dialog.rb
+fi
 
-;;
-
-*)
-echo 'Wrong run method!'
-;;
-esac

data/exec/git_sync/dialog__sync.service

@@ -4,7 +4,7 @@ After=syslog.target network.target
 
 [Service]
 Type=oneshot
-WorkingDirectory={RunPath}
+WorkingDirectory=<%=Dialog.config.path.dialog%>
 ExecStart=/usr/bin/git pull
 StandardOutput=syslog
 StandardError=syslog

data/exec/prereq.sh

@@ -1,2 +1,19 @@
+#!/bin/bash
+########    #######    ########    #######    ########    ########
+##     / / / /    License    \ \ \ \ 
+##    Copyleft culture, Copyright (C) is prohibited here
+##    This work is licensed under a CC BY-SA 4.0
+##    Creative Commons Attribution-ShareAlike 4.0 License
+##    Refer to the http://creativecommons.org/licenses/by-sa/4.0/
+########    #######    ########    #######    ########    ########
+##    / / / /    Code Climate    \ \ \ \ 
+##    Language = bash
+##    Indent = space;    2 chars;
+########    #######    ########    #######    ########    ########
+set -e
+
 zypper ar -cf https://download.opensuse.org/repositories/devel:/languages:/ruby:/extensions/openSUSE_Leap_15.0/devel:languages:ruby:extensions.repo
-zypper --gpg-auto-import-keys --non-interactive in ruby2.5-devel ruby2.5-rubygem-sqlite3 ruby2.5-rubygem-oj redis
+zypper ar -cf https://download.opensuse.org/repositories/devel:/languages:/ruby/openSUSE_Leap_15.0/devel:languages:ruby.repo
+zypper --gpg-auto-import-keys --non-interactive up
+zypper --gpg-auto-import-keys --non-interactive in jemalloc tdlib redis
+zypper --gpg-auto-import-keys --non-interactive in ruby2.5-devel ruby2.5-rubygem-sqlite3 ruby2.5-rubygem-oj ruby2.5-rubygem-yajl-ruby

data/exec/systemd/dialog_background.service

@@ -0,0 +1,29 @@
+[Unit]
+Description= Dialog background runner
+After=syslog.target network.target
+
+[Service]
+Type=simple
+WorkingDirectory=<%=Dialog.config.path.dialog%>
+
+Environment=runScope=_run
+Environment=runMode=background
+
+# Greatly reduce Ruby memory fragmentation and heap usage
+# https://www.mikeperham.com/2018/04/25/taming-rails-memory-bloat/
+Environment=MALLOC_ARENA_MAX=2
+# Use jemalloc
+# https://brandonhilkert.com/blog/reducing-sidekiq-memory-usage-with-jemalloc/
+Environment=LD_PRELOAD=/usr/lib64/libjemalloc.so.2
+
+ExecStart=/usr/bin/ruby ./dialog.rb
+
+RestartSec=5
+TimeoutStopSec=5
+Restart=on-failure
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=dialog_background
+
+[Install]
+WantedBy=default.target

data/exec/systemd/dialog_redis.service

@@ -12,4 +12,4 @@ ExecStart=/usr/sbin/redis-server ./redis_local.conf
 Restart=on-failure
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=default.target

data/exec/systemd/dialog_runMode@.service

@@ -1,3 +1,5 @@
+<% erb.has_key?(:env) ? env = erb[:env] : env = erb[:mode]
+%>
 [Unit]
 Description= Dialog (Telegram Bot abstraction layer) - instance %i
 After=syslog.target network.target
@@ -5,22 +7,30 @@ After=syslog.target network.target
 [Service]
 Type=simple
 WorkingDirectory=<%=Dialog.config.path.dialog%>
-EnvironmentFile=<%=Dialog.config.path.dialog%>/_env/%i
-Environment=runScope=start
-Environment=runMode=<%=mode%>
-Environment=BotFullname=%i
+EnvironmentFile=<%=Dialog.config.path.dialog%>/_env/%i.<%= env %>
+Environment=runScope=_run
+Environment=runMode=<%=erb[:mode]%>
+Environment=runInstance=%i
 
 # Greatly reduce Ruby memory fragmentation and heap usage
 # https://www.mikeperham.com/2018/04/25/taming-rails-memory-bloat/
 Environment=MALLOC_ARENA_MAX=2
+# Use jemalloc
+# https://brandonhilkert.com/blog/reducing-sidekiq-memory-usage-with-jemalloc/
+Environment=LD_PRELOAD=/usr/lib64/libjemalloc.so.2
 
+<% if erb.has_key?(:systemdChain)%>
+ExecStartPre=/usr/bin/ruby ./dialog.rb
+ExecStart=/usr/bin/systemctl --user restart <%=erb[:systemdChain]%>@%i
+<% else %>
 ExecStart=/usr/bin/ruby ./dialog.rb
+<% end %>
 RestartSec=5
 TimeoutStopSec=5
 Restart=on-failure
 StandardOutput=syslog
 StandardError=syslog
-SyslogIdentifier=dialog_<%=mode%>_%i
+SyslogIdentifier=dialog_<%=erb[:mode]%>_%i
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=default.target

data/exec/systemd/dialog_runner@.path

@@ -0,0 +1,8 @@
+[Unit]
+Description=<%=erb[:desc]%>
+
+[Path]
+PathChanged=<%=erb[:path]%>
+
+[Install]
+WantedBy=default.target

data/lib/{triggers/start → begin}/_main.rb

@@ -13,9 +13,11 @@
 module Dialog  
   class << self
     
-    def trigger_start
+    def _run(mode=false)
       args = {}
-      case Dialog.config.common.mode
+      mode == false ? mode = Dialog.config.run.mode : true
+      Dialog.logger.debug "Using mode '#{mode}'"      
+      case mode
       when 'check'
         _require
         jsonDriven(args, :check)
@@ -23,31 +25,27 @@ module Dialog
         _require
         jsonDriven(args, :prod)
         bot(args)
-      when 'dev'
-        _require
-        dialectDriven(args)
-        jsonDriven(args, :prod)
-        bot(args)
-      when 'dev_offline'
+      when 'dialect'
         _require
         dialectDriven(args)
         jsonDriven(args, :check) 
       when 'tdlib'
         start_tdlib(args) 
+      when 'background'
+        start_background(args)         
       else
-         Dialog.logger.error "ERROR: no such mode - '#{Dialog.config.common.mode}'"
+         Dialog.logger.error "ERROR: no such mode - '#{mode}'"
       end      
     end    
     
-  
     
     def dialectDriven(args)
       require 'dialect_lookup'
       state = {
-        dialectFolder:  Dialog.config.dialect.folder, 
+        dialectFolder:  Dialog.config.naming.interface, 
         myAppName:      Dialog.config.naming.app, 
         sqlite:         Dialog.config.dialect.sqlite, 
-        jsonFolder:  "#{Dialog.config.path.dialogRender}/json/"
+        json:           Dialog.config.dialect.json
       }
       ::Dialect::Lookup.new.run(state)
 #       File.delete(Dialog.config.dialect.sqlite)
@@ -58,15 +56,25 @@ module Dialog
       FileUtils.rm_rf(Dir.glob("#{Dialog.config.path.dialogRender}/code_#{type}/*.rb"))     
       args[:dialect] = {}      
       
-      json = "#{Dialog.config.path.dialogRender}/json/#{Dialog.config.dialect.folder}_#{Dialog.config.naming.app}.json"
-      jsonLoad = Oj.load(File.read(json), symbol_keys: true)
-      args[:dialect] = jsonLoad[:stages][Dialog.config.dialect.folder.to_sym]
-      Dialog::Dialect.run(args, type)
-  
-      args[:dialect] ={} 
-      args[:tmp] = {}      
-      Dir.glob(File.join("#{Dialog.config.path.dialogRender}/code_#{type}/**", "*.rb")).sort.each{|f| require f}
-      Dialog.logger.info "Rendered bot code is ok"      
+      json = Dialog.config.dialect.json
+      if File.file?(json)
+        jsonLoad = Oj.load(File.read(json), symbol_keys: true)
+        args[:dialect] = jsonLoad[:stages][Dialog.config.naming.interface.to_sym]
+        Dialog::Dialect.run(args, type)
+    
+        args[:dialect] ={} 
+        args[:tmp] = {}      
+        Dir.glob(File.join("#{Dialog.config.path.dialogRender}/code_#{type}/**", "*.rb")).sort.each{|f| require f}
+        Dialog.logger.info "Rendered bot code is ok"    
+      else
+        Dialog.logger.info "No JSON to render, trying to export"
+        begin
+          trigger_start('dialect')
+        rescue
+          Dialog.logger.error "Seems like no Dialect DB, exiting! Please upload valid JSON or use a valid Dialect DB"          
+          exit(false)
+        end
+      end
     end