jiji 1.2.5

data/lib/jiji/process_manager.rb

@@ -0,0 +1,290 @@
+
+require 'uuidtools'
+require 'yaml'
+require 'jiji/util/fix_yaml_bug'
+
+module JIJI
+  class ProcessManager
+
+    include Enumerable
+
+    def initialize( registry )
+      @registry = registry
+      @executor = registry.back_test_process_executor
+      @rmt = @registry.rmt_process
+      @back_tests = {}
+      @logger = registry.server_logger
+
+      # 既存のバックテスト一覧を読み込む
+      Dir.glob( "#{registry.process_dir}/*" ) {|d|
+        next unless File.directory? d
+        next unless File.exist? "#{d}/#{JIJI::ProcessInfo::PROPERTY_FILE}"
+        begin
+          id = File.basename(d)
+          next if id == "rmt"
+          info = @registry.process_info( id )
+          process = @registry.process_info(id)
+          process.load # 保存しておいた設定情報をロード
+          @back_tests[id] = process
+        rescue
+          @logger.error $!
+        end
+      }
+    end
+
+    # RMTプロセスをスタートする
+    def start
+      if @conf.get([:collector,:collect], true )
+        begin
+          @rmt.load_agents( true )
+          @rmt.start
+        rescue
+          @rmt.logger.error $!
+        end
+      end
+    end
+
+    # すべてのプロセスを停止する
+    def stop
+      @stoped = true
+      @registry.operator( "rmt", false, nil).stop
+      @rmt.stop
+      @executor.stop
+    end
+
+    # バックテストプロセスを列挙する
+    def each( &block )
+      @back_tests.each_pair {|k,v|
+        yield v
+      }
+    end
+
+    # プロセスを取得する
+    def get( id )
+      unless id == "rmt" || @back_tests.key?(id)
+        raise UserError.new( JIJI::ERROR_NOT_FOUND, "process not found")
+      end
+      id == "rmt" ? @rmt.info : @back_tests[id]
+    end
+
+    # プロセスの設定を更新する。
+    def set( process_id, setting )
+      p = get( process_id )
+      failed = {}
+      setting.each_pair {|k,v|
+        case k
+          when "trade_enable"
+            # バックテストの変更は許可しない。
+            unless  process_id == "rmt"
+              raise UserError.new( JIJI::ERROR_ILLEGAL_ARGUMENTS, "illegal id.id=#{process_id}" )
+            end
+            value = ( v == "true" || v == true )
+            @rmt.trade_enable = value
+            @rmt.info["trade_enable"] = value
+          when "agents"
+            p = get( process_id )
+            if ( process_id == "rmt" )
+              failed = @rmt.set_agents(v)
+            else
+              failed = @executor.set_agents(process_id, v)
+            end
+            p["agents"] = create_success_agent_setting( p["agents"], v, failed )
+        end
+      }
+      failed
+    end
+
+    # バックテストプロセスを新規に作成する
+    def create_back_test( name, memo, start_date, end_date, agent_properties )
+      id = UUIDTools::UUID.random_create().to_s
+
+      # プロパティを記録
+      props = {
+        "id"=>id,
+        "name"=>name,
+        "memo"=>memo,
+        "create_date"=>Time.now.to_i,
+        "start_date"=>start_date.to_i,
+        "end_date"=>end_date.to_i,
+        "agents"=>agent_properties,
+        "state"=>:WAITING
+      }
+      return start_process( id, props )
+    end
+
+    # バックテストプロセスを再起動する
+    def restart_test( id, agent_properties=nil )
+      unless id == "rmt" || @back_tests.key?(id)
+        raise UserError.new( JIJI::ERROR_NOT_FOUND, "process not found")
+      end
+      p = get(id)
+      new_id = UUIDTools::UUID.random_create().to_s # 別のIDを再割り当てする。
+      props = {
+        "id"=>new_id,
+        "name"=>p["name"],
+        "memo"=>p["memo"],
+        "create_date"=>Time.now.to_i,
+        "start_date"=>p["start_date"],
+        "end_date"=>p["end_date"],
+        "agents"=>agent_properties || p["agents"],
+        "state"=>:WAITING
+      }
+      result = start_process( new_id, props )
+      delete_back_test( id )
+      return result
+    end
+
+    # バックテストプロセスを削除する
+    # ログファイルもすべて削除。
+    def delete_back_test( id )
+      unless id == "rmt" || @back_tests.key?(id)
+        raise UserError.new( JIJI::ERROR_NOT_FOUND, "process not found")
+      end
+      @executor.delete(id)
+      @back_tests.delete( id )
+      FileUtils.rm_rf "#{@registry.process_dir}/#{id}"
+    end
+
+    attr :registry, true
+    attr :conf, true
+    attr :executor, true
+    attr :rmt, true
+
+  private
+    # バックテストを開始する
+    def start_process( id, props )
+      info = @registry.process_info( id )
+      info.props = props
+      @executor << info
+      @back_tests[id] = info
+      return {
+       "id"=>id,
+       "name"=>props["name"],
+       "create_date"=>props["create_date"]
+      }
+    end
+
+    #更新に成功したエージェントの設定情報を作成する
+    def create_success_agent_setting( current, new_agents, failed )
+      failed.each {|pair|
+        case pair[1][:operation]
+          #新規作成で失敗したものは除外
+          when :add
+            new_agents = new_agents.reject {|i| i["id"] == pair[0]}
+          #更新、削除で失敗したものは現在の設定をそのままキープする
+          when :update
+            new_agents = new_agents.map {|i| 
+               i["id"] != pair[0] ? i : current.find {|j| j["id"] == pair[0] }
+            }
+          when :remove
+            new_agents << current.find {|j| j["id"] == pair[0] }
+        end
+      }
+      return new_agents
+    end
+  end
+
+  class BackTestProcessExecutor
+
+    def initialize
+      @running = nil
+      @mutex = Mutex.new
+      @waiting = []
+    end
+
+    #実行対象とするプロセスを追加する
+    def <<(info)
+      begin
+        btp = @registry.backtest_process(info )
+        btp.load_agents( false )
+        @mutex.synchronize {
+          if @running == nil
+            @running = btp
+            @running.collector.listeners << self
+            @running.start
+          else
+            @waiting << btp
+          end
+        }
+      rescue Exception
+        begin
+          btp.stop if btp
+        rescue Exception
+        ensure
+          FileUtils.rm_rf "#{@process_dir}/#{info.process_id}"
+        end
+        raise $!
+      end
+    end
+
+    #全てのプロセスの実行を停止する
+    def stop
+      @mutex.synchronize {
+        @waiting.each {|i|
+          i.collector.listeners.delete(self)
+          i.stop
+        }
+        @waiting.clear
+        if @running != nil
+          @running.collector.listeners.delete(self)
+          @running.stop
+          @running = nil
+        end
+      }
+    end
+
+    #プロセスを削除する
+    def delete(id)
+      @mutex.synchronize {
+        if @running != nil && @running.info.process_id == id
+          #実行中であれば停止して次に
+          @running.collector.listeners.delete(self)
+          @running.stop
+          run_next
+        else
+          # 待機中であればキューから除外。
+          @waiting = @waiting.reject{|i| i.info.process_id == id }
+        end
+      }
+    end
+
+    #エージエントの設定を更新する
+    def set_agents(id, agents)
+      @mutex.synchronize {
+        p = get(id)
+        p ? p.set_agents(agents) : {}
+      }
+    end
+
+    #idに対応するプロセスがあれば取得する
+    def get(id)
+      if @running != nil && @running.info.process_id == id
+        return @running
+      else
+        return @waiting.find{|p|p.info.process_id == id}
+      end
+    end
+
+    #プロセスの実行終了通知を受け取る
+    def on_finished(state, time)
+      @mutex.synchronize {
+        run_next
+      }
+    end
+
+    attr :registry, true
+
+  private
+    #待ち中のプロセスがあれば実行を開始する。
+    def run_next
+      unless @waiting.empty?
+        @running = @waiting.shift
+        @running.collector.listeners << self
+        @running.start
+      else
+        @running = nil
+      end
+    end
+  end
+
+end

data/lib/jiji/registry.rb

@@ -0,0 +1,349 @@
+
+require 'rubygems'
+require 'yaml'
+require 'jiji/util/fix_yaml_bug'
+require 'needle'
+require 'jiji/collector'
+require 'jiji/configuration'
+require 'jiji/observer'
+require 'jiji/process'
+require 'jiji/process_manager'
+require 'jiji/agent/agent'
+require 'jiji/agent/agent_manager'
+require 'jiji/agent/agent_registry'
+require 'jiji/agent/permitter'
+require 'jiji/agent/util'
+require 'jiji/output'
+require 'jiji/output_manager'
+require 'jiji/operator'
+require "jiji/dao/file_system_dao"
+require "jiji/dao/rate_dao"
+require "jiji/dao/trade_result_dao"
+require 'logger'
+require 'fileutils'
+require 'jiji/util/synchronize_interceptor'
+
+require 'jiji/service/agent_service'
+require 'jiji/service/hello_service'
+require 'jiji/service/rate_service'
+require 'jiji/service/trade_result_service'
+require 'jiji/service/output_service'
+require 'jiji/service/process_service'
+require 'jiji/service/system_service'
+
+require 'jiji/migration/migrator'
+require 'jiji/migration/migrator1_0_3'
+require 'jiji/migration/migrator1_1_0'
+require 'jiji/migration/migrator1_2_0'
+
+require 'jiji/plugin/plugin_loader'
+require 'jiji/plugin/securities_plugin'
+
+module JIJI
+
+  #
+  # レジストリ
+  #
+  class Registry
+
+    def initialize(base, server=nil)
+      @registry = Needle::Registry.new {|r|
+
+        # ベースディレクトリ
+        r.register( :base_dir) { base }
+        # サーバー
+        r.register( :server) { server }
+
+        # 設定値
+        r.register( :conf ) {
+          JIJI::Configuration.new r.base_dir + "/conf/configuration.yaml"
+        }
+        # レート一覧置き場
+        r.register( :rate_dir ) {
+          dir = r.base_dir + "/" + r.conf.get([:dir,:rate_data], "rate_datas")
+          FileUtils.mkdir_p dir
+          dir
+        }
+        # プロセスデータ置き場
+        r.register( :process_dir ) {
+	        process_dir = "#{r.base_dir}/#{r.conf.get([:dir,:process_log], "process_logs")}"
+	        FileUtils.mkdir_p process_dir
+          process_dir
+        }
+        # エージェント置き場
+        r.register( :agent_dir ) {
+          agent = r.conf.get([:dir,:agent], "agents")
+          FileUtils.mkdir_p "#{r.base_dir}/#{agent}"
+          agent
+        }
+        r.register( :shared_lib_dir ) {
+          shared_lib = r.conf.get([:dir,:shared_lib], "shared_lib")
+          FileUtils.mkdir_p "#{r.base_dir}/#{shared_lib}"
+          shared_lib
+        }
+        # 出力データ置き場
+        r.register( :output_dir, :model=>:multiton_initialize ) {|c,p,id|
+          dir = "#{r.process_dir}/#{id}/out"
+          FileUtils.mkdir_p dir
+          dir
+        }
+
+        # バージョンファイル
+        r.register( :version_file ) {
+          r.base_dir + "/data_version"
+        }
+
+        # ロガー
+        r.register( :server_logger ) {
+          dir = "#{r.base_dir}/#{r.conf.get([:dir,:log], "logs")}"
+          FileUtils.mkdir_p dir
+          l = Logger.new( dir + "/log.txt", 10, 512*1024 )
+          l.level = Logger::DEBUG
+          l
+        }
+        r.register( :process_logger, :model=>:multiton_initialize ) {|c,p,id|
+          dir = "#{r.process_dir}/#{id}"
+          FileUtils.mkdir_p dir
+          c = Logger.new( dir + "/log.txt", 10, 512*1024 )
+          r.permitter.proxy( c, [/^(info|debug|warn|error|fatal|close)$/] )
+        }
+
+        # Permitter
+        r.register( :permitter ) {
+          JIJI::Permitter.new( 5, 0 )
+        }
+
+        # Dao
+        r.register( :scales ) {
+          ["1m", "5m", "10m", "30m", "1h", "6h", "1d", "2d", "5d"]
+        }
+        r.register( :rate_dao ) {
+          JIJI::Dao::RateDao.new( r.rate_dir, r.scales )
+        }
+        r.register( :trade_result_dao, :model=>:multiton_initialize ) {|c,p,id|
+          dir = "#{r.process_dir}/#{id}/trade"
+          FileUtils.mkdir_p dir
+          JIJI::Dao::TradeResultDao.new( dir, r.scales )
+        }
+        r.register( :agent_file_dao ) {
+          dir = "#{r.base_dir}/#{r.conf.get([:dir,:agent], "agents")}"
+          sdir = "#{r.base_dir}/#{r.conf.get([:dir,:shared_lib], "shared_lib")}"
+          FileUtils.mkdir_p dir
+          FileUtils.mkdir_p sdir
+          JIJI::Dao::FileSystemDao.new( r.base_dir )
+        }
+
+        # アウトプット
+        r.register( :output_manager ) {
+          JIJI::OutputManager.new(r)
+        }
+        r.register( :output, :model=>:multiton_initialize ) {|c,p,id,agent_id|
+          dir = r.output_dir(id)
+          c = JIJI::Output.new(agent_id, dir, r.scales)
+          r.permitter.proxy( c, [/^(get|put|<<)$/], [/^get$/] )
+        }
+
+        # オブザーバー
+        r.register( :rmt_observer_manager ) {
+          JIJI::WorkerThreadObserverManager.new([
+            r.rate_dao, r.agent_manager("rmt",true)
+          ], r.process_logger("rmt"))
+        }
+        r.register( :backtest_observer_manager, :model=>:multiton_initialize ) {|c,p,id|
+          JIJI::ObserverManager.new( [r.agent_manager(id, false)], r.process_logger(id))
+        }
+
+        # エージェントマネージャ
+        r.register( :agent_manager, :model=>:multiton_initialize ) {|c,p,id,failsafe|
+          c = JIJI::AgentManager.new( id, r.agent_registry, r.process_logger(id), failsafe )
+          c.operator = r.operator(id, false, nil) # 作成段階では常に取引は行なわない。
+          c.output_manager = r.output_manager
+          c.conf = r.conf
+          c.trade_result_dao = r.trade_result_dao(id)
+          c
+        }
+        r.intercept( :agent_manager ).with {
+          SynchronizeInterceptor
+        }.with_options( :id=>:agent_manager )
+
+        # エージェントレジストリ
+        r.register( :agent_registry ) {
+          c = JIJI::AgentRegistry.new( r.agent_dir, r.shared_lib_dir )
+          c.conf = r.conf
+          c.server_logger = r.server_logger
+          c.file_dao = r.agent_file_dao
+          c.load_all
+          c
+        }
+
+        # オペレーター
+        r.register( :operator, :model=>:multiton_initialize ) {|c,p,id,trade_enable,money|
+          c = JIJI::RmtOperator.new(r.securities_plugin_manager.selected,
+              r.process_logger(id), r.trade_result_dao(id), trade_enable, money)
+          c.conf = r.conf
+          r.permitter.proxy( c, [/^(sell|buy|commit)$/], [/^(sell|buy)$/])
+        }
+
+        # コレクター
+        r.register( :rmt_collector ) {
+          c = JIJI::Collector.new
+          c.observer_manager = r.rmt_observer_manager
+          c.conf      = r.conf
+          c.logger    = r.process_logger("rmt")
+          c.client    = r.securities_plugin_manager.selected
+          c
+        }
+        r.register( :backtest_collector, :model=>:multiton_initialize ) {|c,p,id, start_date, end_date|
+          c = JIJI::BackTestCollector.new( r.rate_dao, start_date, end_date )
+          c.observer_manager = r.backtest_observer_manager(id)
+          c.conf      = r.conf
+          c.logger    = r.process_logger(id)
+          c.client    = r.securities_plugin_manager.selected
+          c
+        }
+
+        # RMTプロセス
+        r.register( :rmt_process ) {
+          id = "rmt"
+          info = r.process_info( id )
+          if info.data_exist?
+            info.load
+          else
+            now = Time.now.to_i
+            info.props = {
+              "id"=>id,
+              "name"=>"",
+              "memo"=>"",
+              "create_date"=>now,
+              "start_date"=>now,
+              "end_date"=>now,
+              "agents"=>{},
+              "state"=>:WAITING,
+              "trade_enable"=>false
+            }
+          end
+          c = JIJI::Process.new( info )
+          c.agent_manager =  r.agent_manager(id,true)
+          c.trade_enable = ( info["trade_enable"] == true )
+          c.logger = r.process_logger(id)
+          c.observer_manager = r.rmt_observer_manager
+          c.collector = r.rmt_collector
+          c
+        }
+        # バックテストプロセス
+        r.register( :backtest_process, :model=>:prototype ) {|c,p,info|
+          # 既存のバックテストを読み込む場合、プロパティはnil
+          id = info.process_id
+          c = JIJI::Process.new(info)
+          c.agent_manager =  r.agent_manager(id,false)
+          c.trade_enable = false
+          info["trade_enable"] = false
+          c.logger = r.process_logger(id)
+          c.observer_manager = r.backtest_observer_manager(id)
+          c.collector = r.backtest_collector(id,
+            Time.at( info["start_date"]), Time.at( info["end_date"]))
+          c
+        }
+        r.register( :process_info, :model=>:prototype ) {|c,p,id|
+          JIJI::ProcessInfo.new(id, r.process_dir)
+        }
+        r.register( :back_test_process_executor ) {
+          c = JIJI::BackTestProcessExecutor.new
+          c.registry = r
+          c
+        }
+
+        # プロセスマネージャ
+        r.register( :process_manager ) {
+          c = JIJI::ProcessManager.new( r )
+          c.conf = r.conf
+          c
+        }
+        r.intercept( :process_manager ).with {
+          SynchronizeInterceptor
+        }.with_options( :id=>:process_manager )
+
+        # サービス
+        r.register( :hello_service ) {
+          JIJI::Service::HelloService.new
+        }
+        r.register( :agent_service ) {
+          c = JIJI::Service::AgentService.new
+          c.agent_registry = r.agent_registry
+          c.dao = r.agent_file_dao
+          c.process_manager = r.process_manager
+          c.agent_dir = r.agent_dir
+          c.shared_lib_dir = r.shared_lib_dir
+          c.server_logger = r.server_logger
+          c
+        }
+        r.register( :rate_service ) {
+          c = JIJI::Service::RateService.new
+          c.rate_dao = r.rate_dao
+          c
+        }
+        r.register( :trade_result_service ) {
+          c = JIJI::Service::TradeResultService.new
+          c.process_manager = r.process_manager
+          c.registry = r
+          c
+        }
+        r.register( :output_service ) {
+          c = JIJI::Service::OutputService.new
+          c.process_manager = r.process_manager
+          c.process_dir = r.process_dir
+          c.output_manager = r.output_manager
+          c
+        }
+        r.register( :process_service ) {
+          c = JIJI::Service::ProcessService.new
+          c.process_manager = r.process_manager
+          c
+        }
+        r.register( :system_service ) {
+          c = JIJI::Service::SystemService.new
+          c.server = r.server
+          c
+        }
+
+        # データ移行
+        r.register( :migrator ) {
+          c = JIJI::Migration::Migrator.new
+          c.registry = r
+          c.server_logger = r.server_logger
+          c.version_file = r.version_file
+          c.migrators = [
+            { :version=>Gem::Version.new( "1.0.3" ), :migrator=>JIJI::Migration::Migrator1_0_3.new},
+            { :version=>Gem::Version.new( "1.1.0" ), :migrator=>JIJI::Migration::Migrator1_1_0.new},
+            { :version=>Gem::Version.new( "1.2.0" ), :migrator=>JIJI::Migration::Migrator1_2_0.new}
+          ]
+          c
+        }
+
+        # プラグイン
+        r.register( :plugin_loader ) {
+          c = JIJI::Plugin::Loader.new
+          c.server_logger = r.server_logger
+          c
+        }
+        # Securitiesプラグイン
+        r.register( :securities_plugin_manager ) {
+          c = JIJI::Plugin::SecuritiesPluginManager.new
+          c.conf = r.conf
+          c.server_logger = r.server_logger
+          c
+        }
+      }
+    end
+
+    def method_missing(name, *args)
+      @registry.send( name, *args )
+    end
+
+    def [](name)
+      @registry[name]
+    end
+
+  end
+
+end