tes-request 0.8 → 0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7dd957de5812e55a337c9b095dffa7ad80a64a98
-  data.tar.gz: 8ac21c83d116729acb7999adc568a4abbcd09a84
+SHA256:
+  metadata.gz: 30fe936e59566c32df4a77198ead6bda9409c4186a2d806958e36e0ae8f52b64
+  data.tar.gz: 8e54ca43d1549510541f2222ef64128bfabe3f7d462892d86c1e2cb4837ad999
 SHA512:
-  metadata.gz: 737463005aaaf4908a45fdbce18a78e0720013264e4f8a5d6bb59ddf9fc0598ff9daa4a53cef5a9d79c2a3b1a52e3900656446d34f477d18776f8b5569019196
-  data.tar.gz: 7ff8fe4b8e42967fd1eec73dd4b1e677c5f68179abc6ec555cfd976edc75002a9741ed885fbf6a598b1d73bfd287e1f072389040f94ed8c20c636470679b6145
+  metadata.gz: 2dd71729d9b743a767d7081161bad2bff5174dabe8bfc6ce72a5faa373b71336fd3247f0de6ee386a029e5c75484cd088c82690d4dc063a93b1ff6851dcd316a
+  data.tar.gz: 28c3aa96ce12781a0746e4732a0995c7f6c8a00a4a93927ac27463d28592d1b1578e890572ac911ff0bf7f50867f9898a741ed4487fadf4eaa9be0cbcf12ff03

data/.travis.yml

@@ -8,6 +8,7 @@ rvm:
   - 2.3.4
   - 2.4.0
   - 2.4.1
+  - 2.5.0
 before_install: gem install bundler
 install: bundle
 script:

data/bin/tes-ci-pool

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'tes/request/rspec/ci_pool'
+
+usage = <<EOF
+Usage:
+  #{$0} {tes_pool_url} #{Tes::Request::RSpec::CiPool::SUPPORT_FUNCTIONS.join('|')} {project_dir} {job_type} [res_addition_attr_map_json_file_path]
+
+Example:
+   #{$0} http://tes.com/func/ satisfy apps/test_project func_test apps/test_project/res_attr_add_map.json
+EOF
+
+if ARGV.size < 4
+  warn usage
+  exit 2
+end
+
+tes_pool_url = ARGV.shift
+pool_func = ARGV.shift.to_sym
+project_dir = ARGV.shift
+ci_job_type = ARGV.shift
+res_addition_attr_map_json_file = ARGV.shift
+Tes::Request::RSpec::CiPool.
+    new(tes_pool_url, pool_func).
+    run(project_dir, ci_job_type, res_addition_attr_map_json_file)
+
+
+

data/bin/tes-ci-slicer

@@ -3,11 +3,12 @@
 require 'tes/request/rspec/ci_slicer'
 
 usage = <<EOF
-Usage: % #{$0} {#{Tes::Request::RSpec::CiSlicer::SUPPORT_FILE_TYPES.join('|')}} {project_dir} {job_type} {job_min_pieces} [res_addition_attr_map_json_file_path]
+Usage: % #{$0} {#{Tes::Request::RSpec::CiSlicer::SUPPORT_FILE_TYPES.join('|')}} {project_dir} {job_type} {job_min_pieces} [res_addition_attr_map_json_file_path] [pool_url_to_adapt]
 
 Example:
    #{$0} json apps/test_project ci_merge_test 4
    #{$0} properties apps/test_project func_test 16 apps/test_project/res_attr_add_map.json
+   #{$0} properties apps/test_project func_test 16 apps/test_project/res_attr_add_map.json http://tes.server.com/func_test/
 EOF
 
 if ARGV.size < 4
@@ -20,7 +21,8 @@ project_dir = ARGV.shift
 ci_job_type = ARGV.shift
 job_piece_count = ARGV.shift
 res_addition_attr_map_json_file = ARGV.shift
+pool_url = ARGV.shift
 Tes::Request::RSpec::CiSlicer.
-    new(cfg_type, project_dir, res_addition_attr_map_json_file).
+    new(cfg_type, project_dir, res_addition_attr_map_json_file, pool_url).
     run(ci_job_type, job_piece_count.to_i)
 

data/lib/hash.rb

@@ -17,8 +17,8 @@ class Hash
   # @param [String] chain_str 嵌套取值表达式
   # @return [Object]
   def get_by_chain(chain_str)
-    chains = chain_str.split('.')
-    chains.keep_if { |k| k =~ /.+/ }
-    chains.inject(self) { |t, p| t && t.send(p) }
+    chains = chain_str.split('.').map(&:to_sym)
+    chains.keep_if {|k| k =~ /.+/}
+    chains.inject(self) {|t, p| t && t.send(p)}
   end
 end

data/lib/tes/request/client.rb

@@ -53,12 +53,15 @@ module Tes
       end
 
       def release_all_res(user)
-        res_list = parse_res @driver.get('res')
-        res_list[:data].select {|_k, c| c[:users] && c[:users].include?(user)}.each do |id, _c|
+        get_all_res.select {|_k, c| c[:users] && c[:users].include?(user)}.each do |id, _c|
           release_res(id, user)
         end
       end
 
+      def get_all_res
+        parse_res(@driver.get('res'))[:data]
+      end
+
       private
       def parse_res(res)
         unless res.ok? or res.redirect?

data/lib/tes/request/expression.rb

@@ -1,3 +1,5 @@
+require_relative '../../hash'
+
 module Tes
   module Request
     class Expression

data/lib/tes/request/rspec/ci_pool.rb

@@ -0,0 +1,42 @@
+require 'yaml'
+require 'fileutils'
+require_relative 'distribute'
+require_relative '../client'
+
+module Tes::Request::RSpec
+  class CiPool
+    SUPPORT_FUNCTIONS = %i[satisfy]
+
+    def initialize(tes_pool_url, pool_func) #
+      @pool_url = tes_pool_url
+      @pool_func = pool_func
+      driver = HTTPClient.new(base_url: tes_pool_url)
+      driver.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      client = ::Tes::Request::Client.new(driver)
+      @pool = client.get_all_res
+    end
+
+    def run(*args)
+      send(@pool_func, *args)
+    end
+
+    def satisfy(project_dir, ci_type, res_replace_map_json_file=nil)
+      res_addition_attr_map = (res_replace_map_json_file && JSON.parse(File.read(res_replace_map_json_file)))
+      rspec_distribute = Distribute.new(project_dir)
+      jobs = rspec_distribute.distribute_jobs(ci_type, 1, res_addition_attr_map)
+      not_satisfied_profiles = jobs.inject([]) do |t, job|
+        t << job[:profile] unless job[:profile].request(@pool).all?
+        t
+      end
+      unless not_satisfied_profiles.empty?
+        warn <<EOF
+!!!No matched resources in pool for profiles ==>
+
+#{not_satisfied_profiles.join("\n------------------\n")}
+~~~~~~~~~~~~~~~~~~~~~~~!!!
+EOF
+        raise('No matched resources in pool')
+      end
+    end
+  end
+end

data/lib/tes/request/rspec/ci_slicer.rb

@@ -2,12 +2,13 @@ require 'yaml'
 require 'java-properties'
 require 'fileutils'
 require_relative 'distribute'
+require_relative '../client'
 
 module Tes::Request::RSpec
   class CiSlicer
     SUPPORT_FILE_TYPES = [:yaml, :yml, :json, :properties]
 
-    def initialize(file_type, project_dir, res_replace_map_json_file=nil)
+    def initialize(file_type, project_dir, res_replace_map_json_file=nil, pool_url=nil)
       unless SUPPORT_FILE_TYPES.include?(file_type.to_sym)
         raise(ArgumentError, "Not supported file type:#{file_type}!")
       end
@@ -16,12 +17,21 @@ module Tes::Request::RSpec
       @project_dir = project_dir
       @res_addition_attr_map = (res_replace_map_json_file && JSON.parse(File.read(res_replace_map_json_file)))
       @cfg_target_dir = File.join(@project_dir, '.ci_jobs')
+      @pool_url = pool_url
     end
 
     def run(ci_type, slice_count)
       puts "Generate RSpec distribute jobs #{@cfg_file_type} file for CI"
       rspec_distribute = Distribute.new(@project_dir)
-      jobs = rspec_distribute.distribute_jobs(ci_type, slice_count, @res_addition_attr_map)
+      pool = if @pool_url
+               driver = HTTPClient.new(base_url: @pool_url)
+               driver.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
+               client = ::Tes::Request::Client.new(driver)
+               @pool = client.get_all_res
+             else
+               {}
+             end
+      jobs = rspec_distribute.distribute_jobs(ci_type, slice_count, @res_addition_attr_map, pool)
       save_job_files(jobs, @cfg_target_dir, @cfg_file_type)
       @cfg_target_dir
     end
@@ -29,7 +39,7 @@ module Tes::Request::RSpec
     def spec_tag_param_str(tags)
       case tags
         when Array
-          tags.map { |t| "--tag #{t}" }.join(' ')
+          tags.map {|t| "--tag #{t}"}.join(' ')
         when String
           "--tag #{tags}"
         when nil
@@ -44,17 +54,17 @@ module Tes::Request::RSpec
         raise(ArgumentError, "Not supported file type:#{file_type}!")
       end
 
-      job_configs_for_ci = jobs.map { |j| gen_job_ci_params(j) }
+      job_configs_for_ci = jobs.map {|j| gen_job_ci_params(j)}
       FileUtils.rm_rf(target_dir)
       FileUtils.mkdir(target_dir)
       case file_type
         when :json
           save_file = File.join(target_dir, 'ci_tasks.json')
-          File.open(save_file, 'w') { |f| f.write job_configs_for_ci.to_json }
+          File.open(save_file, 'w') {|f| f.write job_configs_for_ci.to_json}
           puts "Generated #{jobs.size} jobs, Stored in:#{save_file} ."
         when :yml, :yaml
           save_file = File.join(target_dir, 'ci_tasks.yml')
-          File.open(save_file, 'w') { |f| f.write job_configs_for_ci.to_yaml }
+          File.open(save_file, 'w') {|f| f.write job_configs_for_ci.to_yaml}
           puts "Generated #{jobs.size} jobs, Stored in:#{save_file} ."
         when :properties
           job_configs_for_ci.each_with_index do |params, i|
@@ -75,6 +85,7 @@ module Tes::Request::RSpec
       job[:profile].to_s(split)
     end
 
+    private
     def gen_job_ci_params(job)
       {'RSPEC_PARAM' => get_job_rspec_run_args_str(job), 'REQUEST_ASKS' => get_job_env_profile_str(job)}
     end

data/lib/tes/request/rspec/distribute.rb

@@ -24,18 +24,22 @@ module Tes
         attr_reader :project_dir, :ci_cfg
 
         # 生成分发任务的配置结构
+        # @param [String] type task type
+        # @param [Integer] count 分批任务数
+        # @param [Hash] res_addition_attr_map 资源属性需要调整的映射表
+        # @param [Hash,nil] adapt_pool
         # @return [Array<Hash>]
-        def distribute_jobs(type, count, res_addition_attr_map=nil)
+        def distribute_jobs(type, count, res_addition_attr_map={}, adapt_pool = {})
           task_cfg = get_rspec_task(type)
           spec_paths = spec_files(type)
           rspec_parser = Tes::Request::RSpec::ProfileParser.new(spec_paths)
           rspec_parser.parse_profiles!
+          rspec_profiles = rspec_parser.profiles
 
-          gen_pieces(rspec_parser.profiles, count).map do |piece|
-            profile = piece[:profile]
-            if res_addition_attr_map and res_addition_attr_map.size > 0
+          if res_addition_attr_map and res_addition_attr_map.size > 0
+            rspec_profiles.each do |spec|
               res_addition_attr_map.each do |res_filter_pattern, attr_add_map|
-                request_asks = profile.data.select { |ask| ask.to_s.include? res_filter_pattern }
+                request_asks = spec[:profile].data.select {|ask| ask.to_s.include? res_filter_pattern}
                 request_asks.each do |ask|
                   # only add the resource attribution when no request the attribution for the resource
                   attr_add_map.each do |attr_name, attr_limit|
@@ -46,7 +50,19 @@ module Tes
                 end
               end
             end
+          end
 
+
+          if adapt_pool and adapt_pool.size > 0
+            rspec_profiles.delete_if do |spec|
+              pool_not_satisfied = !(spec[:profile].request(adapt_pool).all?)
+              warn "POOL is not satisfied for: #{spec[:file]}" if pool_not_satisfied
+              pool_not_satisfied
+            end
+          end
+
+          gen_pieces(rspec_profiles, count).map do |piece|
+            profile = piece[:profile]
             specs = piece[:specs].inject([]) do |t, spec|
               file_path = spec[:file].sub(/^#{@project_dir}/, '').sub(/^\//, '')
               if (spec[:locations] and spec[:locations].size > 0) or (spec[:ids] and spec[:ids].size > 0)
@@ -63,7 +79,7 @@ module Tes
             end
 
             {profile: profile, specs: specs}
-          end.map { |p| p.merge(tag: task_cfg['tag']) }
+          end.map {|p| p.merge(tag: task_cfg['tag'])}
         end
 
         private
@@ -89,7 +105,6 @@ module Tes
                   piece[:profile] == to_merge_spec[:profile]
             end
             if join_piece
-              # puts 'http://join'
               join_piece[:specs] << to_merge_spec
             else
               # 2. 然后再是资源多少不同的归并
@@ -102,7 +117,6 @@ module Tes
                 end
               end
               if super_piece
-                # puts 'http://inherit'
                 super_piece[:specs] << to_merge_spec
               else
                 # 3. 可整合计算的的归并,但要求已经达到的任务分片数已经达到了要求那么大,否则直接以新建来搞
@@ -112,13 +126,10 @@ module Tes
                         piece[:profile].merge_able?(to_merge_spec[:profile])
                   end
                   if merge_piece
-                    # puts 'http://merge'
                     merge_piece[:profile] = merge_piece[:profile] + to_merge_spec[:profile]
-                    # puts merge_piece[:profile]
                     merge_piece[:specs] << to_merge_spec
                   else
                     # 4. 最后再尝试独立出一个新的piece,在剩余数量达到一半要求的时候
-                    # puts 'http://new'
                     common_jobs << {profile: to_merge_spec[:profile], specs: [to_merge_spec]}
                   end
                 else
@@ -165,7 +176,7 @@ module Tes
                     pattern_filter_lab.call(pattern)
                     Dir[File.join(@project_dir, pattern)]
                   when Array
-                    pattern.inject([]) { |t, ep| t + pattern_filter_lab.call(ep) }
+                    pattern.inject([]) {|t, ep| t + pattern_filter_lab.call(ep)}
                   else
                     raise('Error pattern type')
                 end
@@ -176,7 +187,7 @@ module Tes
             when String
               ret -= Dir[File.join(@project_dir, exclude_pattern)]
             when Array
-              ret -= exclude_pattern.inject([]) { |t, ep| t + Dir[File.join(@project_dir, ep)] }
+              ret -= exclude_pattern.inject([]) {|t, ep| t + Dir[File.join(@project_dir, ep)]}
             else
               raise('Error exclude_pattern type')
           end
@@ -206,7 +217,7 @@ module Tes
                             exclude_pattern
                           end
               res_attrs = res_attrs.split(',')
-              profile_lines.any? { |line| res_attrs.all? { |attr| line =~ /\b#{attr}\b/ } }
+              profile_lines.any? {|line| res_attrs.all? {|attr| line =~ /\b#{attr}\b/}}
             end
           end
         end

data/lib/tes/request/version.rb

@@ -1,5 +1,5 @@
 module Tes
   module Request
-    VERSION = '0.8'
+    VERSION = '0.9'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tes-request
 version: !ruby/object:Gem::Version
-  version: '0.8'
+  version: '0.9'
 platform: ruby
 authors:
 - wuhuizuo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-14 00:00:00.000000000 Z
+date: 2018-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: java-properties
@@ -84,6 +84,7 @@ description: Provide Tes-Client, Tes-Provider to use public Logic and data struc
 email:
 - wuhuizuo@126.com
 executables:
+- tes-ci-pool
 - tes-ci-slicer
 - tes-client
 extensions: []
@@ -97,6 +98,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/tes-ci-pool
 - bin/tes-ci-slicer
 - bin/tes-client
 - lib/hash.rb
@@ -105,6 +107,7 @@ files:
 - lib/tes/request/client.rb
 - lib/tes/request/expression.rb
 - lib/tes/request/profile.rb
+- lib/tes/request/rspec/ci_pool.rb
 - lib/tes/request/rspec/ci_slicer.rb
 - lib/tes/request/rspec/distribute.rb
 - lib/tes/request/rspec/function.rb
@@ -132,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.7.4
 signing_key: 
 specification_version: 4
 summary: Request libs and bins for TES(Test Env Service)