tes-request 0.1

data/lib/tes/request/rspec/ci_slicer.rb

@@ -0,0 +1,82 @@
+require 'yaml'
+require 'java-properties'
+require 'fileutils'
+require_relative 'distribute'
+
+module Tes::Request::RSpec
+  class CiSlicer
+    SUPPORT_FILE_TYPES = [:yaml, :yml, :json, :properties]
+
+    def initialize(file_type, project_dir, res_replace_map_json_file=nil)
+      unless SUPPORT_FILE_TYPES.include?(file_type.to_sym)
+        raise(ArgumentError, "Not supported file type:#{file_type}!")
+      end
+
+      @cfg_file_type = file_type
+      @project_dir = project_dir
+      @res_addition_attr_map = (res_replace_map_json_file && JSON.parse(File.read(res_replace_map_json_file)))
+      @cfg_target_dir = File.join(@project_dir, '.ci_jobs')
+    end
+
+    def run(ci_type, slice_count)
+      puts "Generate RSpec distribute jobs #{@cfg_file_type} file for CI"
+      rspec_distribute = Distribute.new(@project_dir)
+      jobs = rspec_distribute.distribute_jobs(ci_type, slice_count, @res_addition_attr_map)
+      save_job_files(jobs, @cfg_target_dir, @cfg_file_type)
+      @cfg_target_dir
+    end
+
+    def spec_tag_param_str(tags)
+      case tags
+        when Array
+          tags.map { |t| "--tag #{t}" }.join(' ')
+        when String
+          "--tag #{tags}"
+        when nil
+          nil
+        else
+          raise("不支持的类型:#{tags.class}")
+      end
+    end
+
+    def save_job_files(jobs, target_dir, file_type)
+      unless SUPPORT_FILE_TYPES.include?(file_type)
+        raise(ArgumentError, "Not supported file type:#{file_type}!")
+      end
+
+      job_configs_for_ci = jobs.map { |j| gen_job_ci_params(j) }
+      FileUtils.rm_rf(target_dir)
+      FileUtils.mkdir(target_dir)
+      case file_type
+        when :json
+          save_file = File.join(target_dir, 'ci_tasks.json')
+          File.open(save_file, 'w') { |f| f.write job_configs_for_ci.to_json }
+          puts "Generated #{jobs.size} jobs, Stored in:#{save_file} ."
+        when :yml, :yaml
+          save_file = File.join(target_dir, 'ci_tasks.yml')
+          File.open(save_file, 'w') { |f| f.write job_configs_for_ci.to_yaml }
+          puts "Generated #{jobs.size} jobs, Stored in:#{save_file} ."
+        when :properties
+          job_configs_for_ci.each_with_index do |params, i|
+            file = File.join(target_dir, "#{i}.properties")
+            JavaProperties.write(params, file)
+          end
+          puts "Generated #{jobs.size} jobs, Stored in:#{target_dir}/*.properties ."
+      end
+    end
+
+    def get_job_rspec_run_args_str(job, split=' ')
+      tags_str = spec_tag_param_str(job[:tag])
+      paths_str = job[:specs].join(split)
+      tags_str ? (tags_str + split + paths_str) : paths_str
+    end
+
+    def get_job_env_profile_str(job, split=';')
+      job[:profile].to_s(split)
+    end
+
+    def gen_job_ci_params(job)
+      {'RSPEC_PARAM' => get_job_rspec_run_args_str(job), 'REQUEST_ASKS' => get_job_env_profile_str(job)}
+    end
+  end
+end

data/lib/tes/request/rspec/distribute.rb

@@ -0,0 +1,214 @@
+require 'yaml'
+require 'json'
+require_relative 'profile_parser'
+
+module Tes
+  module Request
+    module RSpec
+      class Distribute
+        include Function
+
+        DEFAULT_CI_YAML_FILE = '.ci.yaml'
+        EXCLUDE_CLUSTER_RES_PATTERN_PROFILE = '.ci_exclude_res_pattern'
+
+        @@ci_yaml_file = DEFAULT_CI_YAML_FILE
+        @@ci_exclude_cluster_pattern_file = EXCLUDE_CLUSTER_RES_PATTERN_PROFILE
+
+        # @param [String] project_dir 测试项目的根目录路径
+        # @param [String] ci_yaml_file 测试项目内的描述spec测试的配置文件路径(相对`project_dir`)
+        def initialize(project_dir, ci_yaml_file=@@ci_yaml_file)
+          @project_dir = project_dir
+          @ci_cfg = YAML.load_file(File.join(@project_dir, ci_yaml_file))
+        end
+
+        attr_reader :project_dir, :ci_cfg
+
+        # 生成分发任务的配置结构
+        # @return [Array<Hash>]
+        def distribute_jobs(type, count, res_addition_attr_map=nil)
+          task_cfg = get_rspec_task(type)
+          spec_paths = spec_files(type)
+          rspec_parser = Tes::Request::RSpec::ProfileParser.new(spec_paths)
+          rspec_parser.parse_profiles!
+
+          gen_pieces(rspec_parser.profiles, count).map do |piece|
+            profile = piece[:profile]
+            if res_addition_attr_map and res_addition_attr_map.size > 0
+              res_addition_attr_map.each do |res_filter_pattern, attr_add_map|
+                request_asks = profile.data.select { |ask| ask.to_s.include? res_filter_pattern }
+                request_asks.each do |ask|
+                  # only add the resource attribution when no request the attribution for the resource
+                  attr_add_map.each do |attr_name, attr_limit|
+                    unless ask.data.include?(attr_name)
+                      ask.data[attr_name] = Tes::Request::Expression.new("#{attr_name}#{attr_limit}")
+                    end
+                  end
+                end
+              end
+            end
+
+            specs = piece[:specs].inject([]) do |t, spec|
+              file_path = spec[:file].sub(/^#{@project_dir}/, '').sub(/^\//, '')
+              if (spec[:locations] and spec[:locations].size > 0) or (spec[:ids] and spec[:ids].size > 0)
+                if spec[:locations] and spec[:locations].size > 0
+                  t.push(file_path + ':' + spec[:locations].join(':'))
+                end
+                if spec[:ids] and spec[:ids].size > 0
+                  t.push(file_path + '[' + spec[:ids].join(',') + ']')
+                end
+              else
+                t.push file_path
+              end
+              t
+            end
+
+            {profile: profile, specs: specs}
+          end.map { |p| p.merge(tag: task_cfg['tag']) }
+        end
+
+        private
+        # 生产任务碎片,尽量接近传递的参数值
+        # @param [Fixnum] minimum_pieces
+        # @return [Array]
+        def gen_pieces(profiles, minimum_pieces)
+          ret = []
+          min_spec_count = profiles.size / minimum_pieces
+
+          profiles.each do |to_merge_spec|
+            # 0. 任务发布要求的特殊处理
+            if to_merge_spec[:distribute] && to_merge_spec[:distribute][:standalone]
+              ret << {profile: to_merge_spec[:profile], specs: [to_merge_spec]}
+              next
+            end
+
+            # 1. 优先相同要求的归并
+            join_piece = ret.find do |piece|
+              piece[:specs].size <= min_spec_count and
+                  piece[:profile] == to_merge_spec[:profile]
+            end
+            if join_piece
+              # puts 'http://join'
+              join_piece[:specs] << to_merge_spec
+            else
+              # 2. 然后再是资源多少不同的归并
+              super_piece = ret.find do |piece|
+                if piece[:specs].size <= min_spec_count
+                  cr = piece[:profile] <=> to_merge_spec[:profile]
+                  cr && cr >= 0
+                else
+                  false
+                end
+              end
+              if super_piece
+                # puts 'http://inherit'
+                super_piece[:specs] << to_merge_spec
+              else
+                # 3. 可整合计算的的归并,但要求已经达到的任务分片数已经达到了要求那么大,否则直接以新建来搞
+                if ret.size >= minimum_pieces
+                  merge_piece = ret.find do |piece|
+                    piece[:specs].size <= min_spec_count and
+                        piece[:profile].merge_able?(to_merge_spec[:profile])
+                  end
+                  if merge_piece
+                    # puts 'http://merge'
+                    merge_piece[:profile] = merge_piece[:profile] + to_merge_spec[:profile]
+                    # puts merge_piece[:profile]
+                    merge_piece[:specs] << to_merge_spec
+                  else
+                    # 4. 最后再尝试独立出一个新的piece,在剩余数量达到一半要求的时候
+                    # puts 'http://new'
+                    ret << {profile: to_merge_spec[:profile], specs: [to_merge_spec]}
+                  end
+                else
+                  ret << {profile: to_merge_spec[:profile], specs: [to_merge_spec]}
+                end
+              end
+            end
+          end
+          ret
+        end
+
+
+        # @param [String] type
+        # @return [Hash] rspec ci task info
+        def get_rspec_task(type)
+          raise("No CI Task:#{type}") unless @ci_cfg.key?(type)
+          @ci_cfg[type]
+        end
+
+        # @return [Array<String>]
+        def spec_files(task_type)
+          spec_cfg = get_rspec_task(task_type)['spec']
+          spec_paths = filter_spec_by_path(spec_cfg['pattern'], spec_cfg['exclude_pattern'])
+
+          exclude_cluster_profile_file = File.join(@project_dir, @@ci_exclude_cluster_pattern_file)
+          if File.exists?(exclude_cluster_profile_file)
+            exclude_patterns = File.readlines(exclude_cluster_profile_file).map(&:strip)
+            exclude_spec_by_resource(spec_paths, exclude_patterns)
+          else
+            spec_paths
+          end
+        end
+
+        # @return [Array<String>]
+        def filter_spec_by_path(pattern, exclude_pattern=nil)
+          pattern_filter_lab = ->(p) do
+            spec_info = get_spec_path_info(p)
+            direct_return = (spec_info[:locations] or spec_info[:ids])
+            direct_return ? [File.join(@project_dir, p)] : Dir[File.join(@project_dir, p)]
+          end
+
+          ret = case pattern
+                  when String
+                    pattern_filter_lab.call(pattern)
+                    Dir[File.join(@project_dir, pattern)]
+                  when Array
+                    pattern.inject([]) { |t, ep| t + pattern_filter_lab.call(ep) }
+                  else
+                    raise('Error pattern type')
+                end
+
+          return ret unless exclude_pattern
+
+          case exclude_pattern
+            when String
+              ret -= Dir[File.join(@project_dir, exclude_pattern)]
+            when Array
+              ret -= exclude_pattern.inject([]) { |t, ep| t + Dir[File.join(@project_dir, ep)] }
+            else
+              raise('Error exclude_pattern type')
+          end
+
+          ret
+        end
+
+        # 按照指定资源属性排除要求排除相应的spec路径
+        # @param [Array<String>] spec_paths spec的执行路径列表
+        # @param [Array<String>] res_exclude_patterns,
+        #   每一个元素的格式是这样:
+        #     res_type: res_attr1=2,res_attr3>=4
+        #   或者
+        #     type=res_type,res_attr1=2,res_attr3>=4
+        # @return [Array<String>] 按照`res_exclude_pattern` 剔除后的 `spec_paths`
+        def exclude_spec_by_resource(spec_paths, res_exclude_patterns=[])
+          return spec_paths if res_exclude_patterns.empty?
+
+          spec_paths.reject do |spec_path|
+            spec_info = get_spec_path_info(spec_path)
+            profile_lines = parse_spec_profile_lines(spec_info[:file])
+            res_exclude_patterns.any? do |exclude_pattern|
+              res_attrs = if exclude_pattern =~ /^\w+:\s*.+/
+                            type, attrs = exclude_pattern.split(/:\s*/, 2)
+                            "type=#{type}," + attrs
+                          else
+                            exclude_pattern
+                          end
+              res_attrs = res_attrs.split(',')
+              profile_lines.any? { |line| res_attrs.all? { |attr| line =~ /\b#{attr}\b/ } }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tes/request/rspec/function.rb

@@ -0,0 +1,48 @@
+module Tes
+  module Request
+    module RSpec
+      module Function
+        # 对spec路径获取相关信息(行号或者序号等信息)
+        # @param [String] spec_path
+        # @param [Hash]
+        def get_spec_path_info(spec_path)
+          ids_reg = /\[([\d:,]+)\]$/
+          locations_reg = /:([\d:]+)$/
+          if spec_path =~ ids_reg
+            ids = spec_path.match(ids_reg)[1].split(',')
+            file_path = spec_path.sub(ids_reg, '')
+            {file: file_path, ids: ids}
+          elsif spec_path =~ locations_reg
+            locations = spec_path.match(locations_reg)[1].split(':').map(&:to_i)
+            file_path = spec_path.sub(locations_reg, '')
+            {file: file_path, locations: locations}
+          else
+            {file: spec_path}
+          end
+        end
+
+        # 解析spec文件的测试环境要求文本内容
+        # @return [Array<String>]
+        def parse_spec_profile_lines(spec_file)
+          ret = nil
+          File.open(spec_file, 'r') do |f|
+            f.each_line do |l|
+              case l
+                when /^\s*#\s*@env\s+begin\s*$/
+                  ret = []
+                when /^\s*#\s*@end/
+                  break
+                when /^\s*#/
+                  ret << l.sub(/^\s*#\s*/, '') if ret
+                else
+                  #nothing
+              end
+            end
+          end
+
+          ret && ret.map(&:strip).map { |l| l.split(/\s*;\s*/) }.flatten
+        end
+      end
+    end
+  end
+end

data/lib/tes/request/rspec/profile_parser.rb

@@ -0,0 +1,132 @@
+require_relative '../profile'
+require_relative 'function'
+
+# 分析rspec profile信息
+module Tes
+  module Request
+    module RSpec
+      class ProfileParser
+        def initialize(spec_paths=[])
+          @spec_paths = spec_paths
+        end
+
+        attr_reader :profiles
+
+        def <<(spec_path)
+          @spec_paths << spec_path
+        end
+
+        def parse_profiles!
+          profile_map = {}
+
+          @spec_paths.each do |s_p|
+            spec_info = parse_spec(s_p)
+
+            # 如果相同文件则进行合并
+            if profile_map.key?(spec_info[:file])
+              profile_map[spec_info[:file]] = merge_spec_info(profile_map[spec_info[:file]], spec_info)
+            else
+              profile_map[spec_info[:file]] = spec_info
+            end
+          end
+
+          # 对存在的包含关系的ids尝试合并
+          profile_map.each do |_f, info|
+            info[:ids] = merge_spec_ids(info[:ids]) if info[:ids]
+            info[:locations].sort! if info[:locations]
+          end
+          @profiles = profile_map.values
+        end
+
+        private
+
+        include Function
+
+        # 合并2个spec info数据结构
+        # @param [Hash] a
+        # @param [Hash] b
+        # @return [Hash] merge result, same struct with `a` or `b`
+        def merge_spec_info(a, b)
+          raise('不支持合并不同spec文件的信息') unless a[:file] == b[:file]
+
+          compare_keys = [:ids, :locations]
+          a_compare_keys = a.keys.select { |k| compare_keys.include?(k) }.sort
+          b_compare_keys = b.keys.select { |k| compare_keys.include?(k) }.sort
+
+          case [a_compare_keys, b_compare_keys]
+            # 都有ids的情况
+            when [[:ids], [:ids]], [[:ids, :locations], [:ids]]
+              a.merge(ids: (a[:ids] + b[:ids]).uniq)
+            when [[:ids], [:ids, :locations]]
+              b.merge(ids: (a[:ids] + b[:ids]).uniq)
+
+            # 都有locations的情况
+            when [[:locations], [:locations]], [[:ids, :locations], [:locations]]
+              a.merge(locations: (a[:locations] + b[:locations]).uniq)
+            when [[:locations], [:ids, :locations]]
+              a.merge(locations: (b[:locations] + a[:locations]).uniq)
+
+            # 都有ids和locations
+            when [[:ids, :locations], [:ids, :locations]]
+              a.merge(
+                  ids: (a[:ids] + b[:ids]).uniq,
+                  locations: (a[:locations] + b[:locations]).uniq
+              )
+
+            # 互补
+            when [[:ids], [:locations]], [[:locations], [:ids]]
+              a.merge b
+            else
+              # 只剩下 a_compare_keys 为空 或者 b_compare_keys为空的情况
+              a_compare_keys.empty? ? a : b
+          end
+        end
+
+        # @param [Array<String>] ids
+        def merge_spec_ids(ids)
+          ids.sort.inject([]) do |t, id|
+            id_is_covered = (t.last && id.index(t.last) == 0)
+            id_is_covered ? t : t.push(id)
+          end
+        end
+
+        # @param [Array<String>] str_array, 每个元素的格式可以是这样:
+        #   - `--xxx`
+        #   - `-yy`
+        #   - `-x vvv`
+        #   - `--zz vvv`
+        # @return [Hash<Symbol, Object>]
+        def parse_distribute_profile(str_array)
+          str_array.inject({}) do |d_p, str|
+            args = str.scan(/-+[^-]+\b/).map { |arg| arg.sub(/^-+/, '').split(/\s+/) }
+            d_p_phrase = args.inject({}) { |h, arg| h.merge(arg[0].to_sym => arg[1] || true) }
+            d_p.merge d_p_phrase
+          end
+        end
+
+        # 解析指定路径spec的执行解析
+        # @param [String] spec_path 当前支持3种格式
+        #   - 一种是普通的文件路径格式
+        #   - 一种是文件格式基础上增加行数指定的格式,单个或者多个,单个格式 `:123`,多个直接拼接起来
+        #   - 一种是文件格式基础上增加用例层次路径的格式,单个或者多个,单个格式`[1:2:3]`,多个在括号内用逗号间隔:`[1:2:3,2:3:4]`
+        # @return [Hash] include keys: `:file`, `:profile`, `:distribute`(optional)
+        def parse_spec(spec_path)
+          spec_info = get_spec_path_info spec_path
+
+          profile_lines = parse_spec_profile_lines(spec_info[:file])
+          raise("#{spec_info[:file]} 没有ci profile配置,请确认!") unless profile_lines and profile_lines.size > 0
+
+          distribute_lines = profile_lines.
+              select { |l| l =~ /^\s*@distribute\s+/ }.
+              map { |l| l.sub(/^\s*@distribute\s+/, '') }
+          profile_lines.reject! { |l| l =~ /^\s*@distribute\s+/ }
+
+          spec_info.merge(
+              profile: Profile.new(profile_lines),
+              distribute: parse_distribute_profile(distribute_lines)
+          )
+        end
+      end
+    end
+  end
+end