at_coder_friends 0.5.0 → 0.5.1

data/lib/at_coder_friends/scraping/tasks.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'English'
+
+module AtCoderFriends
+  module Scraping
+    # fetch problems from tasks page
+    module Tasks
+      def fetch_all
+        puts "***** fetch_all #{contest} *****"
+        fetch_assignments.map do |q, url|
+          pbm = fetch_problem(q, url)
+          yield pbm if block_given?
+          pbm
+        end
+      end
+
+      def fetch_assignments
+        url = contest_url('tasks')
+        puts "fetch list from #{url} ..."
+        page = fetch_with_auth(url)
+        page
+          .search('//table[1]//td[1]//a')
+          .each_with_object({}) do |a, h|
+            h[a.text] = a[:href]
+          end
+      end
+
+      def fetch_problem(q, url)
+        puts "fetch problem from #{url} ..."
+        page = fetch_with_auth(url)
+        Problem.new(q) do |pbm|
+          page.search('br').each { |br| br.replace("\n") }
+          pbm.page = page
+        end
+      end
+    end
+  end
+end

data/lib/at_coder_friends/test_runner/base.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'colorize'
+require 'rbconfig'
+
+module AtCoderFriends
+  module TestRunner
+    # run tests for the specified program.
+    class Base
+      include PathUtil
+
+      attr_reader :ctx, :path, :dir, :prg, :base, :ext, :q
+
+      def initialize(ctx)
+        @ctx = ctx
+        @path, @dir, @prg, @base, @ext, @q = split_prg_path(ctx.path)
+      end
+
+      def config
+        ctx.config
+      end
+
+      def test_cmd
+        @test_cmd ||= begin
+          cmds = config.dig('ext_settings', ext, 'test_cmd')
+          cmd = cmds && (cmds[which_os.to_s] || cmds['default'])
+          return nil unless cmd
+
+          cmd.gsub('{dir}', dir).gsub('{base}', base)
+        end
+      end
+
+      def test_loc
+        test_cmd ? 'local' : 'remote'
+      end
+
+      def test_mtd
+        test_cmd ? :local_test : :remote_test
+      end
+
+      def run_test(id, infile, outfile, expfile)
+        return false unless File.exist?(infile) && File.exist?(expfile)
+
+        puts "==== #{id} ===="
+
+        makedirs_unless(File.dirname(outfile))
+        is_success = send(test_mtd, infile, outfile)
+        show_result(
+          is_success,
+          File.read(infile),
+          File.read(outfile),
+          File.read(expfile)
+        )
+        true
+      end
+
+      def local_test(infile, outfile)
+        system("#{test_cmd} < #{infile} > #{outfile}")
+      end
+
+      def remote_test(infile, outfile)
+        is_success, result = call_remote_test(infile)
+        File.write(outfile, result)
+        is_success
+      end
+
+      def call_remote_test(infile)
+        res = ctx.scraping_agent.code_test(infile)
+        (res && res['Result']) || (return [false, 'Remote test failed.'])
+
+        puts "Exit code: #{res.dig('Result', 'ExitCode')}"
+        puts "Time: #{res.dig('Result', 'TimeConsumption')}ms"
+        puts "Memory: #{res.dig('Result', 'MemoryConsumption')}KB"
+
+        res.dig('Result', 'ExitCode').zero? || (return [false, res['Stderr']])
+        [true, res['Stdout']]
+      end
+
+      def show_result(is_success, input, result, expected)
+        print detail_str(input, result, expected)
+        puts result_str(is_success, result, expected)
+      end
+
+      def detail_str(input, result, expected)
+        ret = ''
+        ret += "-- input --\n"
+        ret += input
+        ret += "-- expected --\n"
+        ret += expected
+        ret += "-- result --\n"
+        ret += result
+        ret
+      end
+
+      def result_str(is_success, result, expected)
+        if !is_success
+          '!!!!! RE !!!!!'.red
+        elsif result != expected
+          '!!!!! WA !!!!!'.red
+        else
+          '<< OK >>'.green
+        end
+      end
+
+      def which_os
+        @which_os ||= begin
+          case RbConfig::CONFIG['host_os']
+          when /mswin|msys|mingw|cygwin|bccwin|wince|emc/
+            :windows
+          when /darwin|mac os/
+            :macosx
+          when /linux/
+            :linux
+          when /solaris|bsd/
+            :unix
+          else
+            :unknown
+          end
+        end
+      end
+    end
+  end
+end

data/lib/at_coder_friends/test_runner/judge.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module AtCoderFriends
+  module TestRunner
+    # run test cases for the specified program with judge input/output.
+    class Judge < Base
+      include PathUtil
+
+      attr_reader :data_dir, :result_dir
+
+      def initialize(ctx)
+        super(ctx)
+        @data_dir = cases_dir(dir)
+        @result_dir = cases_dir(tmp_dir(path))
+      end
+
+      def judge_all
+        puts "***** judge_all #{prg} (#{test_loc}) *****"
+        Dir["#{data_dir}/#{q}/in/*.txt"].sort.each do |infile|
+          id = File.basename(infile, '.txt')
+          judge(id, false)
+        end
+      end
+
+      def judge_one(id)
+        puts "***** judge_one #{prg} (#{test_loc}) *****"
+        judge(id, true)
+      end
+
+      def judge(id, detail = true)
+        @detail = detail
+        infile = "#{data_dir}/#{q}/in/#{id}.txt"
+        outfile = "#{result_dir}/#{q}/result/#{id}.txt"
+        expfile = "#{data_dir}/#{q}/out/#{id}.txt"
+        run_test(id, infile, outfile, expfile)
+      end
+
+      def show_result(is_success, input, result, expected)
+        print detail_str(input, result, expected) if @detail
+        puts result_str(is_success, result, expected)
+      end
+    end
+  end
+end

data/lib/at_coder_friends/test_runner/sample.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module AtCoderFriends
+  module TestRunner
+    # run test cases for the specified program with sample input/output.
+    class Sample < Base
+      include PathUtil
+
+      attr_reader :data_dir
+
+      def initialize(ctx)
+        super(ctx)
+        @data_dir = smp_dir(dir)
+      end
+
+      def test_all
+        puts "***** test_all #{prg} (#{test_loc}) *****"
+        1.upto(999) do |i|
+          break unless test(i)
+        end
+      end
+
+      def test_one(n)
+        puts "***** test_one #{prg} (#{test_loc}) *****"
+        test(n)
+      end
+
+      def test(n)
+        id = format('%<q>s_%<n>03d', q: q, n: n)
+        files = %w[in out exp].map { |ext| "#{data_dir}/#{id}.#{ext}" }
+        run_test(id, *files)
+      end
+    end
+  end
+end

data/lib/at_coder_friends/verifier.rb

@@ -5,19 +5,21 @@ require 'fileutils'
 module AtCoderFriends
   # marks and checks if the source has been verified.
   class Verifier
+    include PathUtil
+
     attr_reader :path, :file, :vdir, :vpath
 
     def initialize(ctx)
       @path = ctx.path
       @file = File.basename(path)
-      @vdir = File.join(File.dirname(path), '.tmp')
+      @vdir = tmp_dir(path)
       @vpath = File.join(vdir, "#{file}.verified")
     end
 
     def verify
       return unless File.exist?(path)
 
-      FileUtils.makedirs(vdir) unless Dir.exist?(vdir)
+      makedirs_unless(vdir)
       FileUtils.touch(vpath)
     end
 

data/lib/at_coder_friends/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AtCoderFriends
-  VERSION = '0.5.0'
+  VERSION = '0.5.1'
 end

data/tasks/regression.rake

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+require 'mechanize'
+require 'at_coder_friends'
+
+module AtCoderFriends
+  # tasks for regression test
+  module Regression
+    module_function
+
+    CONTEST_LIST_URL = 'https://kenkoooo.com/atcoder/resources/contests.json'
+    ACF_HOME = File.realpath(File.join(__dir__, '..'))
+    REGRESSION_HOME = File.join(ACF_HOME, 'regression')
+    PAGES_DIR = File.join(REGRESSION_HOME, 'pages')
+    EMIT_ORG_DIR = File.join(REGRESSION_HOME, 'emit_org')
+    EMIT_DIR_FMT = File.join(REGRESSION_HOME, 'emit_%<now>s')
+
+    def setup
+      rmdir_force(PAGES_DIR)
+      rmdir_force(EMIT_ORG_DIR)
+      contest_id_list.each do |contest|
+        begin
+          ctx = context(EMIT_ORG_DIR, contest)
+          ctx.scraping_agent.fetch_all do |pbm|
+            begin
+              html_path = File.join(PAGES_DIR, contest, "#{pbm.q}.html")
+              save_file(html_path, pbm.page.body)
+              pipeline(ctx, pbm)
+            rescue StandardError => e
+              p e
+            end
+          end
+        rescue StandardError => e
+          p e
+        end
+        sleep 3
+      end
+    end
+
+    def check_diff
+      emit_dir = format(EMIT_DIR_FMT, now: Time.now.strftime('%Y%m%d%H%M%S'))
+      rmdir_force(emit_dir)
+
+      local_pbm_list.each do |contest, q, url|
+        ctx = context(emit_dir, contest)
+        pbm = ctx.scraping_agent.fetch_problem(q, url)
+        pipeline(ctx, pbm)
+      end
+
+      system("diff -r #{EMIT_ORG_DIR} #{emit_dir}")
+    end
+
+    def section_list
+      agent = Mechanize.new
+      list = local_pbm_list.flat_map do |contest, q, url|
+        page = agent.get(url)
+        %w[h2 h3].flat_map do
+          page.search('h3').map do |h3|
+            { contest: contest, q: q, text: normalize(h3.content) }
+          end
+        end
+      end
+      list.group_by { |sec| sec[:text] }.each do |k, vs|
+        puts [k, vs.size, vs[0][:contest], vs[0][:q]].join("\t")
+      end
+    end
+
+    def check_parse
+      parse_list
+        .select { |_, _, no_fmt, no_smp| no_fmt || no_smp }
+        .map { |c, q, no_fmt, no_smp| [c, q, f_to_s(no_fmt), f_to_s(no_smp)] }
+        .sort
+        .each { |args| puts args.join("\t") }
+    end
+
+    def parse_list
+      local_pbm_list.map do |contest, q, url|
+        ctx = context('', contest)
+        pbm = ctx.scraping_agent.fetch_problem(q, url)
+        Parser::Main.process(pbm)
+        no_fmt = pbm.fmt.empty?
+        no_smp = pbm.smps.all? { |smp| smp.txt.empty? }
+        [contest, q, no_fmt, no_smp]
+      end
+    end
+
+    def f_to_s(f)
+      f ? '◯' : '-'
+    end
+
+    def contest_id_list
+      uri = URI.parse(CONTEST_LIST_URL)
+      json = Net::HTTP.get(uri)
+      contests = JSON.parse(json)
+      puts "Total #{contests.size} contests"
+      contests.map { |h| h['id'] }
+    end
+
+    def local_pbm_list
+      Dir.glob(PAGES_DIR + '/**/*.html').map do |pbm_path|
+        contest = File.basename(File.dirname(pbm_path))
+        q = File.basename(pbm_path, '.html')
+        url = "file://#{pbm_path}"
+        [contest, q, url]
+      end
+    end
+
+    def context(root, contest)
+      Context.new({}, File.join(root, contest))
+    end
+
+    def rmdir_force(dir)
+      FileUtils.rm_r(dir) if Dir.exist?(dir)
+    end
+
+    def save_file(path, content)
+      dir = File.dirname(path)
+      FileUtils.makedirs(dir) unless Dir.exist?(dir)
+      File.binwrite(path, content)
+    end
+
+    def pipeline(ctx, pbm)
+      @rb_gen ||= RubyGenerator.new
+      @cxx_gen ||= CxxGenerator.new
+      Parser::Main.process(pbm)
+      @rb_gen.process(pbm)
+      @cxx_gen.process(pbm)
+      ctx.emitter.emit(pbm)
+    end
+
+    def normalize(s)
+      s
+        .tr('　０-９Ａ-Ｚａ-ｚ', ' 0-9A-Za-z')
+        .gsub(/[^一-龠_ぁ-ん_ァ-ヶーa-zA-Z0-9 ]/, '')
+        .strip
+    end
+  end
+end
+
+namespace :regression do
+  desc 'setup regression environment'
+  task :setup do
+    AtCoderFriends::Regression.setup
+  end
+
+  desc 'run regression check'
+  task :check_diff do
+    AtCoderFriends::Regression.check_diff
+  end
+
+  desc 'list all section titles'
+  task :section_list do
+    AtCoderFriends::Regression.section_list
+  end
+
+  desc 'checks page parse result'
+  task :check_parse do
+    AtCoderFriends::Regression.check_parse
+  end
+end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: at_coder_friends
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - nejiko96
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-04 00:00:00.000000000 Z
+date: 2019-10-14 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
@@ -145,16 +159,25 @@ files:
 - lib/at_coder_friends/cxx_generator.rb
 - lib/at_coder_friends/emitter.rb
 - lib/at_coder_friends/errors.rb
-- lib/at_coder_friends/format_parser.rb
-- lib/at_coder_friends/judge_test_runner.rb
+- lib/at_coder_friends/parser/constraints_parser.rb
+- lib/at_coder_friends/parser/format_parser.rb
+- lib/at_coder_friends/parser/main.rb
+- lib/at_coder_friends/parser/page_parser.rb
 - lib/at_coder_friends/path_util.rb
 - lib/at_coder_friends/problem.rb
 - lib/at_coder_friends/ruby_generator.rb
-- lib/at_coder_friends/sample_test_runner.rb
-- lib/at_coder_friends/scraping_agent.rb
-- lib/at_coder_friends/test_runner.rb
+- lib/at_coder_friends/scraping/agent.rb
+- lib/at_coder_friends/scraping/authentication.rb
+- lib/at_coder_friends/scraping/custom_test.rb
+- lib/at_coder_friends/scraping/session.rb
+- lib/at_coder_friends/scraping/submission.rb
+- lib/at_coder_friends/scraping/tasks.rb
+- lib/at_coder_friends/test_runner/base.rb
+- lib/at_coder_friends/test_runner/judge.rb
+- lib/at_coder_friends/test_runner/sample.rb
 - lib/at_coder_friends/verifier.rb
 - lib/at_coder_friends/version.rb
+- tasks/regression.rake
 homepage: https://github.com/nejiko96/at_coder_friends
 licenses:
 - MIT