at_coder_friends 0.5.0 → 0.5.1

data/lib/at_coder_friends/parser/page_parser.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module AtCoderFriends
+  module Parser
+    # parses problem page and collect problem information
+    module PageParser
+      module_function
+
+      SECTION_TYPES = [
+        {
+          key: 'constraints',
+          patterns: [
+            '^制約$',
+            '^Constraints$'
+          ]
+        },
+        {
+          key: 'input format',
+          patterns: [
+            '^入出?力(形式)?$',
+            '^Inputs?\s*(,|and)?\s*(Outputs?)?\s*(Format)?$'
+          ]
+        },
+        {
+          key: 'sample input %<no>s',
+          patterns: [
+            '^入力例\s*(?<no>\d+)?$',
+            '^入力\s*(?<no>\d+)$',
+            '^Sample\s*Input\s*(?<no>\d+)?$',
+            '^Input\s*Example\s*(?<no>\d+)?$',
+            '^Input\s*(?<no>\d+)$'
+          ]
+        },
+        {
+          key: 'sample output %<no>s',
+          patterns: [
+            '^出力例\s*(?<no>\d+)?$',
+            '^出力\s*(?<no>\d+)$',
+            '^入力例\s*(?<no>\d+)?\s*に対する出力例$',
+            '^Sample\s*Output\s*(?<no>\d+)?$',
+            '^Output\s*Example\s*(?<no>\d+)?$',
+            '^Output\s*(?<no>\d+)$',
+            '^Output\s*for\s*(the)?\s*Sample\s*Input\s*(?<no>\d+)?$'
+          ]
+        }
+      ].freeze
+
+      def process(pbm)
+        sections = collect_sections(pbm.page)
+        apply_sections(pbm, sections)
+      end
+
+      def collect_sections(page)
+        sections = {}
+        %w[h2 h3].each do |tag|
+          page
+            .search(tag)
+            .each do |h|
+              key = find_key(h)
+              key && sections[key] ||= parse_section(h)
+            end
+        end
+        sections
+      end
+
+      def find_key(h)
+        title = normalize(h.content)
+        SECTION_TYPES.each do |grp|
+          grp[:patterns].each do |pat|
+            m = title.match(/#{pat}/i)
+            next unless m
+
+            no = m.names.include?('no') && m['no'] || '1'
+            return format(grp[:key], no: no)
+          end
+        end
+        nil
+      end
+
+      def parse_section(h)
+        text = ''
+        pre = nil
+        nx = h.next
+        while nx && nx.name != h.name
+          text += nx.content.gsub("\r\n", "\n")
+          %w[pre blockquote].each do |tag|
+            pre ||= (nx.name == tag ? nx : nx.search(tag)[0])
+          end
+          nx = nx.next
+        end
+        code = (pre&.text || '').lstrip.gsub("\r\n", "\n")
+        [text, code]
+      end
+
+      def apply_sections(pbm, sections)
+        sections.each do |key, (text, code)|
+          case key
+          when 'constraints'
+            pbm.desc += text
+          when 'input format'
+            pbm.desc += text
+            pbm.fmt = code
+          when /^sample input (?<no>\d+)$/
+            pbm.add_smp($LAST_MATCH_INFO[:no], :in, code)
+          when /^sample output (?<no>\d+)$/
+            pbm.add_smp($LAST_MATCH_INFO[:no], :exp, code)
+          end
+        end
+      end
+
+      def normalize(s)
+        s
+          .tr('　０-９Ａ-Ｚａ-ｚ', ' 0-9A-Za-z')
+          .gsub(/[^一-龠_ぁ-ん_ァ-ヶーa-zA-Z0-9 ]/, '')
+          .strip
+      end
+    end
+  end
+end

data/lib/at_coder_friends/path_util.rb

@@ -7,6 +7,7 @@ module AtCoderFriends
 
     SMP_DIR = 'data'
     CASES_DIR = 'cases'
+    TMP_DIR = '.tmp'
 
     def contest_name(path)
       dir = File.file?(path) ? File.dirname(path) : path
@@ -27,5 +28,14 @@ module AtCoderFriends
     def cases_dir(dir)
       File.join(dir, CASES_DIR)
     end
+
+    def tmp_dir(path)
+      dir = File.dirname(path)
+      File.join(dir, '.tmp')
+    end
+
+    def makedirs_unless(dir)
+      FileUtils.makedirs(dir) unless Dir.exist?(dir)
+    end
   end
 end

data/lib/at_coder_friends/problem.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 
 module AtCoderFriends
-  DataSample = Struct.new(:no, :ext, :txt) do
+  SampleData = Struct.new(:no, :ext, :txt) do
     def initialize(no, ext, txt)
-      no = no.tr('０-９', '0-9').to_i
-      txt = txt.lstrip.gsub("\r\n", "\n")
-      super(no, ext, txt)
+      super(no.to_i, ext, txt)
     end
   end
 
@@ -15,34 +13,33 @@ module AtCoderFriends
     end
   end
 
-  SourceSample = Struct.new(:ext, :txt)
+  Constraint = Struct.new(:name, :type, :value)
+
+  SourceCode = Struct.new(:ext, :txt)
 
   # holds problem information
   class Problem
-    attr_reader :q, :fmt, :smps, :srcs
-    attr_accessor :html, :desc, :defs
+    attr_reader :q, :smps, :srcs
+    attr_accessor :page, :desc, :fmt, :defs, :constraints
 
     def initialize(q)
       @q = q
-      @html = ''
+      @page = nil
       @desc = ''
       @fmt = ''
       @smps = []
       @defs = []
+      @constraints = []
       @srcs = []
       yield self if block_given?
     end
 
-    def fmt=(f)
-      @fmt = f.lstrip.gsub("\r\n", "\n")
-    end
-
     def add_smp(no, ext, txt)
-      @smps << DataSample.new(no, ext, txt)
+      @smps << SampleData.new(no, ext, txt)
     end
 
     def add_src(ext, txt)
-      @srcs << SourceSample.new(ext, txt)
+      @srcs << SourceCode.new(ext, txt)
     end
   end
 end

data/lib/at_coder_friends/scraping/agent.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'mechanize'
+require 'logger'
+
+module AtCoderFriends
+  module Scraping
+    # common functions for scraping
+    class Agent
+      include AtCoderFriends::PathUtil
+      include Session
+      include Authentication
+      include Tasks
+      include CustomTest
+      include Submission
+
+      BASE_URL = 'https://atcoder.jp/'
+
+      attr_reader :ctx, :agent
+
+      def initialize(ctx)
+        @ctx = ctx
+        @agent = Mechanize.new
+        agent.pre_connect_hooks << proc { sleep 0.1 }
+        agent.log = Logger.new(STDERR) if ctx.options[:debug]
+        load_session
+      end
+
+      def contest
+        @contest ||= contest_name(ctx.path)
+      end
+
+      def config
+        ctx.config
+      end
+
+      def common_url(path)
+        File.join(BASE_URL, path)
+      end
+
+      def contest_url(path = '')
+        File.join(BASE_URL, 'contests', contest, path)
+      end
+
+      def lang_id(ext)
+        config.dig('ext_settings', ext, 'submit_lang') || (
+          msg = <<~MSG
+            submit_lang for .#{ext} is not specified.
+            Available languages:
+            #{lang_list_txt || '(failed to fetch)'}
+          MSG
+          raise AppError, msg
+        )
+      end
+
+      def lang_list_txt
+        lang_list
+          &.map { |opt| "#{opt[:v]} - #{opt[:t]}" }
+          &.join("\n")
+      end
+
+      def lang_list
+        @lang_list ||= begin
+          page = fetch_with_auth(contest_url('custom_test'))
+          form = page.forms[1]
+          sel = form.field_with(name: 'data.LanguageId')
+          sel && sel
+            .options
+            .reject { |opt| opt.value.empty? }
+            .map do |opt|
+              { v: opt.value, t: opt.text }
+            end
+        end
+      end
+    end
+  end
+end

data/lib/at_coder_friends/scraping/authentication.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require 'io/console'
+
+module AtCoderFriends
+  module Scraping
+    # fetch pages and
+    # authenticates user at login page if needed
+    module Authentication
+      XPATH_USERNAME = '//*[@id="navbar-collapse"]/ul[2]/li[2]/a'
+
+      def fetch_with_auth(url)
+        page = fetch_raw(url)
+        page.uri.path == '/login' && page = post_login(page)
+        page.uri.path == '/login' && (raise AppError, 'Authentication failed.')
+        show_username(page)
+        page
+      end
+
+      def fetch_raw(url)
+        begin
+          return agent.get(url)
+        rescue Mechanize::ResponseCodeError => e
+          raise e unless e.response_code == '404'
+          raise e if username_link(e.page)
+        end
+
+        agent.get(common_url('login') + '?continue=' + CGI.escape(url))
+      end
+
+      def post_login(page)
+        user, pass = read_auth
+        form = page.forms[1]
+        form.field_with(name: 'username').value = user
+        form.field_with(name: 'password').value = pass
+        form.submit
+      end
+
+      def read_auth
+        user = config['user'].to_s
+        if user.empty?
+          print('Enter username:')
+          user = STDIN.gets.chomp
+        end
+
+        pass = config['password'].to_s
+        if pass.empty?
+          print("Enter password for #{user}:")
+          pass = STDIN.noecho(&:gets).chomp
+          puts
+        end
+        [user, pass]
+      end
+
+      def show_username(page)
+        username_bak = @username
+        link = username_link(page)
+        @username = (link ? link.text.strip : '-')
+        return if @username == username_bak || @username == '-'
+
+        puts "Logged in as #{@username}"
+      end
+
+      def username_link(page)
+        link = page.search(XPATH_USERNAME)[0]
+        link && link[:href] == '#' && link
+      end
+    end
+  end
+end

data/lib/at_coder_friends/scraping/custom_test.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module AtCoderFriends
+  module Scraping
+    # run tests on custom_test page
+    module CustomTest
+      include AtCoderFriends::PathUtil
+
+      def code_test(infile)
+        path, _dir, _prg, _base, ext, _q = split_prg_path(ctx.path)
+        lang = lang_id(ext)
+        src = File.read(path, encoding: Encoding::UTF_8)
+        data = File.read(infile)
+
+        post_custom_test(lang, src, data)
+        check_custom_test
+      end
+
+      def post_custom_test(lang, src, data)
+        page = fetch_with_auth(contest_url('custom_test'))
+        script = page.search('script').text
+        csrf_token = script.scan(/var csrfToken = "(.*)"/)[0][0]
+
+        page = agent.post(
+          contest_url('custom_test/submit/json'),
+          'data.LanguageId' => lang,
+          'sourceCode' => src,
+          'input' => data,
+          'csrf_token' => csrf_token
+        )
+
+        msg = page.body
+        raise AppError, msg unless msg.empty?
+      end
+
+      def check_custom_test
+        100.times do
+          page = agent.get(contest_url('custom_test/json?reload=true'))
+          data = JSON.parse(page.body)
+          return nil unless data.is_a?(Hash) && data['Result']
+          return data if data.dig('Result', 'Status') == 3
+          return data unless data['Interval']
+
+          sleep 1.0 * data['Interval'] / 1000
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/at_coder_friends/scraping/session.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module AtCoderFriends
+  module Scraping
+    # session management for scraping
+    module Session
+      SESSION_STORE_FMT = File.join(
+        Dir.home, '.at_coder_friends', '%<user>s_session.yml'
+      )
+
+      def load_session
+        agent.cookie_jar.load(session_store) if File.exist?(session_store)
+      end
+
+      def save_session
+        dir = File.dirname(session_store)
+        Dir.mkdir(dir) unless Dir.exist?(dir)
+        agent.cookie_jar.save_as(session_store)
+      end
+
+      def session_store
+        @session_store ||= format(SESSION_STORE_FMT, user: config['user'])
+      end
+    end
+  end
+end

data/lib/at_coder_friends/scraping/submission.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module AtCoderFriends
+  module Scraping
+    # submit sources on submit page
+    module Submission
+      include AtCoderFriends::PathUtil
+
+      def submit
+        path, _dir, prg, _base, ext, q = split_prg_path(ctx.path)
+        puts "***** submit #{prg} *****"
+        lang = lang_id(ext)
+        src = File.read(path, encoding: Encoding::UTF_8)
+
+        post_submit(q, lang, src)
+      end
+
+      def post_submit(q, lang, src)
+        page = fetch_with_auth(contest_url('submit'))
+        form = page.forms[1]
+        form.field_with(name: 'data.TaskScreenName') do |sel|
+          option = sel.options.find { |op| op.text.start_with?(q) }
+          option&.select || (raise AppError, "unknown problem:#{q}.")
+        end
+        form.add_field!('data.LanguageId', lang)
+        form.field_with(name: 'sourceCode').value = src
+        form.submit
+      end
+    end
+  end
+end