ohloh_scm 2.5.1 → 3.0.1

data/lib/ohloh_scm/git/scm.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module OhlohScm
+  module Git
+    class Scm < OhlohScm::Scm
+      def initialize(core:, url:, branch_name:, username:, password:)
+        super
+        @branch_name = branch_name || 'master'
+      end
+
+      # == Example:
+      #   remote_core = OhlohScm::Factory.get_core(url: 'https://github.com/ruby/ruby')
+      #   local_core = OhlohScm::Factory.get_core(url: '/tmp/ruby-src')
+      #   local_core.scm.pull(remote_core.scm)
+      def pull(from, callback)
+        case from
+        when Cvs::Scm then convert_to_git(from, callback)
+        else clone_or_fetch(from, callback)
+        end
+      end
+
+      def vcs_path
+        "#{url}/.git"
+      end
+
+      private
+
+      def clone_or_fetch(remote_scm, callback)
+        callback.update(0, 1)
+        if status.branch?(branch_name)
+          clean_and_checkout_branch # must be on correct branch, but we want to be careful.
+          fetch_new_commits(remote_scm)
+        else
+          clone_and_create_tracking_branch(remote_scm)
+        end
+        clean_up_disk
+        callback.update(1, 1)
+      end
+
+      def fetch_new_commits(remote_scm)
+        run "cd '#{url}' && git fetch --tags --update-head-ok "\
+              "'#{remote_scm.url}' #{branch_name}:#{branch_name}"
+      end
+
+      def clone_and_create_tracking_branch(remote_scm)
+        run "rm -rf '#{url}'"
+        run "git clone -q -n '#{remote_scm.url}' '#{url}'"
+        create_tracking_branch(remote_scm.branch_name) # ensure the correct branch exists locally
+        clean_and_checkout_branch # switch to the correct branch
+      end
+
+      # We need very high reliability and this sequence gets the job done every time.
+      def clean_and_checkout_branch
+        return unless status.scm_dir_exist?
+
+        run "cd '#{url}' && git clean -f -d -x"
+        return unless status.branch?(branch_name)
+
+        run "cd '#{url}' && git checkout #{branch_name} --"
+        run "cd '#{url}' && git reset --hard heads/#{branch_name} --"
+      end
+
+      def create_tracking_branch(branch_name)
+        return if branch_name.to_s.empty?
+        return if activity.branches.include?(branch_name)
+
+        run "cd '#{url}' && git branch -f #{branch_name} origin/#{branch_name}"
+      end
+
+      # Deletes everything but the *.git* folder in the working directory.
+      def clean_up_disk
+        return unless Dir.exist?(url)
+
+        run "cd #{url} && find . -maxdepth 1 -not -name .git -not -name . -print0"\
+              ' | xargs -0 rm -rf --'
+      end
+
+      def convert_to_git(remote_scm, callback)
+        callback.update(0, 1)
+
+        commits = remote_scm.activity.commits(after: activity.read_token)
+        check_empty_repository(commits)
+
+        if commits && !commits.empty?
+          setup_dir_and_convert_commits(commits, callback)
+        else
+          logger.info { 'Already up-to-date.' }
+        end
+      end
+
+      def setup_dir_and_convert_commits(commits, callback)
+        set_up_working_directory
+        convert(commits, callback)
+        callback.update(commits.size, commits.size)
+      end
+
+      def convert(commits, callback)
+        commits.each_with_index do |r, i|
+          callback.update(i, commits.size)
+          create_git_commit(r, i, commits.size)
+        end
+      end
+
+      def check_empty_repository(commits)
+        raise 'Empty repository' if !activity.read_token && commits.empty?
+      end
+
+      def set_up_working_directory
+        # Start by making sure we are in a known good state. Set up our working directory.
+        clean_up_disk
+        clean_and_checkout_branch
+      end
+
+      def handle_checkout_error(commit)
+        logger.error { $ERROR_INFO.inspect }
+        # If we fail to checkout, it's often because there is junk of some kind
+        # in our working directory.
+        logger.info { 'Checkout failed. Cleaning and trying again...' }
+        clean_up_disk
+        commit.scm.checkout(commit, url)
+      end
+
+      def create_git_commit(commit, index, size)
+        logger.info { "Downloading revision #{commit.token} (#{index + 1} of #{size})... " }
+        checkout(commit)
+        logger.debug { "Committing revision #{commit.token} (#{index + 1} of #{size})... " }
+        activity.commit_all(commit)
+      end
+
+      def checkout(commit)
+        commit.scm.checkout(commit, url)
+      rescue StandardError
+        handle_checkout_error(commit)
+      end
+    end
+  end
+end

data/lib/ohloh_scm/git/status.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module OhlohScm
+  module Git
+    class Status < OhlohScm::Status
+      def branch?(name = scm.branch_name)
+        return unless scm_dir_exist?
+
+        activity.branches.include?(name)
+      end
+    end
+  end
+end

data/lib/ohloh_scm/git/validation.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module OhlohScm
+  module Git
+    class Validation < OhlohScm::Validation
+      private
+
+      def validate_server_connection
+        msg = "The server did not respond to the 'git-ls-remote' command. Is the URL correct?"
+        @errors << [:failed, msg] unless status.exist?
+      end
+
+      def public_url_regex
+        %r{^(http|https|git)://(\w+@)?[\w\-\.]+(:\d+)?/[\w\-\./\~\+]*$}
+      end
+    end
+  end
+end

data/lib/ohloh_scm/git_svn.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'git_svn/scm'
+require_relative 'git_svn/status'
+require_relative 'git_svn/activity'
+require_relative 'git_svn/validation'
+
+module OhlohScm
+  module GitSvn
+  end
+end

data/lib/ohloh_scm/git_svn/activity.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module OhlohScm
+  module GitSvn
+    class Activity < OhlohScm::Activity
+      def_delegators :scm, :url
+
+      def commit_count(opts = {})
+        cmd = "#{after_revision(opts)} | grep -E -e '^r[0-9]+.*lines$' | wc -l"
+        git_svn_log(cmd: cmd, oneline: false).to_i
+      end
+
+      def commits(opts = {})
+        parsed_commits = []
+        open_log_file(opts) do |io|
+          parsed_commits = OhlohScm::SvnParser.parse(io)
+        end
+        parsed_commits
+      end
+
+      def commit_tokens(opts = {})
+        cmd = "#{after_revision(opts)} | #{extract_revision_number}"
+        git_svn_log(cmd: cmd, oneline: false).split.map(&:to_i)
+      end
+
+      def each_commit(opts = {})
+        commits(opts).each { |commit| yield commit }
+      end
+
+      def head_token
+        cmd = "--limit=1 | #{extract_revision_number}"
+        git_svn_log(cmd: cmd, oneline: false)
+      end
+
+      def cat_file(commit, diff)
+        cat(git_commit(commit), diff.path)
+      end
+
+      def cat_file_parent(commit, diff)
+        cat("#{git_commit(commit)}^", diff.path)
+      end
+
+      private
+
+      def open_log_file(opts = {})
+        cmd = "-v #{after_revision(opts)} | #{string_encoder_path} > #{log_filename}"
+        git_svn_log(cmd: cmd, oneline: false)
+        File.open(log_filename, 'r') { |io| yield io }
+      end
+
+      def log_filename
+        File.join('/tmp', url.gsub(/\W/, '') + '.log')
+      end
+
+      def after_revision(opts)
+        next_token = opts[:after].to_i + 1
+        next_head_token = head_token.to_i + 1
+        "-r#{next_token}:#{next_head_token}"
+      end
+
+      def extract_revision_number
+        "grep '^r[0-9].*|' | awk -F'|' '{print $1}' | cut -c 2-"
+      end
+
+      def git_svn_log(cmd:, oneline:)
+        oneline_flag = '--oneline' if oneline
+        run("#{git_svn_log_cmd} #{oneline_flag} #{cmd}").strip
+      end
+
+      def cat(revision, file_path)
+        file_path = file_path.to_s
+        run("cd #{url} && git show #{revision}:#{file_path.shellescape}").strip
+      end
+
+      def git_commit(commit)
+        run("cd #{url} && git svn find-rev r#{commit.token}").strip
+      end
+
+      def git_svn_log_cmd
+        "cd #{url} && git svn log"
+      end
+    end
+  end
+end

data/lib/ohloh_scm/git_svn/scm.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module OhlohScm
+  module GitSvn
+    class Scm < OhlohScm::Scm
+      def initialize(core:, url:, branch_name:, username:, password:)
+        super
+        @branch_name = branch_name || :master
+      end
+
+      def pull(source_scm, callback)
+        @source_scm = source_scm
+        convert_to_git(callback)
+      end
+
+      def accept_ssl_certificate_cmd
+        File.expand_path('../../../.bin/accept_svn_ssl_certificate', __dir__)
+      end
+
+      def vcs_path
+        "#{url}/.git"
+      end
+
+      private
+
+      def convert_to_git(callback)
+        callback.update(0, 1)
+        if FileTest.exist?(git_path)
+          accept_certificate_if_prompted
+          fetch
+        else
+          clone
+        end
+
+        clean_up_disk
+        callback.update(1, 1)
+      end
+
+      def git_path
+        File.join(url, '/.git')
+      end
+
+      def clone
+        prepare_dest_dir
+        accept_certificate_if_prompted
+
+        cmd = "#{password_prompt} git svn clone --quiet #{username_opts}"\
+                " '#{@source_scm.url}' '#{url}'"
+        run(cmd)
+      end
+
+      def accept_certificate_if_prompted
+        # git svn does not support non iteractive and serv-certificate options
+        # Permanently accept svn certificate when it prompts
+        opts = username_and_password_opts
+        run "#{accept_ssl_certificate_cmd} svn info #{opts} '#{@source_scm.url}'"
+      end
+
+      def username_and_password_opts
+        username = username.to_s.empty? ? '' : "--username #{@source_scm.username}"
+        password = password.to_s.empty? ? '' : "--password='#{@source_scm.password}'"
+        "#{username} #{password}"
+      end
+
+      def password_prompt
+        password.to_s.empty? ? '' : "echo #{password} |"
+      end
+
+      def username_opts
+        username.to_s.empty? ? '' : "--username #{username}"
+      end
+
+      def prepare_dest_dir
+        FileUtils.mkdir_p(url)
+        FileUtils.rm_rf(url)
+      end
+
+      def fetch
+        cmd = "cd #{url} && git svn fetch"
+        run(cmd)
+      end
+
+      def clean_up_disk
+        return unless  File.exist?(url)
+
+        run "cd #{url} && find . -maxdepth 1 -not -name .git -not -name . -print0"\
+              ' | xargs -0 rm -rf --'
+      end
+    end
+  end
+end

data/lib/ohloh_scm/git_svn/status.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module OhlohScm
+  module GitSvn
+    class Status < OhlohScm::Status
+    end
+  end
+end

data/lib/ohloh_scm/git_svn/validation.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module OhlohScm
+  module GitSvn
+    class Validation < OhlohScm::Validation
+    end
+  end
+end

data/lib/ohloh_scm/hg.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative 'parser/hg_parser'
+require_relative 'hg/scm'
+require_relative 'hg/status'
+require_relative 'hg/activity'
+require_relative 'hg/validation'
+
+module OhlohScm
+  module Hg
+  end
+end

data/lib/ohloh_scm/hg/activity.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require 'shellwords'
+
+# rubocop:disable Metrics/ClassLength
+module OhlohScm
+  module Hg
+    class Activity < OhlohScm::Activity
+      def commit_count(opts = {})
+        commit_tokens(opts).size
+      end
+
+      def commit_tokens(opts = {})
+        hg_log_with_opts, after = hg_log_cmd_builder(opts)
+        # We reverse the final result in Ruby, rather than passing the --reverse flag to hg.
+        # That's because the -f (follow) flag doesn't behave the same in both directions.
+        # Basically, we're trying very hard to make this act just like Git.
+        tokens = run("cd '#{url}' && #{hg_log_with_opts} --template='{node}\\n'")
+                 .split("\n").reverse
+
+        # Since hg v4, --follow-first does not play well with -r N:n,
+        #   using them together always returns all commits.
+        # To find trunk commits since a given revision,
+        #   we get all trunk commits and drop older commits before given revision.
+        tokens = tokens.drop(tokens.index(after)) if tokens.any? && after && after != 0
+
+        # This includes everything after *and including* after.
+        tokens.shift if tokens.first == after
+        tokens
+      end
+
+      # Returns a list of shallow commits (i.e., the diffs are not populated).
+      def commits(opts = {})
+        hg_log_with_opts, after = hg_log_cmd_builder(opts)
+
+        log = run("cd '#{url}' && #{hg_log_with_opts} --style #{OhlohScm::HgParser.style_path}")
+
+        commit_objects = OhlohScm::HgParser.parse(log).reverse
+        commit_objects.shift if commit_objects.first&.token == after
+        commit_objects
+      end
+
+      # Returns a single commit, including its diffs
+      def verbose_commit(token)
+        cmd = "cd '#{url}' && hg log -v -r #{token} "\
+              " --style #{OhlohScm::HgParser.verbose_style_path} | #{string_encoder_path}"
+        OhlohScm::HgParser.parse(run(cmd)).first
+      end
+
+      # Yields each commit after +after+, including its diffs.
+      # The log is stored in a temporary file.
+      # This is designed to prevent excessive RAM usage when we encounter a massive repository.
+      # Only a single commit is ever held in memory at once.
+      def each_commit(opts = {})
+        after = opts[:after] || 0
+        open_log_file(opts) do |io|
+          commits = OhlohScm::HgParser.parse(io)
+          # NOTE: commits.reverse.each & commits.reverse_each produce different ordered sequences.
+          # rubocop:disable Performance/ReverseEach
+          commits.reverse.each do |commit|
+            yield commit if block_given? && commit.token != after
+          end
+          # rubocop:enable Performance/ReverseEach
+        end
+      end
+
+      def export(dest_dir, token = 'tip')
+        run("cd '#{url}' && hg archive -r #{token} '#{dest_dir}'")
+
+        # Hg leaves a little cookie crumb in the export directory. Remove it.
+        file_path = File.join(dest_dir, '.hg_archival.txt')
+        File.delete(file_path) if File.exist?(file_path)
+      end
+
+      def tags
+        tag_strings = run("cd '#{url}' && hg tags").split(/\n/)
+        tag_strings.map do |tag_string|
+          parsed_str = tag_string.split(' ')
+          rev_number_and_hash = parsed_str.pop
+          tag_name = parsed_str.join(' ')
+          rev = rev_number_and_hash.slice(/\A\d+/)
+          time_string = run("cd '#{url}' && hg log -r #{rev} | grep 'date:' | sed 's/date://'")
+          [tag_name, rev, Time.parse(time_string)]
+        end
+      end
+
+      def cat_file(commit, diff)
+        hg_client.cat_file(commit.token, diff.path)
+      end
+
+      def cat_file_parent(commit, diff)
+        tokens = parent_tokens(commit)
+        hg_client.cat_file(tokens.first, diff.path) if tokens.first
+      end
+
+      def head_token
+        branch_opts = "--rev #{scm.branch_name_or_default}"
+        # This only returns first 12 characters.
+        # How can we make it return the entire hash?
+        token = run("hg id --debug -i -q #{url} #{branch_opts}").strip
+
+        # Recent versions of Hg now somtimes append a '+' char to the token.
+        # Strip the trailing '+', if any.
+        token = token[0..-2] if token[-1..-1] == '+'
+
+        token
+      end
+
+      def head
+        verbose_commit(head_token)
+      end
+
+      def cleanup
+        hg_client.shutdown
+      end
+
+      private
+
+      # Our standards require +opts={ after: ... }+ to include everything AFTER +after+.
+      # However, hg returns everything after and INCLUDING +after+.
+      # Therefore, consumers of this endpoint must check for and reject the duplicate commit.
+      def open_log_file(opts = {})
+        get_hg_log(opts)
+        File.open(log_filename, 'r') { |io| yield io }
+      ensure
+        File.delete(log_filename) if File.exist?(log_filename)
+      end
+
+      def get_hg_log(opts)
+        hg_log_with_opts, after = hg_log_cmd_builder(opts)
+        if after == head_token # There are no new commits
+          # As a time optimization,
+          #  just create an empty file rather than fetch a log we know will be empty.
+          File.write(log_filename, '')
+        else
+          cmd = "cd '#{url}' && #{hg_log_with_opts}"\
+                " --style #{OhlohScm::HgParser.verbose_style_path} | #{string_encoder_path}"\
+                " > #{log_filename}"
+          run cmd
+        end
+      end
+
+      def parent_tokens(commit)
+        hg_client.parent_tokens(commit.token)
+      end
+
+      def cat(revision, path)
+        out, err = run_with_err("cd '#{url}' && hg cat -r #{revision} #{escape(path)}")
+        return if err =~ /No such file in rev/i
+        raise err unless err&.empty?
+
+        out
+      end
+
+      # Escape bash-significant characters in the filename
+      # Example:
+      #     "Foo Bar & Baz" => "Foo\ Bar\ \&\ Baz"
+      def escape(path)
+        Shellwords.escape(path)
+      end
+
+      def hg_log_cmd_builder(opts)
+        after = opts[:after] || 0
+        up_to = opts[:up_to] || :tip
+
+        options = if opts[:trunk_only]
+                    '--follow-first'
+                  else
+                    branch = scm.branch_name_or_default
+                    "-r '#{up_to}:#{after} and (branch(#{branch}) or ancestors(#{branch}))'"
+                  end
+
+        ["hg log -v #{options}", after]
+      end
+
+      def hg_client
+        @hg_client ||= setup_hg_client
+      end
+
+      def setup_hg_client
+        hg_client = PyBridge::HgClient.new(url)
+        hg_client.start
+        hg_client
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength