mass_git_clone 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cb092e9b18bec254cd426940d49b95600ffbd23ff549f5baee27e3c41cdd8e40
+  data.tar.gz: 5e73ba35f65106cda87d2f78679fef93d9fe4badcf141f9ef38f9fc4d54a0626
+SHA512:
+  metadata.gz: fcf74cf540f0af2fb833a57ee6161c6879e575e2fdaf33ef989f159a8393af900a2bc0261f40d80237a5c2c5d28e3644c9ebf01f96efbc18f4e8c22bfc637c11
+  data.tar.gz: feceed95021732b251ff1b411b4e03787e1801b1db9acda9fdcfda8aa9c1491aefe2e493da196dade95678b491ba663b97b5de4427e9703a6c8fb5e8849e6d0b

data/.github/workflows/ci.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby:
+          - "2.7"
+          - "3.0"
+          - "3.1"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Rubocop
+        run: bundle exec rubocop
+
+      - name: Syntax Tree
+        run: bundle exec stree check Gemfile $(git ls-files '*.rb') $(git ls-files '*.rake')
+
+  publish:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Release Gem
+        uses: discourse/publish-rubygems-action@v2
+        env:
+          RUBYGEMS_API_KEY: ${{ secrets.RUBYGEMS_API_KEY }}
+          GIT_EMAIL: team@discourse.org
+          GIT_NAME: discoursebot

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+pkg

data/.rubocop.yml

@@ -0,0 +1,5 @@
+inherit_gem:
+  rubocop-discourse: stree-compat.yml
+
+Discourse/NoChdir:
+  Enabled: false

data/.streerc

@@ -0,0 +1 @@
+--print-width=100

data/Gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+source "https://rubygems.org"
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Civilized Discourse Construction Kit
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1 @@
+# mass_git_clone

data/Rakefile

@@ -0,0 +1,2 @@
+# frozen_string_literal: true
+require "bundler/gem_tasks"

data/bin/mass_git_clone

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/mass_git_clone'
+
+
+if ARGV[0].nil? || ARGV.size > 1 || STDIN.tty?
+  STDERR.puts "Usage `mass_git_clone <repo_base_directory>`. Pass newline-separated repo list via STDIN."
+  exit 1
+end
+
+MassGitClone.mass_clone(repo_base_dir: ARGV[0], repo_list: STDIN.each_line.to_a)

data/lib/mass_git_clone.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "open3"
+require "set"
+require "parallel"
+
+def run(*args)
+  out, err, status = Open3.capture3(*args)
+  raise <<~ERROR if !status.success?
+    Status #{status.exitstatus} running #{args.inspect}
+
+    stdout:
+    #{out}
+
+    stderr:
+    #{err}
+  ERROR
+  out
+end
+
+def prefixed_puts(str)
+  prefix = "[reset-all-repos]"
+  if n = Parallel.worker_number
+    prefix += "[t#{n}]"
+  end
+  print "#{prefix} #{str}\n"
+end
+
+# Clone or update a repo with the given URL
+# On failure, return the repo url
+# On success, return nil
+def update_repo(repo_url)
+  dir_name = File.basename(repo_url, ".git")
+  if Dir.exist?(dir_name)
+    prefixed_puts "Updating #{dir_name}..."
+
+    run "git", "-C", dir_name, "remote", "set-url", "origin", repo_url
+
+    default_branch =
+      File.basename(
+        run("git", "-C", dir_name, "symbolic-ref", "--short", "refs/remotes/origin/HEAD").strip
+      )
+
+    run "git", "-C", dir_name, "checkout", "-f", default_branch
+    run "git", "-C", dir_name, "fetch", "origin", default_branch
+    run "git", "-C", dir_name, "reset", "--hard", "origin/#{default_branch}"
+    run "git", "-C", dir_name, "clean", "-f", "-d"
+  else
+    prefixed_puts "Cloning #{dir_name}..."
+    run "git", "clone", "--quiet", repo_url, dir_name
+  end
+  nil
+rescue => e
+  prefixed_puts "Error while working on #{repo_url}\n\n#{e.message}"
+  repo_url
+end
+
+module MassGitClone
+  def self.mass_clone(repo_base_dir:, repo_list:)
+    use_ssh =
+      begin
+        run "git", "ls-remote", "git@github.com:discourse/discourse", "main"
+        prefixed_puts "Using SSH for GitHub..."
+        true
+      rescue => e
+        prefixed_puts "SSH failed. Using https for GitHub..."
+        false
+      end
+
+    all_repos = repo_list.map(&:strip).filter { |l| l.length > 0 }
+    if all_repos.size === 0
+      prefixed_puts "No plugin URLs supplied"
+      exit 1
+    end
+
+    all_repos =
+      all_repos.map do |repo|
+        if !repo.match?(%r{\A[\w-]+/[\w-]+\z})
+          repo # Full URL, leave untouched
+        elsif use_ssh
+          "git@github.com:#{repo}"
+        else
+          "https://github.com/#{repo}"
+        end
+      end
+
+    Dir.mkdir(repo_base_dir) if !Dir.exist?(repo_base_dir)
+
+    Dir.chdir(repo_base_dir) do
+      failures =
+        Parallel
+          .map(all_repos, in_threads: ENV["PARALLEL"]&.to_i || 10) do |repo_url|
+            update_repo(repo_url)
+          end
+          .reject(&:nil?)
+
+      final_dirs = Dir.glob("*")
+      expected_dirs = Set.new(all_repos.map { |r| File.basename(r, ".git") })
+      to_delete = final_dirs.reject { |dir| expected_dirs.include?(dir) }
+
+      if to_delete.any?
+        puts
+        prefixed_puts "Deleting #{to_delete.size} stale directories"
+        to_delete.each { |dir| FileUtils.rm_rf(dir) }
+      end
+
+      puts
+      if failures.any?
+        prefixed_puts "#{failures.count} repo(s) failed to update"
+        prefixed_puts failures.map { |url| "    #{url}" }.join("\n")
+        exit 1
+      else
+        prefixed_puts "Done - all repositories are up-to-date 🚀"
+      end
+    end
+  end
+end

data/mass_git_clone.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "mass_git_clone"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Discourse Team"]
+
+  spec.summary       = %q{Tool for maintaining clones of a large number of git repositories}
+  spec.description   = %q{Tool for maintaining clones of a large number of git repositories}
+  spec.homepage      = "https://github.com/discourse/mass_git_clone"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 2.7.0'
+
+  spec.add_dependency "parallel", "~> 1.0"
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "rubocop-discourse"
+  spec.add_development_dependency "syntax_tree"
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: mass_git_clone
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Discourse Team
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-01-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-discourse
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: syntax_tree
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Tool for maintaining clones of a large number of git repositories
+email:
+executables:
+- mass_git_clone
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/ci.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- ".streerc"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/mass_git_clone
+- lib/mass_git_clone.rb
+- mass_git_clone.gemspec
+homepage: https://github.com/discourse/mass_git_clone
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: Tool for maintaining clones of a large number of git repositories
+test_files: []