hound_list_sync 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b7fcf5cbe88779791757fec939b6e6c29bc58abbb9392a94ee3172682620a5fe
+  data.tar.gz: 7946d13ddc068dce7b6f18136a016d9f083268c51b86ddf87b72aecd6446ce91
+SHA512:
+  metadata.gz: 85241d521699419f54a6c530e4bc78cc6b29966e5d358c426063bb363ff9c5076f5bfb6f8011f723d5c7a6bcb5ab31e9943c06687467ca169f693d54c5f359dc
+  data.tar.gz: 0d1827473c47d289ee1fba941bff6940e647cb7f3f8784ff711a2891bab30cf91a1386256d82234de12c05dfb0cabb3966b8f508844dc1ce74332fa39de5f10f

data/.github/workflows/main.yml

@@ -0,0 +1,16 @@
+name: Ruby
+
+on: [push,pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.5.5
+        bundler-cache: true
+    - name: Run the default task
+      run: bundle exec rake

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,31 @@
+AllCops:
+  TargetRubyVersion: 2.5
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Style/FormatStringToken:
+  Enabled: false
+
+Gemspec/RequiredRubyVersion:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in hound_list_sync.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rspec", "~> 3.0"
+
+gem "rubocop", "~> 1.7"
+gem "rubocop-rake"
+gem "rubocop-rspec"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Dmitry Bochkarev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,149 @@
+# hound_list_sync
+
+There is a great program to search by source code - [hound](https://github.com/hound-search/hound). But there is a drawback - you need to keep the list of indexed repositories up to date.
+**hound_list_sync** solves this problem for organizations on Github and the list of projects in Gitlab.
+
+The concept is simple: you specify the base settings of the hound and a list of extensions applied to it.
+
+It is possible to extend the base configuration with a static repository list and repository lists downloaded from github for organization or projects in gitlab.
+
+## Installation
+
+### Prerequisites
+
+- [ruby](https://ruby-lang.org) >= 2.4
+
+```sh
+gem install hound_list_sync
+```
+
+## Usage
+
+<details>
+
+<summary>hound_list_sync --help</summary>
+
+```txt
+Usage: hound_list_sync [options]
+        --base=FILE                  Base Hound Config(required)
+        --out=FILE                   Resulting Hound Config(required)
+        --extension=FILE             Extension config, allow to have multiple
+    -h, --help                       Print this help
+
+    Base config is regular hound config:
+        {
+          "dbpath" : "db",
+          "vcs-config" : {
+            "git": {
+              "ref" : "main"
+            }
+          },
+          "repos" : {
+            "Hound" : {
+              "url" : "https://github.com/hound-search/hound.git"
+            }
+          }
+        }
+
+    Extensions configs allow to specify how to enrich base config:
+        {
+          "repos": {
+            "hound_list_sync": {
+              "url": "https://github.com/DmitryBochkarev/hound_list_sync.git"
+            }
+          },
+          "lists": {
+            "wallarm": {
+              "hosting": "github",
+              "org": "wallarm",
+              "credentials": {
+                "login": "DmitryBochkarev",
+                "pass": "[OAuth Token]"
+              }
+            },
+            "example.com": {
+              "hosting": "gitlab",
+              "api_endpoint": "https://gitlab.example.com",
+              "token": "[OAuth Token]",
+              "allow_list": [
+                "example/site/", "example/backoffice/"
+              ],
+              "block_list": [
+                ".*secrets.*"
+              ]
+            }
+          }
+        }
+```
+
+</details>
+
+Let's take the [basic config](https://github.com/hound-search/hound/blob/main/config-example.json) for the hound as an example:
+
+```json
+{
+  "dbpath": "db",
+  "vcs-config": {
+    "git": {
+      "ref": "main"
+    }
+  },
+  "repos": {
+    "Hound": {
+      "url": "https://github.com/hound-search/hound.git"
+    }
+  }
+}
+```
+
+Suppose we want to additionally index the list of repositories for the github organization and the list of projects on gitlab:
+
+```json
+{
+  "lists": {
+    "github.com": {
+      "hosting": "github",
+      "org": "github",
+      "credentials": {
+        "login": "DmitryBochkarev",
+        "pass": "[OAuth token]"
+      },
+      "block_list": [
+        ".*secrets.*",
+        ".*example.*",
+        ".*terraform.*",
+        "\\Agithub/docs\\z"
+      ]
+    },
+    "example.com": {
+      "hosting": "gitlab",
+      "api_endpoint": "https://gitlab.example.com",
+      "token": "[OAuth Token]",
+      "allow_list": ["example/site/", "example/backoffice/"],
+      "block_list": [".*secrets.*"]
+    }
+  }
+}
+```
+
+You can specify `allow_list` and `block_list` lists with regular expressions that filter the final list of repositories.
+
+```sh
+hound_list_sync --base=hound_base.json --extension=hound_extension.json --out=hound_config.json
+```
+
+This will give us a hound configuration to index the organization's repositories https://github.com/github and the list of projects from our server at gitlab.example.com.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/DmitryBochkarev/hound_list_sync.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "hound_list_sync"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/hound_list_sync

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "hound_list_sync"
+require "hound_list_sync/cli"
+
+require "logger"
+require "optparse"
+
+HELP = <<-TXT
+    Base config is regular hound config:
+        {
+          "dbpath" : "db",
+          "vcs-config" : {
+            "git": {
+              "ref" : "main"
+            }
+          },
+          "repos" : {
+            "Hound" : {
+              "url" : "https://github.com/hound-search/hound.git"
+            }
+          }
+        }
+
+    Extensions configs allow to specify how to enrich base config:
+        {
+          "repos": {
+            "hound_list_sync": {
+              "url": "https://github.com/DmitryBochkarev/hound_list_sync.git"
+            }
+          },
+          "lists": {
+            "wallarm": {
+              "hosting": "github",
+              "org": "wallarm",
+              "credentials": {
+                "login": "DmitryBochkarev",
+                "pass": "[OAuth Token]"
+              }
+            },
+            "example.com": {
+              "hosting": "gitlab",
+              "api_endpoint": "https://gitlab.example.com",
+              "token": "[OAuth Token]",
+              "allow_list": [
+                "example/site/", "example/backoffice/"
+              ],
+              "block_list": [
+                ".*secrets.*"
+              ]
+            }
+          }
+        }
+TXT
+
+options = { extensions: [] }
+opt_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: hound_list_sync [options]"
+
+  opts.on("--base=FILE", "Base Hound Config(required)") do |v|
+    options[:base] = v
+  end
+
+  opts.on("--out=FILE", "Resulting Hound Config(required)") do |v|
+    options[:out] = v
+  end
+
+  opts.on("--extension=FILE", "Extension config, allow to have multiple") do |v|
+    options[:extensions].push(v)
+  end
+
+  opts.on("-h", "--help", "Print this help") do |_v|
+    puts opts
+    puts
+    puts HELP
+
+    exit
+  end
+end
+
+opt_parser.parse!(ARGV)
+
+p options
+unless options[:base]
+  puts "missing --base=FILE"
+  puts opt_parser
+
+  exit(1)
+end
+
+unless options[:out]
+  puts "missing --out=FILE"
+  puts opt_parser
+
+  exit(1)
+end
+
+logger = Logger.new($stdout)
+
+HoundListSync::CLI.new(
+  options.fetch(:base),
+  options.fetch(:extensions),
+  options.fetch(:out),
+  logger: logger
+).run

data/hound_list_sync.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "lib/hound_list_sync/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "hound_list_sync"
+  spec.version       = HoundListSync::VERSION
+  spec.authors       = ["Dmitry Bochkarev"]
+  spec.email         = ["dimabochkarev@gmail.com"]
+
+  spec.summary       = "Auto generate Hound config from remote lists"
+  spec.description   = "Sync Hound repositories with Github organization and Gitlab projects"
+  spec.homepage      = "https://github.com/DmitryBochkarev/hound_list_sync"
+  spec.license       = "MIT"
+  spec.required_ruby_version = ">= 2.4.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/DmitryBochkarev/hound_list_sync"
+
+  spec.add_dependency "faraday", "~> 1.5.1"
+  spec.add_dependency "faraday_middleware", "~> 1.0.0"
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+end

data/lib/hound_list_sync.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "hound_list_sync/version"
+
+module HoundListSync
+  class Error < StandardError; end
+end
+
+require_relative "hound_list_sync/http"
+require_relative "hound_list_sync/repositories"
+require_relative "hound_list_sync/hound_config"
+require_relative "hound_list_sync/extension"
+require_relative "hound_list_sync/extensions"

data/lib/hound_list_sync/cli.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "json"
+
+module HoundListSync
+  class CLI
+    def initialize(base, extensions, out, logger:)
+      @base = base
+      @extensions = extensions
+      @out = out
+      @logger = logger
+    end
+
+    def run
+      @logger.info("Loading base config #{@base}")
+      base = HoundConfig.new(read_config(@base))
+
+      if @extensions.any?
+        @logger.info("Loading extensions from #{@extensions.join(", ")}")
+        extensions =
+          Extensions.new(
+            @extensions.map { |e| read_config(e) },
+            http: Http::Net.new(logger: @logger)
+          )
+
+        @logger.info("Extending base config")
+        base.extend_with(extensions)
+      end
+
+      @logger.info("Total repositories: #{base.total_repos}")
+
+      @logger.info("Generating new config")
+      config = JSON.pretty_generate(base)
+      old_config = File.read(@out) if File.exist?(@out)
+
+      if config == old_config
+        @logger.info("Config not changed")
+      else
+        @logger.info("Saving to #{@out}")
+        File.write(@out, config)
+      end
+
+      @logger.info("Done")
+    end
+
+    def read_config(file)
+      unless File.exist?(file)
+        @logger.error("File missing: #{file}")
+        exit(1)
+      end
+
+      JSON.parse(File.read(file))
+    end
+  end
+end

data/lib/hound_list_sync/extension.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  class Extension
+    def initialize(conf, http:)
+      @conf = conf
+      @http = http
+    end
+
+    def repositories
+      repositories = []
+
+      repositories.push(Repositories::Raw.new(@conf["repos"])) if @conf.key?("repos")
+
+      if @conf.key?("lists")
+        repositories.push(
+          Repositories::Join.new(
+            @conf["lists"].map { |name, conf| list_repositories(name, conf) }
+          )
+        )
+      end
+
+      Repositories::Join.new(repositories)
+    end
+
+    def list_repositories(name, conf) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      repositories =
+        case conf["hosting"]
+        when "github"
+          Repositories::GithubOrg.new(
+            conf.fetch("org"),
+            http: @http,
+            api_endpoint: conf["api_endpoint"],
+            credentials: conf.fetch("credentials", {}).transform_keys(&:to_sym)
+          )
+        when "gitlab"
+          Repositories::Gitlab.new(
+            conf.fetch("api_endpoint"),
+            http: @http,
+            token: conf["token"]
+          )
+        else
+          raise "Invalid config #{name}: #{conf}"
+        end
+
+      if conf["allow_list"].is_a?(Array) && conf["allow_list"]&.any?
+        repositories = Repositories::AllowList.new(
+          repositories,
+          names: conf["allow_list"]
+        )
+      end
+
+      if conf["block_list"].is_a?(Array) && conf["block_list"]&.any?
+        repositories = Repositories::BlockList.new(
+          repositories,
+          names: conf["block_list"]
+        )
+      end
+
+      repositories
+    end
+  end
+end

data/lib/hound_list_sync/extensions.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  class Extensions
+    def initialize(raw, http:)
+      @raw = raw
+      @http = http
+    end
+
+    def repositories
+      Repositories::Join.new(
+        @raw.map { |conf| extension(conf).repositories }
+      )
+    end
+
+    def extension(conf)
+      Extension.new(conf, http: @http)
+    end
+  end
+end

data/lib/hound_list_sync/hound_config.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  class HoundConfig
+    def initialize(raw)
+      @raw = raw
+      @raw["repos"] ||= {}
+    end
+
+    def extend_with(extensions)
+      extensions.repositories.each do |repo|
+        next unless repo.indexable?
+
+        @raw["repos"][repo.name] = repo.to_config
+      end
+    end
+
+    def to_json(*args)
+      @raw.to_json(*args)
+    end
+
+    def total_repos
+      @raw["repos"].keys.length
+    end
+  end
+end

data/lib/hound_list_sync/http.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "faraday_middleware"
+
+module HoundListSync
+  module Http
+    class Response
+      attr_reader :status, :headers, :body
+
+      def initialize(status, headers, body)
+        @status = status
+        @headers = headers
+        @body = body
+      end
+    end
+
+    def get(_url, headers:, basic_auth: []) # rubocop:disable Lint/UnusedMethodArgument
+      raise "#{self.class} has not implemented method '#{__method__}'"
+    end
+
+    class Net
+      include Http
+
+      HEADERS = {
+        "User-Agent" => "Hound-List-Sync/#{HoundListSync::VERSION} Faraday/#{Faraday::VERSION} ruby/#{RUBY_VERSION}"
+      }.freeze
+
+      def initialize(logger: nil)
+        @logger = logger
+      end
+
+      def get(url, headers:, basic_auth: [])
+        conn = Faraday.new(url: url, headers: HEADERS.merge(headers)) do |faraday|
+          faraday.basic_auth(*basic_auth) if basic_auth.any?
+
+          faraday.request :retry
+
+          faraday.response :follow_redirects
+          faraday.response :raise_error
+
+          if @logger
+            faraday.response :logger, @logger do |logger|
+              logger.filter(/(Authorization: )(.+)/i, '\1[REMOVED]')
+            end
+          end
+        end
+
+        resp = conn.get
+
+        Response.new(resp.status, resp.headers, resp.body)
+      end
+    end
+  end
+end

data/lib/hound_list_sync/http/fake.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  module Http
+    class Fake
+      include Http
+
+      Error = Class.new(StandardError)
+      NoMoreResponses = Class.new(Error)
+
+      class Get
+        attr_reader :url, :headers, :basic_auth, :response
+
+        def initialize(url, headers:, basic_auth:, response:)
+          @url = url
+          @headers = headers
+          @basic_auth = basic_auth
+          @response = response
+        end
+      end
+
+      attr_reader :requests, :responses
+
+      def initialize(responses)
+        @requests = []
+        @responses = responses
+      end
+
+      def get(url, headers:, basic_auth: [])
+        response = @responses.shift
+
+        raise NoMoreResponses, "no more response for GET #{url}" unless response
+
+        @requests.push(Get.new(url, headers: headers, basic_auth: basic_auth, response: response))
+
+        response
+      end
+    end
+  end
+end

data/lib/hound_list_sync/repositories.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  module Repositories
+    module Repo
+      def name
+        raise "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      def to_config
+        raise "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      def indexable?
+        raise "#{self.class} has not implemented method '#{__method__}'"
+      end
+    end
+  end
+end
+
+require_relative "repositories/github_org"
+require_relative "repositories/gitlab"
+require_relative "repositories/allow_list"
+require_relative "repositories/block_list"
+require_relative "repositories/join"
+require_relative "repositories/raw"

data/lib/hound_list_sync/repositories/allow_list.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  module Repositories
+    class AllowList
+      include Repositories
+      include Enumerable
+
+      def initialize(original, names: [])
+        @original = original
+        @names = names.map { |n| ::Regexp.new(n, ::Regexp::MULTILINE | ::Regexp::IGNORECASE) }
+      end
+
+      def each
+        return to_enum unless block_given?
+
+        @original.each do |repo|
+          next unless @names.any? { |name| name.match?(repo.name) }
+
+          yield repo
+        end
+      end
+    end
+  end
+end

data/lib/hound_list_sync/repositories/block_list.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  module Repositories
+    class BlockList
+      include Repositories
+      include Enumerable
+
+      def initialize(original, names: [])
+        @original = original
+        @names = names.map { |n| ::Regexp.new(n, ::Regexp::MULTILINE | ::Regexp::IGNORECASE) }
+      end
+
+      def each
+        return to_enum unless block_given?
+
+        @original.each do |repo|
+          next if @names.any? { |name| name.match?(repo.name) }
+
+          yield repo
+        end
+      end
+    end
+  end
+end

data/lib/hound_list_sync/repositories/github_org.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "json"
+
+module HoundListSync
+  module Repositories
+    class GithubOrg
+      include Repositories
+      include Enumerable
+
+      URL_TEMPLATE = "%{base}/orgs/%{org}/repos?page=%{page}"
+      API_GITHUB_COM = "https://api.github.com"
+      HEADERS = { "Accept" => "application/vnd.github.v3+json" }.freeze
+
+      class Repo
+        include Repositories::Repo
+
+        def initialize(raw)
+          @raw = raw
+        end
+
+        def indexable?
+          !@raw["archived"] && !@raw["disabled"]
+        end
+
+        def name
+          @raw["full_name"]
+        end
+
+        def to_config
+          {
+            "url" => url,
+            "url-pattern" => {
+              "base-url" => base_url,
+              "anchor" => anchor
+            },
+            "vcs-config" => {
+              "ref" => branch
+            }
+          }
+        end
+
+        def url
+          @raw["ssh_url"]
+        end
+
+        def branch
+          @raw["default_branch"] || "master"
+        end
+
+        def base_url
+          "#{@raw["html_url"]}/blob/#{branch}/{path}{anchor}"
+        end
+
+        def anchor
+          "#L{line}"
+        end
+      end
+
+      def initialize(
+        orgname,
+        api_endpoint: nil,
+        credentials: {},
+        url_template: URL_TEMPLATE,
+        http: HoundListSync::Http::Net.new
+      )
+        @orgname = orgname
+        @api_endpoint = api_endpoint || API_GITHUB_COM
+        @credentials = credentials
+        @url_template = url_template
+        @http = http
+      end
+
+      def each
+        return to_enum unless block_given?
+
+        page = 1
+
+        loop do
+          repos =
+            JSON.parse(
+              @http
+                .get(url(page), headers: HEADERS, basic_auth: basic_auth)
+                .body
+            )
+
+          break if repos.length.zero?
+
+          repos.each do |repo|
+            yield Repo.new(repo)
+          end
+
+          page += 1
+        end
+      end
+
+      def url(page)
+        format(@url_template, base: @api_endpoint, org: @orgname, page: page)
+      end
+
+      def basic_auth
+        @basic_auth ||=
+          if @credentials.any?
+            [@credentials.fetch(:login), @credentials.fetch(:pass)]
+          else
+            []
+          end
+      end
+    end
+  end
+end

data/lib/hound_list_sync/repositories/gitlab.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "json"
+
+module HoundListSync
+  module Repositories
+    class Gitlab
+      include Repositories
+      include Enumerable
+
+      URL_TEMPLATE = "%{base}/api/v4/projects?page=%{page}"
+
+      class Repo
+        include Repositories::Repo
+
+        def initialize(raw)
+          @raw = raw
+        end
+
+        def indexable?
+          !@raw["archived"] && !@raw["empty_repo"]
+        end
+
+        def name
+          @raw["path_with_namespace"]
+        end
+
+        def to_config
+          {
+            "url" => url,
+            "url-pattern" => {
+              "base-url" => base_url,
+              "anchor" => anchor
+            },
+            "vcs-config" => {
+              "ref" => branch
+            }
+          }
+        end
+
+        def url
+          @raw["ssh_url_to_repo"]
+        end
+
+        def branch
+          @raw["default_branch"] || "master"
+        end
+
+        def base_url
+          "#{@raw["web_url"]}/blob/#{branch}/{path}{anchor}"
+        end
+
+        def anchor
+          "#L{line}"
+        end
+      end
+
+      def initialize(
+        api_endpoint,
+        token: nil,
+        url_template: URL_TEMPLATE,
+        http: HoundListSync::Http::Net.new
+      )
+        @api_endpoint = api_endpoint
+        @token = token
+        @url_template = url_template
+        @http = http
+      end
+
+      def each
+        return to_enum unless block_given?
+
+        page = "1"
+
+        loop do
+          response = @http.get(url(page), headers: auth_headers)
+
+          repos = JSON.parse(response.body)
+
+          break if repos.length.zero?
+
+          repos.each do |repo|
+            yield Repo.new(repo)
+          end
+
+          next_page = response.headers["x-next-page"]
+          break if next_page.empty?
+
+          page = next_page
+        end
+      end
+
+      def url(page)
+        format(@url_template, base: @api_endpoint, page: page)
+      end
+
+      def auth_headers
+        @auth_headers ||=
+          if @token.nil?
+            {}
+          else
+            { "Authorization" => "Bearer #{@token}" }
+          end
+      end
+    end
+  end
+end

data/lib/hound_list_sync/repositories/join.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  module Repositories
+    class Join
+      include Repositories
+      include Enumerable
+
+      def initialize(list)
+        @list = list
+      end
+
+      def each(&block)
+        return to_enum unless block_given?
+
+        @list.each do |repositories|
+          repositories.each(&block)
+        end
+      end
+    end
+  end
+end

data/lib/hound_list_sync/repositories/raw.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  module Repositories
+    class Raw
+      include Repositories
+      include Enumerable
+
+      class Repo
+        include Repositories::Repo
+
+        def initialize(name, config)
+          @name = name
+          @config = config
+        end
+
+        attr_reader :name
+
+        def to_config
+          @config
+        end
+
+        def indexable?
+          true
+        end
+      end
+
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def each
+        return to_enum unless block_given?
+
+        @raw.each do |name, config|
+          yield Repo.new(name, config)
+        end
+      end
+    end
+  end
+end

data/lib/hound_list_sync/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module HoundListSync
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: hound_list_sync
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dmitry Bochkarev
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-07-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: Sync Hound repositories with Github organization and Gitlab projects
+email:
+- dimabochkarev@gmail.com
+executables:
+- hound_list_sync
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/main.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/hound_list_sync
+- hound_list_sync.gemspec
+- lib/hound_list_sync.rb
+- lib/hound_list_sync/cli.rb
+- lib/hound_list_sync/extension.rb
+- lib/hound_list_sync/extensions.rb
+- lib/hound_list_sync/hound_config.rb
+- lib/hound_list_sync/http.rb
+- lib/hound_list_sync/http/fake.rb
+- lib/hound_list_sync/repositories.rb
+- lib/hound_list_sync/repositories/allow_list.rb
+- lib/hound_list_sync/repositories/block_list.rb
+- lib/hound_list_sync/repositories/github_org.rb
+- lib/hound_list_sync/repositories/gitlab.rb
+- lib/hound_list_sync/repositories/join.rb
+- lib/hound_list_sync/repositories/raw.rb
+- lib/hound_list_sync/version.rb
+homepage: https://github.com/DmitryBochkarev/hound_list_sync
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/DmitryBochkarev/hound_list_sync
+  source_code_uri: https://github.com/DmitryBochkarev/hound_list_sync
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.8
+signing_key: 
+specification_version: 4
+summary: Auto generate Hound config from remote lists
+test_files: []