hound_list_sync 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/main.yml +16 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.rubocop.yml +31 -0
- data/Gemfile +14 -0
- data/LICENSE.txt +21 -0
- data/README.md +149 -0
- data/Rakefile +12 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/exe/hound_list_sync +106 -0
- data/hound_list_sync.gemspec +29 -0
- data/lib/hound_list_sync.rb +13 -0
- data/lib/hound_list_sync/cli.rb +55 -0
- data/lib/hound_list_sync/extension.rb +63 -0
- data/lib/hound_list_sync/extensions.rb +20 -0
- data/lib/hound_list_sync/hound_config.rb +26 -0
- data/lib/hound_list_sync/http.rb +55 -0
- data/lib/hound_list_sync/http/fake.rb +40 -0
- data/lib/hound_list_sync/repositories.rb +26 -0
- data/lib/hound_list_sync/repositories/allow_list.rb +25 -0
- data/lib/hound_list_sync/repositories/block_list.rb +25 -0
- data/lib/hound_list_sync/repositories/github_org.rb +111 -0
- data/lib/hound_list_sync/repositories/gitlab.rb +107 -0
- data/lib/hound_list_sync/repositories/join.rb +22 -0
- data/lib/hound_list_sync/repositories/raw.rb +41 -0
- data/lib/hound_list_sync/version.rb +5 -0
- metadata +101 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: b7fcf5cbe88779791757fec939b6e6c29bc58abbb9392a94ee3172682620a5fe
|
4
|
+
data.tar.gz: 7946d13ddc068dce7b6f18136a016d9f083268c51b86ddf87b72aecd6446ce91
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 85241d521699419f54a6c530e4bc78cc6b29966e5d358c426063bb363ff9c5076f5bfb6f8011f723d5c7a6bcb5ab31e9943c06687467ca169f693d54c5f359dc
|
7
|
+
data.tar.gz: 0d1827473c47d289ee1fba941bff6940e647cb7f3f8784ff711a2891bab30cf91a1386256d82234de12c05dfb0cabb3966b8f508844dc1ce74332fa39de5f10f
|
@@ -0,0 +1,16 @@
|
|
1
|
+
name: Ruby
|
2
|
+
|
3
|
+
on: [push,pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ubuntu-latest
|
8
|
+
steps:
|
9
|
+
- uses: actions/checkout@v2
|
10
|
+
- name: Set up Ruby
|
11
|
+
uses: ruby/setup-ruby@v1
|
12
|
+
with:
|
13
|
+
ruby-version: 2.5.5
|
14
|
+
bundler-cache: true
|
15
|
+
- name: Run the default task
|
16
|
+
run: bundle exec rake
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
AllCops:
|
2
|
+
TargetRubyVersion: 2.5
|
3
|
+
|
4
|
+
Style/StringLiterals:
|
5
|
+
Enabled: true
|
6
|
+
EnforcedStyle: double_quotes
|
7
|
+
|
8
|
+
Style/StringLiteralsInInterpolation:
|
9
|
+
Enabled: true
|
10
|
+
EnforcedStyle: double_quotes
|
11
|
+
|
12
|
+
Layout/LineLength:
|
13
|
+
Max: 120
|
14
|
+
|
15
|
+
Metrics/BlockLength:
|
16
|
+
Enabled: false
|
17
|
+
|
18
|
+
Metrics/MethodLength:
|
19
|
+
Enabled: false
|
20
|
+
|
21
|
+
Style/Documentation:
|
22
|
+
Enabled: false
|
23
|
+
|
24
|
+
Metrics/AbcSize:
|
25
|
+
Enabled: false
|
26
|
+
|
27
|
+
Style/FormatStringToken:
|
28
|
+
Enabled: false
|
29
|
+
|
30
|
+
Gemspec/RequiredRubyVersion:
|
31
|
+
Enabled: false
|
data/Gemfile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
# Specify your gem's dependencies in hound_list_sync.gemspec
|
6
|
+
gemspec
|
7
|
+
|
8
|
+
gem "rake", "~> 13.0"
|
9
|
+
|
10
|
+
gem "rspec", "~> 3.0"
|
11
|
+
|
12
|
+
gem "rubocop", "~> 1.7"
|
13
|
+
gem "rubocop-rake"
|
14
|
+
gem "rubocop-rspec"
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2021 Dmitry Bochkarev
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,149 @@
|
|
1
|
+
# hound_list_sync
|
2
|
+
|
3
|
+
There is a great program to search by source code - [hound](https://github.com/hound-search/hound). But there is a drawback - you need to keep the list of indexed repositories up to date.
|
4
|
+
**hound_list_sync** solves this problem for organizations on Github and the list of projects in Gitlab.
|
5
|
+
|
6
|
+
The concept is simple: you specify the base settings of the hound and a list of extensions applied to it.
|
7
|
+
|
8
|
+
It is possible to extend the base configuration with a static repository list and repository lists downloaded from github for organization or projects in gitlab.
|
9
|
+
|
10
|
+
## Installation
|
11
|
+
|
12
|
+
### Prerequisites
|
13
|
+
|
14
|
+
- [ruby](https://ruby-lang.org) >= 2.4
|
15
|
+
|
16
|
+
```sh
|
17
|
+
gem install hound_list_sync
|
18
|
+
```
|
19
|
+
|
20
|
+
## Usage
|
21
|
+
|
22
|
+
<details>
|
23
|
+
|
24
|
+
<summary>hound_list_sync --help</summary>
|
25
|
+
|
26
|
+
```txt
|
27
|
+
Usage: hound_list_sync [options]
|
28
|
+
--base=FILE Base Hound Config(required)
|
29
|
+
--out=FILE Resulting Hound Config(required)
|
30
|
+
--extension=FILE Extension config, allow to have multiple
|
31
|
+
-h, --help Print this help
|
32
|
+
|
33
|
+
Base config is regular hound config:
|
34
|
+
{
|
35
|
+
"dbpath" : "db",
|
36
|
+
"vcs-config" : {
|
37
|
+
"git": {
|
38
|
+
"ref" : "main"
|
39
|
+
}
|
40
|
+
},
|
41
|
+
"repos" : {
|
42
|
+
"Hound" : {
|
43
|
+
"url" : "https://github.com/hound-search/hound.git"
|
44
|
+
}
|
45
|
+
}
|
46
|
+
}
|
47
|
+
|
48
|
+
Extensions configs allow to specify how to enrich base config:
|
49
|
+
{
|
50
|
+
"repos": {
|
51
|
+
"hound_list_sync": {
|
52
|
+
"url": "https://github.com/DmitryBochkarev/hound_list_sync.git"
|
53
|
+
}
|
54
|
+
},
|
55
|
+
"lists": {
|
56
|
+
"wallarm": {
|
57
|
+
"hosting": "github",
|
58
|
+
"org": "wallarm",
|
59
|
+
"credentials": {
|
60
|
+
"login": "DmitryBochkarev",
|
61
|
+
"pass": "[OAuth Token]"
|
62
|
+
}
|
63
|
+
},
|
64
|
+
"example.com": {
|
65
|
+
"hosting": "gitlab",
|
66
|
+
"api_endpoint": "https://gitlab.example.com",
|
67
|
+
"token": "[OAuth Token]",
|
68
|
+
"allow_list": [
|
69
|
+
"example/site/", "example/backoffice/"
|
70
|
+
],
|
71
|
+
"block_list": [
|
72
|
+
".*secrets.*"
|
73
|
+
]
|
74
|
+
}
|
75
|
+
}
|
76
|
+
}
|
77
|
+
```
|
78
|
+
|
79
|
+
</details>
|
80
|
+
|
81
|
+
Let's take the [basic config](https://github.com/hound-search/hound/blob/main/config-example.json) for the hound as an example:
|
82
|
+
|
83
|
+
```json
|
84
|
+
{
|
85
|
+
"dbpath": "db",
|
86
|
+
"vcs-config": {
|
87
|
+
"git": {
|
88
|
+
"ref": "main"
|
89
|
+
}
|
90
|
+
},
|
91
|
+
"repos": {
|
92
|
+
"Hound": {
|
93
|
+
"url": "https://github.com/hound-search/hound.git"
|
94
|
+
}
|
95
|
+
}
|
96
|
+
}
|
97
|
+
```
|
98
|
+
|
99
|
+
Suppose we want to additionally index the list of repositories for the github organization and the list of projects on gitlab:
|
100
|
+
|
101
|
+
```json
|
102
|
+
{
|
103
|
+
"lists": {
|
104
|
+
"github.com": {
|
105
|
+
"hosting": "github",
|
106
|
+
"org": "github",
|
107
|
+
"credentials": {
|
108
|
+
"login": "DmitryBochkarev",
|
109
|
+
"pass": "[OAuth token]"
|
110
|
+
},
|
111
|
+
"block_list": [
|
112
|
+
".*secrets.*",
|
113
|
+
".*example.*",
|
114
|
+
".*terraform.*",
|
115
|
+
"\\Agithub/docs\\z"
|
116
|
+
]
|
117
|
+
},
|
118
|
+
"example.com": {
|
119
|
+
"hosting": "gitlab",
|
120
|
+
"api_endpoint": "https://gitlab.example.com",
|
121
|
+
"token": "[OAuth Token]",
|
122
|
+
"allow_list": ["example/site/", "example/backoffice/"],
|
123
|
+
"block_list": [".*secrets.*"]
|
124
|
+
}
|
125
|
+
}
|
126
|
+
}
|
127
|
+
```
|
128
|
+
|
129
|
+
You can specify `allow_list` and `block_list` lists with regular expressions that filter the final list of repositories.
|
130
|
+
|
131
|
+
```sh
|
132
|
+
hound_list_sync --base=hound_base.json --extension=hound_extension.json --out=hound_config.json
|
133
|
+
```
|
134
|
+
|
135
|
+
This will give us a hound configuration to index the organization's repositories https://github.com/github and the list of projects from our server at gitlab.example.com.
|
136
|
+
|
137
|
+
## Development
|
138
|
+
|
139
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
140
|
+
|
141
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
142
|
+
|
143
|
+
## Contributing
|
144
|
+
|
145
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/DmitryBochkarev/hound_list_sync.
|
146
|
+
|
147
|
+
## License
|
148
|
+
|
149
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "bundler/setup"
|
5
|
+
require "hound_list_sync"
|
6
|
+
|
7
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
8
|
+
# with your gem easier. You can also use a different console, if you like.
|
9
|
+
|
10
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
11
|
+
# require "pry"
|
12
|
+
# Pry.start
|
13
|
+
|
14
|
+
require "irb"
|
15
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/exe/hound_list_sync
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require "hound_list_sync"
|
5
|
+
require "hound_list_sync/cli"
|
6
|
+
|
7
|
+
require "logger"
|
8
|
+
require "optparse"
|
9
|
+
|
10
|
+
HELP = <<-TXT
|
11
|
+
Base config is regular hound config:
|
12
|
+
{
|
13
|
+
"dbpath" : "db",
|
14
|
+
"vcs-config" : {
|
15
|
+
"git": {
|
16
|
+
"ref" : "main"
|
17
|
+
}
|
18
|
+
},
|
19
|
+
"repos" : {
|
20
|
+
"Hound" : {
|
21
|
+
"url" : "https://github.com/hound-search/hound.git"
|
22
|
+
}
|
23
|
+
}
|
24
|
+
}
|
25
|
+
|
26
|
+
Extensions configs allow to specify how to enrich base config:
|
27
|
+
{
|
28
|
+
"repos": {
|
29
|
+
"hound_list_sync": {
|
30
|
+
"url": "https://github.com/DmitryBochkarev/hound_list_sync.git"
|
31
|
+
}
|
32
|
+
},
|
33
|
+
"lists": {
|
34
|
+
"wallarm": {
|
35
|
+
"hosting": "github",
|
36
|
+
"org": "wallarm",
|
37
|
+
"credentials": {
|
38
|
+
"login": "DmitryBochkarev",
|
39
|
+
"pass": "[OAuth Token]"
|
40
|
+
}
|
41
|
+
},
|
42
|
+
"example.com": {
|
43
|
+
"hosting": "gitlab",
|
44
|
+
"api_endpoint": "https://gitlab.example.com",
|
45
|
+
"token": "[OAuth Token]",
|
46
|
+
"allow_list": [
|
47
|
+
"example/site/", "example/backoffice/"
|
48
|
+
],
|
49
|
+
"block_list": [
|
50
|
+
".*secrets.*"
|
51
|
+
]
|
52
|
+
}
|
53
|
+
}
|
54
|
+
}
|
55
|
+
TXT
|
56
|
+
|
57
|
+
options = { extensions: [] }
|
58
|
+
opt_parser = OptionParser.new do |opts|
|
59
|
+
opts.banner = "Usage: hound_list_sync [options]"
|
60
|
+
|
61
|
+
opts.on("--base=FILE", "Base Hound Config(required)") do |v|
|
62
|
+
options[:base] = v
|
63
|
+
end
|
64
|
+
|
65
|
+
opts.on("--out=FILE", "Resulting Hound Config(required)") do |v|
|
66
|
+
options[:out] = v
|
67
|
+
end
|
68
|
+
|
69
|
+
opts.on("--extension=FILE", "Extension config, allow to have multiple") do |v|
|
70
|
+
options[:extensions].push(v)
|
71
|
+
end
|
72
|
+
|
73
|
+
opts.on("-h", "--help", "Print this help") do |_v|
|
74
|
+
puts opts
|
75
|
+
puts
|
76
|
+
puts HELP
|
77
|
+
|
78
|
+
exit
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
opt_parser.parse!(ARGV)
|
83
|
+
|
84
|
+
p options
|
85
|
+
unless options[:base]
|
86
|
+
puts "missing --base=FILE"
|
87
|
+
puts opt_parser
|
88
|
+
|
89
|
+
exit(1)
|
90
|
+
end
|
91
|
+
|
92
|
+
unless options[:out]
|
93
|
+
puts "missing --out=FILE"
|
94
|
+
puts opt_parser
|
95
|
+
|
96
|
+
exit(1)
|
97
|
+
end
|
98
|
+
|
99
|
+
logger = Logger.new($stdout)
|
100
|
+
|
101
|
+
HoundListSync::CLI.new(
|
102
|
+
options.fetch(:base),
|
103
|
+
options.fetch(:extensions),
|
104
|
+
options.fetch(:out),
|
105
|
+
logger: logger
|
106
|
+
).run
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "lib/hound_list_sync/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "hound_list_sync"
|
7
|
+
spec.version = HoundListSync::VERSION
|
8
|
+
spec.authors = ["Dmitry Bochkarev"]
|
9
|
+
spec.email = ["dimabochkarev@gmail.com"]
|
10
|
+
|
11
|
+
spec.summary = "Auto generate Hound config from remote lists"
|
12
|
+
spec.description = "Sync Hound repositories with Github organization and Gitlab projects"
|
13
|
+
spec.homepage = "https://github.com/DmitryBochkarev/hound_list_sync"
|
14
|
+
spec.license = "MIT"
|
15
|
+
spec.required_ruby_version = ">= 2.4.0"
|
16
|
+
|
17
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
18
|
+
spec.metadata["source_code_uri"] = "https://github.com/DmitryBochkarev/hound_list_sync"
|
19
|
+
|
20
|
+
spec.add_dependency "faraday", "~> 1.5.1"
|
21
|
+
spec.add_dependency "faraday_middleware", "~> 1.0.0"
|
22
|
+
|
23
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
24
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
|
25
|
+
end
|
26
|
+
spec.bindir = "exe"
|
27
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
28
|
+
spec.require_paths = ["lib"]
|
29
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "hound_list_sync/version"
|
4
|
+
|
5
|
+
module HoundListSync
|
6
|
+
class Error < StandardError; end
|
7
|
+
end
|
8
|
+
|
9
|
+
require_relative "hound_list_sync/http"
|
10
|
+
require_relative "hound_list_sync/repositories"
|
11
|
+
require_relative "hound_list_sync/hound_config"
|
12
|
+
require_relative "hound_list_sync/extension"
|
13
|
+
require_relative "hound_list_sync/extensions"
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
|
5
|
+
module HoundListSync
|
6
|
+
class CLI
|
7
|
+
def initialize(base, extensions, out, logger:)
|
8
|
+
@base = base
|
9
|
+
@extensions = extensions
|
10
|
+
@out = out
|
11
|
+
@logger = logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def run
|
15
|
+
@logger.info("Loading base config #{@base}")
|
16
|
+
base = HoundConfig.new(read_config(@base))
|
17
|
+
|
18
|
+
if @extensions.any?
|
19
|
+
@logger.info("Loading extensions from #{@extensions.join(", ")}")
|
20
|
+
extensions =
|
21
|
+
Extensions.new(
|
22
|
+
@extensions.map { |e| read_config(e) },
|
23
|
+
http: Http::Net.new(logger: @logger)
|
24
|
+
)
|
25
|
+
|
26
|
+
@logger.info("Extending base config")
|
27
|
+
base.extend_with(extensions)
|
28
|
+
end
|
29
|
+
|
30
|
+
@logger.info("Total repositories: #{base.total_repos}")
|
31
|
+
|
32
|
+
@logger.info("Generating new config")
|
33
|
+
config = JSON.pretty_generate(base)
|
34
|
+
old_config = File.read(@out) if File.exist?(@out)
|
35
|
+
|
36
|
+
if config == old_config
|
37
|
+
@logger.info("Config not changed")
|
38
|
+
else
|
39
|
+
@logger.info("Saving to #{@out}")
|
40
|
+
File.write(@out, config)
|
41
|
+
end
|
42
|
+
|
43
|
+
@logger.info("Done")
|
44
|
+
end
|
45
|
+
|
46
|
+
def read_config(file)
|
47
|
+
unless File.exist?(file)
|
48
|
+
@logger.error("File missing: #{file}")
|
49
|
+
exit(1)
|
50
|
+
end
|
51
|
+
|
52
|
+
JSON.parse(File.read(file))
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HoundListSync
|
4
|
+
class Extension
|
5
|
+
def initialize(conf, http:)
|
6
|
+
@conf = conf
|
7
|
+
@http = http
|
8
|
+
end
|
9
|
+
|
10
|
+
def repositories
|
11
|
+
repositories = []
|
12
|
+
|
13
|
+
repositories.push(Repositories::Raw.new(@conf["repos"])) if @conf.key?("repos")
|
14
|
+
|
15
|
+
if @conf.key?("lists")
|
16
|
+
repositories.push(
|
17
|
+
Repositories::Join.new(
|
18
|
+
@conf["lists"].map { |name, conf| list_repositories(name, conf) }
|
19
|
+
)
|
20
|
+
)
|
21
|
+
end
|
22
|
+
|
23
|
+
Repositories::Join.new(repositories)
|
24
|
+
end
|
25
|
+
|
26
|
+
def list_repositories(name, conf) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
27
|
+
repositories =
|
28
|
+
case conf["hosting"]
|
29
|
+
when "github"
|
30
|
+
Repositories::GithubOrg.new(
|
31
|
+
conf.fetch("org"),
|
32
|
+
http: @http,
|
33
|
+
api_endpoint: conf["api_endpoint"],
|
34
|
+
credentials: conf.fetch("credentials", {}).transform_keys(&:to_sym)
|
35
|
+
)
|
36
|
+
when "gitlab"
|
37
|
+
Repositories::Gitlab.new(
|
38
|
+
conf.fetch("api_endpoint"),
|
39
|
+
http: @http,
|
40
|
+
token: conf["token"]
|
41
|
+
)
|
42
|
+
else
|
43
|
+
raise "Invalid config #{name}: #{conf}"
|
44
|
+
end
|
45
|
+
|
46
|
+
if conf["allow_list"].is_a?(Array) && conf["allow_list"]&.any?
|
47
|
+
repositories = Repositories::AllowList.new(
|
48
|
+
repositories,
|
49
|
+
names: conf["allow_list"]
|
50
|
+
)
|
51
|
+
end
|
52
|
+
|
53
|
+
if conf["block_list"].is_a?(Array) && conf["block_list"]&.any?
|
54
|
+
repositories = Repositories::BlockList.new(
|
55
|
+
repositories,
|
56
|
+
names: conf["block_list"]
|
57
|
+
)
|
58
|
+
end
|
59
|
+
|
60
|
+
repositories
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HoundListSync
|
4
|
+
class Extensions
|
5
|
+
def initialize(raw, http:)
|
6
|
+
@raw = raw
|
7
|
+
@http = http
|
8
|
+
end
|
9
|
+
|
10
|
+
def repositories
|
11
|
+
Repositories::Join.new(
|
12
|
+
@raw.map { |conf| extension(conf).repositories }
|
13
|
+
)
|
14
|
+
end
|
15
|
+
|
16
|
+
def extension(conf)
|
17
|
+
Extension.new(conf, http: @http)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HoundListSync
|
4
|
+
class HoundConfig
|
5
|
+
def initialize(raw)
|
6
|
+
@raw = raw
|
7
|
+
@raw["repos"] ||= {}
|
8
|
+
end
|
9
|
+
|
10
|
+
def extend_with(extensions)
|
11
|
+
extensions.repositories.each do |repo|
|
12
|
+
next unless repo.indexable?
|
13
|
+
|
14
|
+
@raw["repos"][repo.name] = repo.to_config
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def to_json(*args)
|
19
|
+
@raw.to_json(*args)
|
20
|
+
end
|
21
|
+
|
22
|
+
def total_repos
|
23
|
+
@raw["repos"].keys.length
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "faraday"
|
4
|
+
require "faraday_middleware"
|
5
|
+
|
6
|
+
module HoundListSync
|
7
|
+
module Http
|
8
|
+
class Response
|
9
|
+
attr_reader :status, :headers, :body
|
10
|
+
|
11
|
+
def initialize(status, headers, body)
|
12
|
+
@status = status
|
13
|
+
@headers = headers
|
14
|
+
@body = body
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def get(_url, headers:, basic_auth: []) # rubocop:disable Lint/UnusedMethodArgument
|
19
|
+
raise "#{self.class} has not implemented method '#{__method__}'"
|
20
|
+
end
|
21
|
+
|
22
|
+
class Net
|
23
|
+
include Http
|
24
|
+
|
25
|
+
HEADERS = {
|
26
|
+
"User-Agent" => "Hound-List-Sync/#{HoundListSync::VERSION} Faraday/#{Faraday::VERSION} ruby/#{RUBY_VERSION}"
|
27
|
+
}.freeze
|
28
|
+
|
29
|
+
def initialize(logger: nil)
|
30
|
+
@logger = logger
|
31
|
+
end
|
32
|
+
|
33
|
+
def get(url, headers:, basic_auth: [])
|
34
|
+
conn = Faraday.new(url: url, headers: HEADERS.merge(headers)) do |faraday|
|
35
|
+
faraday.basic_auth(*basic_auth) if basic_auth.any?
|
36
|
+
|
37
|
+
faraday.request :retry
|
38
|
+
|
39
|
+
faraday.response :follow_redirects
|
40
|
+
faraday.response :raise_error
|
41
|
+
|
42
|
+
if @logger
|
43
|
+
faraday.response :logger, @logger do |logger|
|
44
|
+
logger.filter(/(Authorization: )(.+)/i, '\1[REMOVED]')
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
resp = conn.get
|
50
|
+
|
51
|
+
Response.new(resp.status, resp.headers, resp.body)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HoundListSync
|
4
|
+
module Http
|
5
|
+
class Fake
|
6
|
+
include Http
|
7
|
+
|
8
|
+
Error = Class.new(StandardError)
|
9
|
+
NoMoreResponses = Class.new(Error)
|
10
|
+
|
11
|
+
class Get
|
12
|
+
attr_reader :url, :headers, :basic_auth, :response
|
13
|
+
|
14
|
+
def initialize(url, headers:, basic_auth:, response:)
|
15
|
+
@url = url
|
16
|
+
@headers = headers
|
17
|
+
@basic_auth = basic_auth
|
18
|
+
@response = response
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
attr_reader :requests, :responses
|
23
|
+
|
24
|
+
def initialize(responses)
|
25
|
+
@requests = []
|
26
|
+
@responses = responses
|
27
|
+
end
|
28
|
+
|
29
|
+
def get(url, headers:, basic_auth: [])
|
30
|
+
response = @responses.shift
|
31
|
+
|
32
|
+
raise NoMoreResponses, "no more response for GET #{url}" unless response
|
33
|
+
|
34
|
+
@requests.push(Get.new(url, headers: headers, basic_auth: basic_auth, response: response))
|
35
|
+
|
36
|
+
response
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HoundListSync
|
4
|
+
module Repositories
|
5
|
+
module Repo
|
6
|
+
def name
|
7
|
+
raise "#{self.class} has not implemented method '#{__method__}'"
|
8
|
+
end
|
9
|
+
|
10
|
+
def to_config
|
11
|
+
raise "#{self.class} has not implemented method '#{__method__}'"
|
12
|
+
end
|
13
|
+
|
14
|
+
def indexable?
|
15
|
+
raise "#{self.class} has not implemented method '#{__method__}'"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
require_relative "repositories/github_org"
|
22
|
+
require_relative "repositories/gitlab"
|
23
|
+
require_relative "repositories/allow_list"
|
24
|
+
require_relative "repositories/block_list"
|
25
|
+
require_relative "repositories/join"
|
26
|
+
require_relative "repositories/raw"
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HoundListSync
|
4
|
+
module Repositories
|
5
|
+
class AllowList
|
6
|
+
include Repositories
|
7
|
+
include Enumerable
|
8
|
+
|
9
|
+
def initialize(original, names: [])
|
10
|
+
@original = original
|
11
|
+
@names = names.map { |n| ::Regexp.new(n, ::Regexp::MULTILINE | ::Regexp::IGNORECASE) }
|
12
|
+
end
|
13
|
+
|
14
|
+
def each
|
15
|
+
return to_enum unless block_given?
|
16
|
+
|
17
|
+
@original.each do |repo|
|
18
|
+
next unless @names.any? { |name| name.match?(repo.name) }
|
19
|
+
|
20
|
+
yield repo
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HoundListSync
|
4
|
+
module Repositories
|
5
|
+
class BlockList
|
6
|
+
include Repositories
|
7
|
+
include Enumerable
|
8
|
+
|
9
|
+
def initialize(original, names: [])
|
10
|
+
@original = original
|
11
|
+
@names = names.map { |n| ::Regexp.new(n, ::Regexp::MULTILINE | ::Regexp::IGNORECASE) }
|
12
|
+
end
|
13
|
+
|
14
|
+
def each
|
15
|
+
return to_enum unless block_given?
|
16
|
+
|
17
|
+
@original.each do |repo|
|
18
|
+
next if @names.any? { |name| name.match?(repo.name) }
|
19
|
+
|
20
|
+
yield repo
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
|
5
|
+
module HoundListSync
|
6
|
+
module Repositories
|
7
|
+
class GithubOrg
|
8
|
+
include Repositories
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
URL_TEMPLATE = "%{base}/orgs/%{org}/repos?page=%{page}"
|
12
|
+
API_GITHUB_COM = "https://api.github.com"
|
13
|
+
HEADERS = { "Accept" => "application/vnd.github.v3+json" }.freeze
|
14
|
+
|
15
|
+
class Repo
|
16
|
+
include Repositories::Repo
|
17
|
+
|
18
|
+
def initialize(raw)
|
19
|
+
@raw = raw
|
20
|
+
end
|
21
|
+
|
22
|
+
def indexable?
|
23
|
+
!@raw["archived"] && !@raw["disabled"]
|
24
|
+
end
|
25
|
+
|
26
|
+
def name
|
27
|
+
@raw["full_name"]
|
28
|
+
end
|
29
|
+
|
30
|
+
def to_config
|
31
|
+
{
|
32
|
+
"url" => url,
|
33
|
+
"url-pattern" => {
|
34
|
+
"base-url" => base_url,
|
35
|
+
"anchor" => anchor
|
36
|
+
},
|
37
|
+
"vcs-config" => {
|
38
|
+
"ref" => branch
|
39
|
+
}
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
def url
|
44
|
+
@raw["ssh_url"]
|
45
|
+
end
|
46
|
+
|
47
|
+
def branch
|
48
|
+
@raw["default_branch"] || "master"
|
49
|
+
end
|
50
|
+
|
51
|
+
def base_url
|
52
|
+
"#{@raw["html_url"]}/blob/#{branch}/{path}{anchor}"
|
53
|
+
end
|
54
|
+
|
55
|
+
def anchor
|
56
|
+
"#L{line}"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def initialize(
|
61
|
+
orgname,
|
62
|
+
api_endpoint: nil,
|
63
|
+
credentials: {},
|
64
|
+
url_template: URL_TEMPLATE,
|
65
|
+
http: HoundListSync::Http::Net.new
|
66
|
+
)
|
67
|
+
@orgname = orgname
|
68
|
+
@api_endpoint = api_endpoint || API_GITHUB_COM
|
69
|
+
@credentials = credentials
|
70
|
+
@url_template = url_template
|
71
|
+
@http = http
|
72
|
+
end
|
73
|
+
|
74
|
+
def each
|
75
|
+
return to_enum unless block_given?
|
76
|
+
|
77
|
+
page = 1
|
78
|
+
|
79
|
+
loop do
|
80
|
+
repos =
|
81
|
+
JSON.parse(
|
82
|
+
@http
|
83
|
+
.get(url(page), headers: HEADERS, basic_auth: basic_auth)
|
84
|
+
.body
|
85
|
+
)
|
86
|
+
|
87
|
+
break if repos.length.zero?
|
88
|
+
|
89
|
+
repos.each do |repo|
|
90
|
+
yield Repo.new(repo)
|
91
|
+
end
|
92
|
+
|
93
|
+
page += 1
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def url(page)
|
98
|
+
format(@url_template, base: @api_endpoint, org: @orgname, page: page)
|
99
|
+
end
|
100
|
+
|
101
|
+
def basic_auth
|
102
|
+
@basic_auth ||=
|
103
|
+
if @credentials.any?
|
104
|
+
[@credentials.fetch(:login), @credentials.fetch(:pass)]
|
105
|
+
else
|
106
|
+
[]
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "json"
|
4
|
+
|
5
|
+
module HoundListSync
|
6
|
+
module Repositories
|
7
|
+
class Gitlab
|
8
|
+
include Repositories
|
9
|
+
include Enumerable
|
10
|
+
|
11
|
+
URL_TEMPLATE = "%{base}/api/v4/projects?page=%{page}"
|
12
|
+
|
13
|
+
class Repo
|
14
|
+
include Repositories::Repo
|
15
|
+
|
16
|
+
def initialize(raw)
|
17
|
+
@raw = raw
|
18
|
+
end
|
19
|
+
|
20
|
+
def indexable?
|
21
|
+
!@raw["archived"] && !@raw["empty_repo"]
|
22
|
+
end
|
23
|
+
|
24
|
+
def name
|
25
|
+
@raw["path_with_namespace"]
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_config
|
29
|
+
{
|
30
|
+
"url" => url,
|
31
|
+
"url-pattern" => {
|
32
|
+
"base-url" => base_url,
|
33
|
+
"anchor" => anchor
|
34
|
+
},
|
35
|
+
"vcs-config" => {
|
36
|
+
"ref" => branch
|
37
|
+
}
|
38
|
+
}
|
39
|
+
end
|
40
|
+
|
41
|
+
def url
|
42
|
+
@raw["ssh_url_to_repo"]
|
43
|
+
end
|
44
|
+
|
45
|
+
def branch
|
46
|
+
@raw["default_branch"] || "master"
|
47
|
+
end
|
48
|
+
|
49
|
+
def base_url
|
50
|
+
"#{@raw["web_url"]}/blob/#{branch}/{path}{anchor}"
|
51
|
+
end
|
52
|
+
|
53
|
+
def anchor
|
54
|
+
"#L{line}"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def initialize(
|
59
|
+
api_endpoint,
|
60
|
+
token: nil,
|
61
|
+
url_template: URL_TEMPLATE,
|
62
|
+
http: HoundListSync::Http::Net.new
|
63
|
+
)
|
64
|
+
@api_endpoint = api_endpoint
|
65
|
+
@token = token
|
66
|
+
@url_template = url_template
|
67
|
+
@http = http
|
68
|
+
end
|
69
|
+
|
70
|
+
def each
|
71
|
+
return to_enum unless block_given?
|
72
|
+
|
73
|
+
page = "1"
|
74
|
+
|
75
|
+
loop do
|
76
|
+
response = @http.get(url(page), headers: auth_headers)
|
77
|
+
|
78
|
+
repos = JSON.parse(response.body)
|
79
|
+
|
80
|
+
break if repos.length.zero?
|
81
|
+
|
82
|
+
repos.each do |repo|
|
83
|
+
yield Repo.new(repo)
|
84
|
+
end
|
85
|
+
|
86
|
+
next_page = response.headers["x-next-page"]
|
87
|
+
break if next_page.empty?
|
88
|
+
|
89
|
+
page = next_page
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def url(page)
|
94
|
+
format(@url_template, base: @api_endpoint, page: page)
|
95
|
+
end
|
96
|
+
|
97
|
+
def auth_headers
|
98
|
+
@auth_headers ||=
|
99
|
+
if @token.nil?
|
100
|
+
{}
|
101
|
+
else
|
102
|
+
{ "Authorization" => "Bearer #{@token}" }
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HoundListSync
|
4
|
+
module Repositories
|
5
|
+
class Join
|
6
|
+
include Repositories
|
7
|
+
include Enumerable
|
8
|
+
|
9
|
+
def initialize(list)
|
10
|
+
@list = list
|
11
|
+
end
|
12
|
+
|
13
|
+
def each(&block)
|
14
|
+
return to_enum unless block_given?
|
15
|
+
|
16
|
+
@list.each do |repositories|
|
17
|
+
repositories.each(&block)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HoundListSync
|
4
|
+
module Repositories
|
5
|
+
class Raw
|
6
|
+
include Repositories
|
7
|
+
include Enumerable
|
8
|
+
|
9
|
+
class Repo
|
10
|
+
include Repositories::Repo
|
11
|
+
|
12
|
+
def initialize(name, config)
|
13
|
+
@name = name
|
14
|
+
@config = config
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :name
|
18
|
+
|
19
|
+
def to_config
|
20
|
+
@config
|
21
|
+
end
|
22
|
+
|
23
|
+
def indexable?
|
24
|
+
true
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def initialize(raw)
|
29
|
+
@raw = raw
|
30
|
+
end
|
31
|
+
|
32
|
+
def each
|
33
|
+
return to_enum unless block_given?
|
34
|
+
|
35
|
+
@raw.each do |name, config|
|
36
|
+
yield Repo.new(name, config)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
metadata
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: hound_list_sync
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Dmitry Bochkarev
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2021-07-25 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: faraday
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.5.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.5.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: faraday_middleware
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 1.0.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 1.0.0
|
41
|
+
description: Sync Hound repositories with Github organization and Gitlab projects
|
42
|
+
email:
|
43
|
+
- dimabochkarev@gmail.com
|
44
|
+
executables:
|
45
|
+
- hound_list_sync
|
46
|
+
extensions: []
|
47
|
+
extra_rdoc_files: []
|
48
|
+
files:
|
49
|
+
- ".github/workflows/main.yml"
|
50
|
+
- ".gitignore"
|
51
|
+
- ".rspec"
|
52
|
+
- ".rubocop.yml"
|
53
|
+
- Gemfile
|
54
|
+
- LICENSE.txt
|
55
|
+
- README.md
|
56
|
+
- Rakefile
|
57
|
+
- bin/console
|
58
|
+
- bin/setup
|
59
|
+
- exe/hound_list_sync
|
60
|
+
- hound_list_sync.gemspec
|
61
|
+
- lib/hound_list_sync.rb
|
62
|
+
- lib/hound_list_sync/cli.rb
|
63
|
+
- lib/hound_list_sync/extension.rb
|
64
|
+
- lib/hound_list_sync/extensions.rb
|
65
|
+
- lib/hound_list_sync/hound_config.rb
|
66
|
+
- lib/hound_list_sync/http.rb
|
67
|
+
- lib/hound_list_sync/http/fake.rb
|
68
|
+
- lib/hound_list_sync/repositories.rb
|
69
|
+
- lib/hound_list_sync/repositories/allow_list.rb
|
70
|
+
- lib/hound_list_sync/repositories/block_list.rb
|
71
|
+
- lib/hound_list_sync/repositories/github_org.rb
|
72
|
+
- lib/hound_list_sync/repositories/gitlab.rb
|
73
|
+
- lib/hound_list_sync/repositories/join.rb
|
74
|
+
- lib/hound_list_sync/repositories/raw.rb
|
75
|
+
- lib/hound_list_sync/version.rb
|
76
|
+
homepage: https://github.com/DmitryBochkarev/hound_list_sync
|
77
|
+
licenses:
|
78
|
+
- MIT
|
79
|
+
metadata:
|
80
|
+
homepage_uri: https://github.com/DmitryBochkarev/hound_list_sync
|
81
|
+
source_code_uri: https://github.com/DmitryBochkarev/hound_list_sync
|
82
|
+
post_install_message:
|
83
|
+
rdoc_options: []
|
84
|
+
require_paths:
|
85
|
+
- lib
|
86
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: 2.4.0
|
91
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
92
|
+
requirements:
|
93
|
+
- - ">="
|
94
|
+
- !ruby/object:Gem::Version
|
95
|
+
version: '0'
|
96
|
+
requirements: []
|
97
|
+
rubygems_version: 3.0.8
|
98
|
+
signing_key:
|
99
|
+
specification_version: 4
|
100
|
+
summary: Auto generate Hound config from remote lists
|
101
|
+
test_files: []
|