github-cloner 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +18 -0
- data/.rubocop.yml +10 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +2 -0
- data/LICENSE +22 -0
- data/README.md +199 -0
- data/Rakefile +26 -0
- data/bin/github-cloner +59 -0
- data/github_cloner.gemspec +44 -0
- data/lib/github_cloner/filename_cleaner.rb +98 -0
- data/lib/github_cloner/github_cloner.rb +107 -0
- data/lib/github_cloner/opt_parser.rb +95 -0
- data/lib/github_cloner/version.rb +3 -0
- data/lib/github_cloner.rb +6 -0
- data/test/lib/github_copier/test_github_cloner.rb +8 -0
- data/test/test_helper.rb +8 -0
- metadata +249 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 4f23c7164f8ba19e237369bd2d5b52d6b9652c43
|
4
|
+
data.tar.gz: 1fadfc3cea2dc1fe7fdf1d2d5d6ca5a6d68c2302
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: aabe5efd932df2e1eb11b77042a525c249a2ab755b9d319f5014f4285360095e5d57e655e2454a6a1beb8b93b498d17858847f89405ab8e10178c747617d6aaf
|
7
|
+
data.tar.gz: 4ab3ce865ff77eeb25ef2cbd2657156e69aeaeadb731e5b65e5a844f2a4329a97d37ee79eec48aada6e4348f2488782aa203c417c1bec30b51f2dd320623e38e
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2016 Burin Choomnuan
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,199 @@
|
|
1
|
+
## GithubCloner
|
2
|
+
|
3
|
+
Export/clone Github repository (public/private) by user or organization name
|
4
|
+
|
5
|
+
[][gem]
|
6
|
+
[][gemnasium]
|
7
|
+
|
8
|
+
[gem]: http://badge.fury.io/rb/github-cloner
|
9
|
+
[gemnasium]: https://gemnasium.com/agilecreativity/github-cloner
|
10
|
+
|
11
|
+
### Start with Why
|
12
|
+
|
13
|
+
- Be able to get grab all of interesting codes from a given user quickly
|
14
|
+
- Be able to clone specific language for a given user/organization
|
15
|
+
- Be able to clone all of organization/users in one go if desired
|
16
|
+
- Be able to clone private/public repositories for a given user/organization
|
17
|
+
- Be able to learn quickly or look at how something works from the source code
|
18
|
+
- I am too lazy to do this manually or through some other shell script
|
19
|
+
|
20
|
+
### Installation
|
21
|
+
|
22
|
+
```sh
|
23
|
+
# Install the gem
|
24
|
+
gem install github-cloner
|
25
|
+
|
26
|
+
# refresh your gem just in case
|
27
|
+
rbenv rehash
|
28
|
+
|
29
|
+
# Get list of options just type the name of the gem
|
30
|
+
github-cloner
|
31
|
+
```
|
32
|
+
|
33
|
+
You should see something like
|
34
|
+
|
35
|
+
```
|
36
|
+
Usage: github-cloner [options]
|
37
|
+
|
38
|
+
Specific options:
|
39
|
+
-b, --base-dir BASE_DIR where BASE_DIR is the directory where the repositories will be cloned to (mandatory)
|
40
|
+
If not specified, current directory will be used
|
41
|
+
-u, --user USER The Github USER that will be cloned from (mandatory)
|
42
|
+
-o, --org [ORG] The Github's organization name to be used if specified (optional)
|
43
|
+
where ORG is the organization that the user belongs to
|
44
|
+
-t, --oauth-token [OAUTH_TOKEN] The Github's oauth_token for authentication (optional - only required to list/clone private repositories)
|
45
|
+
where OAUTH_TOKEN is from the user's Github setting
|
46
|
+
-l, --language [LANG] Clone only project of type LANG (optional)
|
47
|
+
where LANG is main language as shown on Github
|
48
|
+
-a, --[no-]all-repos All repository only (optional)
|
49
|
+
default to original/non-forked repositories only
|
50
|
+
-g, --[no-]group-by-user Group the output by {BASE_DIR}/{USER}/{LANG}
|
51
|
+
default to {BASE_DIR}/{LANG}/{USER}
|
52
|
+
-c, --[no-]clone Clone the repositories to the path specified (optional)
|
53
|
+
default to --no-clone e.g. dry-run only
|
54
|
+
|
55
|
+
Common options:
|
56
|
+
-h, --help Show this message
|
57
|
+
```
|
58
|
+
|
59
|
+
- List repositories by user or organization id
|
60
|
+
|
61
|
+
```sh
|
62
|
+
# List all original/non-forked repositories by `awesome_user`
|
63
|
+
$github-cloner --user awesome_user
|
64
|
+
|
65
|
+
## List all original/non-forked repositories of a user `awesome_user` that belongs to `AwesomeCo`
|
66
|
+
$github-cloner --user awesome_user --org AwesomeCo
|
67
|
+
|
68
|
+
## List all origina/non-forked repositories by user `awesome_user` including private repository
|
69
|
+
# Note: for this to work you will need to have the proper access with the right token
|
70
|
+
$github-cloner --user awesome_user --oauth-token GITHUB_TOKEN_FOR_THIS_USER
|
71
|
+
|
72
|
+
## List all repositories by user `awesome_user` include forked repositories
|
73
|
+
$github-cloner --user awesome_user --oauth-token GITHUB_TOKEN_FOR_THIS_USER
|
74
|
+
```
|
75
|
+
|
76
|
+
- List and clone repositories by user or organization id using `--clone` option
|
77
|
+
|
78
|
+
```sh
|
79
|
+
## Clone all original (non-fork) public `JavaScript` repositores for user `awesome_user` to `~/Desktop/github`
|
80
|
+
# Note: --base-dir is optional, if not specified then the current directory will be used
|
81
|
+
# --language must be quoted if the value include any spaces e.g. "Emacs Lisp" for this to to work properly
|
82
|
+
$github-cloner --user awesome_user \
|
83
|
+
--base-dir ~/Desktop/github \
|
84
|
+
--language "JavaScript" \
|
85
|
+
--clone
|
86
|
+
|
87
|
+
## Clone all public/private repositories for `awesome_user` which are member of `AwesomeCo` organization to `~/Desktop/github`
|
88
|
+
# Note: the option `--all` to include all forked repositories
|
89
|
+
$github-cloner --user awesome_user \
|
90
|
+
--org AwesomeCo \
|
91
|
+
--all-repos \
|
92
|
+
--base-dir ~/Desktop/github \
|
93
|
+
--oauth-token GITHUB_TOKEN_FOR_AWESOME_USER \
|
94
|
+
--clone
|
95
|
+
|
96
|
+
## Clone specific type of project (e.g. `Java` in this case) public/private repositories for `awesome_user`
|
97
|
+
## which are member of `AwesomeCo` organization to `~/Desktop/github`
|
98
|
+
$github-cloner --user awesome_user \
|
99
|
+
--org AwesomeCo \
|
100
|
+
--all-repos \
|
101
|
+
--language "Java" \
|
102
|
+
--base-dir ~/Desktop/github \
|
103
|
+
--oauth-token GITHUB_TOKEN_FOR_AWESOME_USER \
|
104
|
+
--clone
|
105
|
+
```
|
106
|
+
|
107
|
+
### Example Sessions
|
108
|
+
|
109
|
+
- List repositories by a given user (dry-run)
|
110
|
+
|
111
|
+
```
|
112
|
+
$github-cloner -b ~/Desktop/projects -u littlebee -l Ruby
|
113
|
+
------------------------------------------
|
114
|
+
List of languages by littlebee
|
115
|
+
Makefile
|
116
|
+
CoffeeScript
|
117
|
+
Ruby
|
118
|
+
JavaScript
|
119
|
+
Arduino
|
120
|
+
------------------------------------------
|
121
|
+
------------------------------------------
|
122
|
+
List of all repositories by littlebee
|
123
|
+
1/15: littlebee/Makefile/arduino-mk
|
124
|
+
2/15: littlebee/CoffeeScript/bumble-build
|
125
|
+
3/15: littlebee/CoffeeScript/bumble-docs
|
126
|
+
4/15: littlebee/CoffeeScript/bumble-strings
|
127
|
+
5/15: littlebee/CoffeeScript/bumble-test
|
128
|
+
6/15: littlebee/CoffeeScript/bumble-util
|
129
|
+
7/15: littlebee/CoffeeScript/git-log-utils
|
130
|
+
8/15: littlebee/CoffeeScript/git-status-utils
|
131
|
+
9/15: littlebee/CoffeeScript/git-time-machine
|
132
|
+
10/15: littlebee/CoffeeScript/notjs
|
133
|
+
11/15: littlebee/CoffeeScript/publish
|
134
|
+
12/15: littlebee/CoffeeScript/react-focus-trap-amd
|
135
|
+
13/15: littlebee/Ruby/got
|
136
|
+
14/15: littlebee/JavaScript/selectable-collection
|
137
|
+
15/15: littlebee/Arduino/solar-sunflower
|
138
|
+
------------------------------------------
|
139
|
+
FYI: dry-run only, no action taken!!
|
140
|
+
Process 1 of 1 : git clone git@github.com:littlebee/got.git /Users/bchoomnuan/Desktop/projects/Ruby/littlebee/got
|
141
|
+
```
|
142
|
+
|
143
|
+
- List and clone repositories for a given user (e.g. `--clone` option used)
|
144
|
+
|
145
|
+
```
|
146
|
+
$github-cloner -b ~/Desktop/projects -u littlebee -l Ruby -c
|
147
|
+
------------------------------------------
|
148
|
+
List of languages by littlebee
|
149
|
+
Makefile
|
150
|
+
CoffeeScript
|
151
|
+
Ruby
|
152
|
+
JavaScript
|
153
|
+
Arduino
|
154
|
+
------------------------------------------
|
155
|
+
------------------------------------------
|
156
|
+
List of all repositories by littlebee
|
157
|
+
1/15: littlebee/Makefile/arduino-mk
|
158
|
+
2/15: littlebee/CoffeeScript/bumble-build
|
159
|
+
3/15: littlebee/CoffeeScript/bumble-docs
|
160
|
+
4/15: littlebee/CoffeeScript/bumble-strings
|
161
|
+
5/15: littlebee/CoffeeScript/bumble-test
|
162
|
+
6/15: littlebee/CoffeeScript/bumble-util
|
163
|
+
7/15: littlebee/CoffeeScript/git-log-utils
|
164
|
+
8/15: littlebee/CoffeeScript/git-status-utils
|
165
|
+
9/15: littlebee/CoffeeScript/git-time-machine
|
166
|
+
10/15: littlebee/CoffeeScript/notjs
|
167
|
+
11/15: littlebee/CoffeeScript/publish
|
168
|
+
12/15: littlebee/CoffeeScript/react-focus-trap-amd
|
169
|
+
13/15: littlebee/Ruby/got
|
170
|
+
14/15: littlebee/JavaScript/selectable-collection
|
171
|
+
15/15: littlebee/Arduino/solar-sunflower
|
172
|
+
------------------------------------------
|
173
|
+
Process 1 of 1 : git clone git@github.com:littlebee/got.git /Users/bchoomnuan/Desktop/projects/Ruby/littlebee/got
|
174
|
+
```
|
175
|
+
|
176
|
+
### TODO
|
177
|
+
|
178
|
+
- Replace system call with the ruby library like [grit](https://github.com/mojombo/grit) or something similar
|
179
|
+
- Allow the `https` when performing the clone
|
180
|
+
|
181
|
+
### Related Projects
|
182
|
+
|
183
|
+
- [github_cloner](https://github.com/nashby/github_cloner) by Vasiliy Ermolovich
|
184
|
+
|
185
|
+
### Contributing
|
186
|
+
|
187
|
+
1. Fork it
|
188
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
189
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
190
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
191
|
+
5. Create new Pull Request
|
192
|
+
|
193
|
+
[Thor]: https://github.com/erikhuda/thor
|
194
|
+
[Minitest]: https://github.com/seattlerb/minitest
|
195
|
+
[RSpec]: https://github.com/rspec
|
196
|
+
[Guard]: https://github.com/guard/guard
|
197
|
+
[Yard]: https://github.com/lsegal/yard
|
198
|
+
[Pry]: https://github.com/pry/pry
|
199
|
+
[Rubocop]: https://github.com/bbatsov/rubocop
|
data/Rakefile
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'bundler/gem_tasks'
|
2
|
+
require 'rake/testtask'
|
3
|
+
project_name = 'github_cloner'
|
4
|
+
Rake::TestTask.new do |t|
|
5
|
+
t.libs << "lib/#{project_name}"
|
6
|
+
t.test_files = FileList["test/lib/#{project_name}/test_*.rb"]
|
7
|
+
t.verbose = true
|
8
|
+
end
|
9
|
+
task default: [:test, :rubocop]
|
10
|
+
task :pry do
|
11
|
+
require 'pry'
|
12
|
+
require 'awesome_print'
|
13
|
+
require_relative 'lib/github_cloner'
|
14
|
+
include GithubCloner
|
15
|
+
ARGV.clear
|
16
|
+
Pry.start
|
17
|
+
end
|
18
|
+
require 'rubocop/rake_task'
|
19
|
+
desc 'Run RuboCop on the lib directory'
|
20
|
+
RuboCop::RakeTask.new(:rubocop) do |task|
|
21
|
+
task.patterns = ['lib/**/*.rb']
|
22
|
+
# only show the files with failures
|
23
|
+
task.formatters = ['files']
|
24
|
+
# don't abort rake on failure
|
25
|
+
task.fail_on_error = false
|
26
|
+
end
|
data/bin/github-cloner
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require_relative '../lib/github_cloner'
|
3
|
+
|
4
|
+
if ARGV.empty?
|
5
|
+
OptParser.parse(%w[--help])
|
6
|
+
end
|
7
|
+
|
8
|
+
include GithubCloner
|
9
|
+
|
10
|
+
# Now we are ready to take the order
|
11
|
+
# Show help if user forgot to type anything
|
12
|
+
options = OptParser.parse(ARGV)
|
13
|
+
|
14
|
+
# Mandatory options
|
15
|
+
raise "base_dir is required" if options.base_dir.nil?
|
16
|
+
raise "user is required" if options.user.nil?
|
17
|
+
|
18
|
+
# Make use of the user name
|
19
|
+
gh_id = options.user
|
20
|
+
|
21
|
+
# Note: set to specific language to limit the cloning
|
22
|
+
if options.language
|
23
|
+
language = options.language #'Emacs Lisp'
|
24
|
+
end
|
25
|
+
|
26
|
+
args = {
|
27
|
+
base_dir: options.base_dir, # mandatory
|
28
|
+
oauth_token: options.oauth_token, # optional required for private repos
|
29
|
+
user: options.user, # optional if org is used?
|
30
|
+
org: options.org, # optional if user is used?
|
31
|
+
all_repos: options.all_repos, # default to false
|
32
|
+
group_by_user: options.group_by_user, # default to false
|
33
|
+
clone_repos: options.clone_repos # default to false
|
34
|
+
}
|
35
|
+
|
36
|
+
repos = GithubCloner::GithubUtils.list_all(args)
|
37
|
+
|
38
|
+
# List of language that have been used by the given user/organization
|
39
|
+
languages = repos.map(&:language).uniq.compact
|
40
|
+
puts "------------------------------------------"
|
41
|
+
puts "List of languages by #{gh_id}"
|
42
|
+
puts languages
|
43
|
+
puts "------------------------------------------"
|
44
|
+
|
45
|
+
repos_hash = GithubCloner::GithubUtils.repos_by_language(repos, languages)
|
46
|
+
|
47
|
+
result = GithubCloner::GithubUtils.repos_as_list(repos_hash)
|
48
|
+
|
49
|
+
if result && !result.empty?
|
50
|
+
puts "------------------------------------------"
|
51
|
+
puts "List of all repositories by #{gh_id}"
|
52
|
+
result.each_with_index do |r,i|
|
53
|
+
puts "#{i+1}/#{result.size}: #{r}"
|
54
|
+
end
|
55
|
+
puts "------------------------------------------"
|
56
|
+
|
57
|
+
result.keep_if { |i| i.split(File::SEPARATOR)[1] == language } if language
|
58
|
+
GithubCloner::GithubUtils.clone_all(result,args)
|
59
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'github_cloner/version'
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = 'github-cloner'
|
7
|
+
spec.version = GithubCloner::VERSION
|
8
|
+
spec.authors = ['Burin Choomnuan']
|
9
|
+
spec.email = ['agilecreativity@gmail.com']
|
10
|
+
spec.summary = %q{Clone/list Github repository for a given user/organization include private/public in one go}
|
11
|
+
spec.description = %q(
|
12
|
+
Clone/list multiple Github repositories for a given user/organization including private repos easily.
|
13
|
+
e.g.$github-cloner --base-dir ~/projects --user awesome_dev --language "Emacs Lisp" --clone
|
14
|
+
).gsub(/^\s+/, " ")
|
15
|
+
spec.homepage = 'https://github.com/agilecreativity/github-cloner'
|
16
|
+
spec.required_ruby_version = ">= 2.0.0"
|
17
|
+
spec.license = 'MIT'
|
18
|
+
spec.files = Dir.glob('{bin,lib,spec,test}/**/*') + %w(Gemfile
|
19
|
+
Rakefile
|
20
|
+
github_cloner.gemspec
|
21
|
+
README.md
|
22
|
+
CHANGELOG.md
|
23
|
+
LICENSE
|
24
|
+
.rubocop.yml
|
25
|
+
.gitignore)
|
26
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
27
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
28
|
+
spec.require_paths = ['lib']
|
29
|
+
|
30
|
+
spec.add_runtime_dependency 'github_api', '~> 0.13'
|
31
|
+
|
32
|
+
spec.add_development_dependency 'awesome_print', '~> 1.6'
|
33
|
+
spec.add_development_dependency 'bundler', '~> 1.10'
|
34
|
+
spec.add_development_dependency 'gem-ctags', '~> 1.0'
|
35
|
+
spec.add_development_dependency 'guard', '~> 2.12'
|
36
|
+
spec.add_development_dependency 'guard-minitest', '~> 2.4'
|
37
|
+
spec.add_development_dependency 'minitest', '~> 5.6'
|
38
|
+
spec.add_development_dependency 'minitest-spec-context', '~> 0.0'
|
39
|
+
spec.add_development_dependency 'pry', '~> 0.10'
|
40
|
+
spec.add_development_dependency 'pry-byebug', '~> 3.1' if RUBY_VERSION >= '2.0.0'
|
41
|
+
spec.add_development_dependency 'rake', '~> 10.4'
|
42
|
+
spec.add_development_dependency 'rubocop', '~> 0.31'
|
43
|
+
spec.add_development_dependency 'yard', '~> 0.8'
|
44
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
module FilenameCleaner
|
2
|
+
DOT = "."
|
3
|
+
class << self
|
4
|
+
# Sanitize the any name with or without extension
|
5
|
+
#
|
6
|
+
# @param [String] name the input string
|
7
|
+
# @param [String] sep_char the separator char to be used
|
8
|
+
# @param [Boolean] have_extension indicate if the the input file should be
|
9
|
+
# treated as having extension or not having one
|
10
|
+
# @return [String] output string with special chars replaced withe specified string
|
11
|
+
def sanitize(name, sep_char = ".", have_extension = false)
|
12
|
+
if have_extension
|
13
|
+
sanitize_name_with_extension(name, sep_char)
|
14
|
+
else
|
15
|
+
sanitize_name_without_extension(name, sep_char)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# Get formatted name for existing file
|
20
|
+
#
|
21
|
+
# @param [String] filename the input filename
|
22
|
+
# @param [Hash<Symbol, Object>] opts the hash value for options to be applied
|
23
|
+
#
|
24
|
+
# @option opts [String] :sep_char The separator string
|
25
|
+
# @option opts [Boolean] :downcase Convert each word to lower case
|
26
|
+
# @option opts [Boolean] :capitalize: Capitalize each word in the name
|
27
|
+
def formatted_name(filename, opts = {})
|
28
|
+
sep_char = opts[:sep_char] || "."
|
29
|
+
sanitized_name = FilenameCleaner.sanitize(filename, sep_char, true)
|
30
|
+
|
31
|
+
# First split the two part so that only name is used!
|
32
|
+
basename = File.basename(sanitized_name, ".*")
|
33
|
+
extname = File.extname(sanitized_name)
|
34
|
+
if opts[:downcase]
|
35
|
+
basename = basename.split(sep_char).map(&:downcase).join(sep_char)
|
36
|
+
end
|
37
|
+
if opts[:capitalize]
|
38
|
+
basename = basename.split(sep_char).map(&:capitalize).join(sep_char)
|
39
|
+
end
|
40
|
+
"#{basename}#{extname}"
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
# Sanitize the any name with or without any extension
|
46
|
+
#
|
47
|
+
# @param [String] name the input string
|
48
|
+
# @param [String] sep_char the separator char to be used
|
49
|
+
# @return [String] output string with special chars replaced withe specified string
|
50
|
+
def sanitize_name_without_extension(name, sep_char = ".")
|
51
|
+
replace_dot(sanitize_with_dot(name), sep_char)
|
52
|
+
end
|
53
|
+
|
54
|
+
# Sanitize filename that works with file with extension
|
55
|
+
#
|
56
|
+
# @param [String] name the input filename with extension
|
57
|
+
# @return [String] the output file with special characters replaced
|
58
|
+
def sanitize_name_with_extension(name, sep_char = ".")
|
59
|
+
extension = File.extname(name)
|
60
|
+
name_only = File.basename(name, ".*")
|
61
|
+
name_only = replace_dot(sanitize_with_dot(name_only), sep_char)
|
62
|
+
"#{name_only}#{extension}"
|
63
|
+
end
|
64
|
+
|
65
|
+
# Replace the multiple special characters with a dot string
|
66
|
+
#
|
67
|
+
# @param [String] input_name input file
|
68
|
+
# @return [String] the new name with special characters replaced or removed.
|
69
|
+
def sanitize_with_dot(input_name)
|
70
|
+
# Don't mutate the input name
|
71
|
+
name = input_name.clone
|
72
|
+
|
73
|
+
# Replace any special characters with a dot
|
74
|
+
name.gsub!(/[^0-9A-Za-z\-_ ]/, DOT)
|
75
|
+
|
76
|
+
# Replace multiple occurrences of a given character with a dot
|
77
|
+
["-", "_", " "].each do |c|
|
78
|
+
name.gsub!(/#{Regexp.quote(c)}+/, DOT)
|
79
|
+
end
|
80
|
+
|
81
|
+
# Replace multiple occurrence of dot with one dot
|
82
|
+
name.gsub!(/#{Regexp.quote(DOT)}+/, DOT)
|
83
|
+
|
84
|
+
# Remove the last char if it is a dot
|
85
|
+
name.gsub!(/\.$/, "") if name[-1] == DOT
|
86
|
+
|
87
|
+
# return the result
|
88
|
+
name
|
89
|
+
end
|
90
|
+
|
91
|
+
# Replace 'dot' string with a given string if specified
|
92
|
+
def replace_dot(string, replace = nil)
|
93
|
+
result = string.clone
|
94
|
+
result.gsub!(/#{Regexp.quote(DOT)}+/, replace) if replace
|
95
|
+
result
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
module GithubCloner
|
2
|
+
class GithubUtils
|
3
|
+
class << self
|
4
|
+
# List all repositories
|
5
|
+
def list_all(opts = {})
|
6
|
+
repos = []
|
7
|
+
begin
|
8
|
+
args = {
|
9
|
+
per_page: opts[:per_page] || 100,
|
10
|
+
all_repos: opts[:all_repos]
|
11
|
+
}.merge!(opts)
|
12
|
+
|
13
|
+
# If we are using :org then we must delete :user from the options
|
14
|
+
if args[:org]
|
15
|
+
args.merge!(org: opts[:org])
|
16
|
+
args.delete(:user)
|
17
|
+
else
|
18
|
+
args.merge!(user: opts[:user])
|
19
|
+
args.delete(:org)
|
20
|
+
end
|
21
|
+
|
22
|
+
# require this for only with org and with private repos
|
23
|
+
args.merge!(oauth_token: args[:oauth_token]) if args[:oauth_token]
|
24
|
+
##puts "FYI: your options #{args}"
|
25
|
+
|
26
|
+
# Only required if we use oauth_token which allow use to list private repositories
|
27
|
+
github = Github.new(args)
|
28
|
+
|
29
|
+
# List all repositories
|
30
|
+
page = 1
|
31
|
+
loop do
|
32
|
+
args.merge!(page: page)
|
33
|
+
|
34
|
+
list = github.repos.list(args)
|
35
|
+
|
36
|
+
break if list.empty?
|
37
|
+
page += 1
|
38
|
+
repos << list.select do |r|
|
39
|
+
if !args[:fork]
|
40
|
+
r if !r['fork']
|
41
|
+
else
|
42
|
+
r
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
repos.flatten!
|
47
|
+
rescue => e
|
48
|
+
puts "Error getting getting repository for user/org #{e}"
|
49
|
+
exit(1)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# Clone list of repos of the following forms
|
54
|
+
#
|
55
|
+
def clone_all(projects, args = {})
|
56
|
+
base_path = File.expand_path(args[:base_dir])
|
57
|
+
|
58
|
+
if args[:clone_repos]
|
59
|
+
FileUtils.mkdir_p(base_path)
|
60
|
+
else
|
61
|
+
puts "FYI: dry-run only, no action taken!!" unless args[:clone_repos]
|
62
|
+
end
|
63
|
+
|
64
|
+
projects.each_with_index do |project, i|
|
65
|
+
org_name, language, repo_name = project.split(File::SEPARATOR)
|
66
|
+
|
67
|
+
# Note: need to cleanup the language like 'Emacs Lisp' to 'Emacs_Lisp' or 'emacs_lisp'
|
68
|
+
language = FilenameCleaner.sanitize(language, '_', false)
|
69
|
+
|
70
|
+
if args[:group_by_user]
|
71
|
+
output_path = [base_path, org_name, language, repo_name].join(File::SEPARATOR)
|
72
|
+
else
|
73
|
+
output_path = [base_path, language, org_name, repo_name].join(File::SEPARATOR)
|
74
|
+
end
|
75
|
+
puts "Process #{i+1} of #{projects.size} : git clone git@github.com:#{[org_name, repo_name].join(File::SEPARATOR)}.git #{output_path}"
|
76
|
+
if args[:clone_repos]
|
77
|
+
system("git clone git@github.com:#{[org_name, repo_name].join(File::SEPARATOR)}.git #{output_path} 2> /dev/null")
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def repos_as_list(repos)
|
83
|
+
# Store the result in this list
|
84
|
+
list = []
|
85
|
+
repos.each do |k,v|
|
86
|
+
v.each do |i|
|
87
|
+
user_name, repo_name = i.split(File::SEPARATOR)
|
88
|
+
list << [user_name, k, repo_name].join(File::SEPARATOR)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
list
|
92
|
+
end
|
93
|
+
|
94
|
+
# Map of repository by language
|
95
|
+
#
|
96
|
+
# Map language to list of repository
|
97
|
+
# @return [Hash<String, Array<String>>] map of language to repos
|
98
|
+
def repos_by_language(repos, languages)
|
99
|
+
hash = {}
|
100
|
+
languages.each do |l|
|
101
|
+
hash[l] = repos.collect { |r| r['full_name'] if r['language'] == l }.compact
|
102
|
+
end
|
103
|
+
hash
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'ostruct'
|
3
|
+
module GithubCloner
|
4
|
+
class OptParser
|
5
|
+
# Return a structure describing the options.
|
6
|
+
def self.parse(args)
|
7
|
+
# The options specified on the command line will be collected in *options*.
|
8
|
+
options = OpenStruct.new
|
9
|
+
|
10
|
+
# Set the sensible default for the options explicitly
|
11
|
+
options.base_dir = "."
|
12
|
+
options.all_repos = false
|
13
|
+
options.group_by_user = false
|
14
|
+
options.clone_repos = false
|
15
|
+
|
16
|
+
# The parser
|
17
|
+
opt_parser = OptionParser.new do |opts|
|
18
|
+
opts.banner = "Usage: github-cloner [options]"
|
19
|
+
|
20
|
+
opts.separator ""
|
21
|
+
opts.separator "Specific options:"
|
22
|
+
|
23
|
+
# Mandatory argument
|
24
|
+
opts.on("-b", "--base-dir BASE_DIR",
|
25
|
+
"where BASE_DIR is the directory where the repositories will be cloned to (mandatory)",
|
26
|
+
"If not specified, current directory will be used") do |dir|
|
27
|
+
options.base_dir = dir
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on("-u", "--user USER",
|
31
|
+
"The Github USER that will be cloned from (mandatory)") do |user|
|
32
|
+
options.user = user
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on("-o", "--org [ORG]",
|
36
|
+
"The Github's organization name to be used if specified (optional)",
|
37
|
+
"where ORG is the organization that the user belongs to") do |org|
|
38
|
+
options.org = org
|
39
|
+
end
|
40
|
+
|
41
|
+
opts.on("-t", "--oauth-token [OAUTH_TOKEN]",
|
42
|
+
"The Github's oauth_token for authentication (optional - only required to list/clone private repositories)",
|
43
|
+
"where OAUTH_TOKEN is from the user's Github setting") do |token|
|
44
|
+
options.oauth_token = token
|
45
|
+
end
|
46
|
+
|
47
|
+
opts.on("-l", "--language [LANG]",
|
48
|
+
"Clone only project of type LANG (optional)",
|
49
|
+
"where LANG is main language as shown on Github") do |lang|
|
50
|
+
options.language = lang
|
51
|
+
end
|
52
|
+
|
53
|
+
# Boolean switch.
|
54
|
+
opts.on("-a", "--[no-]all-repos",
|
55
|
+
"All repository only (optional)",
|
56
|
+
"default to original/non-forked repositories only") do |a|
|
57
|
+
options.all_repos = a
|
58
|
+
end
|
59
|
+
|
60
|
+
opts.on("-g", "--[no-]group-by-user",
|
61
|
+
"Group the output by {BASE_DIR}/{USER}/{LANG}",
|
62
|
+
"default to {BASE_DIR}/{LANG}/{USER}") do |gbu|
|
63
|
+
options.group_by_user = gbu
|
64
|
+
end
|
65
|
+
|
66
|
+
opts.on( "-c", "--[no-]clone",
|
67
|
+
"Clone the repositories to the path specified (optional)",
|
68
|
+
"default to --no-clone e.g. dry-run only") do |c|
|
69
|
+
options.clone_repos = c
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.separator ""
|
73
|
+
opts.separator "Common options:"
|
74
|
+
|
75
|
+
# No argument, shows at tail. This will print an options summary.
|
76
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
77
|
+
puts opts
|
78
|
+
puts ""
|
79
|
+
puts "Example Usage:"
|
80
|
+
puts ""
|
81
|
+
puts "a) List the 'JavaScript' repositories for a given user (dry-run)"
|
82
|
+
puts "github-cloner -b ~/Desktop/projects -u awesome_user -l JavaScript"
|
83
|
+
puts ""
|
84
|
+
puts "b) Clone the 'JavaScript' repositories for a given user (note: --clone or -c option)"
|
85
|
+
puts "github-cloner -b ~/Desktop/projects -u awesome_user -l JavaScript -c"
|
86
|
+
puts ""
|
87
|
+
exit
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
opt_parser.parse!(args)
|
92
|
+
options
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
data/test/test_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,249 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: github-cloner
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Burin Choomnuan
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-04-06 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: github_api
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.13'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.13'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: awesome_print
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.6'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.6'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.10'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.10'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: gem-ctags
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: guard
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '2.12'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '2.12'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: guard-minitest
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '2.4'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '2.4'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: minitest
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '5.6'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '5.6'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: minitest-spec-context
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0.0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0.0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: pry
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - "~>"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0.10'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - "~>"
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0.10'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: pry-byebug
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '3.1'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '3.1'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: rake
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '10.4'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '10.4'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: rubocop
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - "~>"
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0.31'
|
174
|
+
type: :development
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - "~>"
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0.31'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: yard
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - "~>"
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0.8'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - "~>"
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '0.8'
|
195
|
+
description: " Clone/list multiple Github repositories for a given user/organization
|
196
|
+
including private repos easily.\n e.g.$github-cloner --base-dir ~/projects --user
|
197
|
+
awesome_dev --language \"Emacs Lisp\" --clone\n "
|
198
|
+
email:
|
199
|
+
- agilecreativity@gmail.com
|
200
|
+
executables:
|
201
|
+
- github-cloner
|
202
|
+
extensions: []
|
203
|
+
extra_rdoc_files: []
|
204
|
+
files:
|
205
|
+
- ".gitignore"
|
206
|
+
- ".rubocop.yml"
|
207
|
+
- CHANGELOG.md
|
208
|
+
- Gemfile
|
209
|
+
- LICENSE
|
210
|
+
- README.md
|
211
|
+
- Rakefile
|
212
|
+
- bin/github-cloner
|
213
|
+
- github_cloner.gemspec
|
214
|
+
- lib/github_cloner.rb
|
215
|
+
- lib/github_cloner/filename_cleaner.rb
|
216
|
+
- lib/github_cloner/github_cloner.rb
|
217
|
+
- lib/github_cloner/opt_parser.rb
|
218
|
+
- lib/github_cloner/version.rb
|
219
|
+
- test/lib/github_copier/test_github_cloner.rb
|
220
|
+
- test/test_helper.rb
|
221
|
+
homepage: https://github.com/agilecreativity/github-cloner
|
222
|
+
licenses:
|
223
|
+
- MIT
|
224
|
+
metadata: {}
|
225
|
+
post_install_message:
|
226
|
+
rdoc_options: []
|
227
|
+
require_paths:
|
228
|
+
- lib
|
229
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
230
|
+
requirements:
|
231
|
+
- - ">="
|
232
|
+
- !ruby/object:Gem::Version
|
233
|
+
version: 2.0.0
|
234
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
235
|
+
requirements:
|
236
|
+
- - ">="
|
237
|
+
- !ruby/object:Gem::Version
|
238
|
+
version: '0'
|
239
|
+
requirements: []
|
240
|
+
rubyforge_project:
|
241
|
+
rubygems_version: 2.4.5.1
|
242
|
+
signing_key:
|
243
|
+
specification_version: 4
|
244
|
+
summary: Clone/list Github repository for a given user/organization include private/public
|
245
|
+
in one go
|
246
|
+
test_files:
|
247
|
+
- test/lib/github_copier/test_github_cloner.rb
|
248
|
+
- test/test_helper.rb
|
249
|
+
has_rdoc:
|