hubba 0.5.1 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Manifest.txt +4 -11
- data/README.md +127 -1
- data/lib/hubba.rb +36 -3
- data/lib/hubba/config.rb +51 -0
- data/lib/hubba/github.rb +149 -79
- data/lib/hubba/reposet.rb +83 -0
- data/lib/hubba/stats.rb +184 -239
- data/lib/hubba/update.rb +44 -0
- data/lib/hubba/update_traffic.rb +52 -0
- data/lib/hubba/version.rb +2 -2
- data/test/test_config.rb +10 -0
- metadata +6 -13
- data/lib/hubba/cache.rb +0 -61
- data/lib/hubba/client.rb +0 -79
- data/test/cache/users~geraldb~orgs.json +0 -46
- data/test/cache/users~geraldb~repos.json +0 -263
- data/test/stats/jekyll~minima.json +0 -25
- data/test/stats/openblockchains~awesome-blockchains.json +0 -27
- data/test/stats/opendatajson~factbook.json.json +0 -39
- data/test/stats/poole~hyde.json +0 -21
- data/test/test_cache.rb +0 -43
- data/test/test_stats.rb +0 -122
- data/test/test_stats_tmp.rb +0 -41
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6dfdf158420e74f4a2810c18e46cab065aa2dcce
|
4
|
+
data.tar.gz: 51357a2974422d2e9c4c0bb7b961d08ee43297bd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7388e0646b07392b35ad13ea738c440b49f37cca283b4abf380071069ce48ef22bbe2d9fe293e266056ee1c7c386c92547740bb84a74cf3c03651a1f48ac48b2
|
7
|
+
data.tar.gz: 4ffbc0a84814db71694e6f0d750c7269916ae30f46a64ca79679976de222e06cbb63918b73b47c679ef8b92fa4fc9abce86f2cc3128903c1e2c8da634cfd96e2
|
data/Manifest.txt
CHANGED
@@ -3,19 +3,12 @@ Manifest.txt
|
|
3
3
|
README.md
|
4
4
|
Rakefile
|
5
5
|
lib/hubba.rb
|
6
|
-
lib/hubba/
|
7
|
-
lib/hubba/client.rb
|
6
|
+
lib/hubba/config.rb
|
8
7
|
lib/hubba/github.rb
|
8
|
+
lib/hubba/reposet.rb
|
9
9
|
lib/hubba/stats.rb
|
10
|
+
lib/hubba/update.rb
|
11
|
+
lib/hubba/update_traffic.rb
|
10
12
|
lib/hubba/version.rb
|
11
|
-
test/cache/users~geraldb~orgs.json
|
12
|
-
test/cache/users~geraldb~repos.json
|
13
13
|
test/helper.rb
|
14
|
-
test/stats/jekyll~minima.json
|
15
|
-
test/stats/openblockchains~awesome-blockchains.json
|
16
|
-
test/stats/opendatajson~factbook.json.json
|
17
|
-
test/stats/poole~hyde.json
|
18
|
-
test/test_cache.rb
|
19
14
|
test/test_config.rb
|
20
|
-
test/test_stats.rb
|
21
|
-
test/test_stats_tmp.rb
|
data/README.md
CHANGED
@@ -10,8 +10,134 @@ hubba gem - (yet) another (lite) GitHub HTTP API client / library
|
|
10
10
|
|
11
11
|
## Usage
|
12
12
|
|
13
|
-
TBD
|
14
13
|
|
14
|
+
### Step 0: Secrets, Secrets, Secrets - Your Authentication Token
|
15
|
+
|
16
|
+
Note: Set your GitHub env credentials (personal access token preferred) e.g.
|
17
|
+
|
18
|
+
```
|
19
|
+
set HUBBA_TOKEN=abcdef0123456789
|
20
|
+
# - or -
|
21
|
+
set HUBBA_USER=you
|
22
|
+
set HUBBA_PASSWORD=topsecret
|
23
|
+
```
|
24
|
+
|
25
|
+
|
26
|
+
### Step 1: Get a list of all your repos
|
27
|
+
|
28
|
+
Use the GitHub API to get a list of all your repos:
|
29
|
+
|
30
|
+
``` ruby
|
31
|
+
require 'hubba'
|
32
|
+
|
33
|
+
h = Hubba.reposet( 'geraldb' )
|
34
|
+
pp h
|
35
|
+
|
36
|
+
File.open( './repos.yml', 'w' ) { |f| f.write( h.to_yaml ) }
|
37
|
+
```
|
38
|
+
|
39
|
+
resulting in:
|
40
|
+
|
41
|
+
``` yaml
|
42
|
+
geraldb (11):
|
43
|
+
- austria
|
44
|
+
- catalog
|
45
|
+
- chelitas
|
46
|
+
- geraldb.github.io
|
47
|
+
- logos
|
48
|
+
- sandbox
|
49
|
+
- talks
|
50
|
+
- web-proxy-win
|
51
|
+
- webcomponents
|
52
|
+
- webpub-reader
|
53
|
+
- wine.db.tools
|
54
|
+
|
55
|
+
openfootball (41):
|
56
|
+
- africa-cup
|
57
|
+
- austria
|
58
|
+
- club-world-cup
|
59
|
+
- clubs
|
60
|
+
- confed-cup
|
61
|
+
- copa-america
|
62
|
+
- copa-libertadores
|
63
|
+
- copa-sudamericana
|
64
|
+
- deutschland
|
65
|
+
# ...
|
66
|
+
```
|
67
|
+
|
68
|
+
|
69
|
+
Note: If you have more than one account (e.g. an extra robot account or such)
|
70
|
+
you can add them along e.g.
|
71
|
+
|
72
|
+
|
73
|
+
``` ruby
|
74
|
+
h = Hubba.reposet( 'geraldb', 'yorobot' )
|
75
|
+
pp h
|
76
|
+
```
|
77
|
+
|
78
|
+
|
79
|
+
Note: By default all your repos from organizations get auto-included -
|
80
|
+
use the `orgs: false` option to turn off auto-inclusion.
|
81
|
+
|
82
|
+
Note: By default all (personal) repos, that is, repos in your primary (first)
|
83
|
+
account that are forks get auto-excluded.
|
84
|
+
|
85
|
+
|
86
|
+
|
87
|
+
### Step 2: Update repo statistics (daily / weekly / monthly)
|
88
|
+
|
89
|
+
|
90
|
+
#### Basics (commits, stars, forks, topics, etc.)
|
91
|
+
|
92
|
+
Use `update_stats` to
|
93
|
+
to get the latest commit, star count and more for all your repos
|
94
|
+
listed in `./repos.yml` via the GitHub API:
|
95
|
+
|
96
|
+
``` ruby
|
97
|
+
Hubba.update_stats( './repos.yml' )
|
98
|
+
```
|
99
|
+
|
100
|
+
Note: By default the datafiles (one per repo)
|
101
|
+
get stored in the `./data` directory.
|
102
|
+
|
103
|
+
|
104
|
+
#### Traffic (page views, clones, referrers, etc.)
|
105
|
+
|
106
|
+
Use `update_traffic` to
|
107
|
+
to get the latest traffic stats (page views, clones, referrers, etc.)
|
108
|
+
for all your repos listed in `./repos.yml` via the GitHub API.
|
109
|
+
|
110
|
+
``` ruby
|
111
|
+
Hubba.update_traffic( './repos.yml' )
|
112
|
+
```
|
113
|
+
|
114
|
+
Note: Access to traffic statistics via the GitHub API
|
115
|
+
requires push access for your GitHub (personal access) token.
|
116
|
+
|
117
|
+
|
118
|
+
|
119
|
+
|
120
|
+
### Step 3: Generate some statistics / analytics reports
|
121
|
+
|
122
|
+
|
123
|
+
See the [hubba-reports gem](https://github.com/rubycoco/git/tree/master/hubba-reports) on how to use pre-built/pre-packaged ready-to-use
|
124
|
+
github statistics / analytics reports.
|
125
|
+
|
126
|
+
|
127
|
+
|
128
|
+
That's all for now.
|
129
|
+
|
130
|
+
|
131
|
+
|
132
|
+
## Installation
|
133
|
+
|
134
|
+
Use
|
135
|
+
|
136
|
+
gem install hubba
|
137
|
+
|
138
|
+
or add the gem to your Gemfile
|
139
|
+
|
140
|
+
gem 'hubba'
|
15
141
|
|
16
142
|
|
17
143
|
## License
|
data/lib/hubba.rb
CHANGED
@@ -1,13 +1,46 @@
|
|
1
1
|
# 3rd party (our own)
|
2
2
|
require 'webclient'
|
3
3
|
|
4
|
+
|
5
|
+
###############
|
6
|
+
## helpers
|
7
|
+
|
8
|
+
def save_json( path, data ) ## data - hash or array
|
9
|
+
File.open( path, 'w:utf-8' ) do |f|
|
10
|
+
f.write( JSON.pretty_generate( data ))
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def save_yaml( path, data )
|
15
|
+
File.open( path, 'w:utf-8' ) do |f|
|
16
|
+
f.write( data.to_yaml )
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
|
4
22
|
# our own code
|
5
23
|
require 'hubba/version' # note: let version always go first
|
6
|
-
require 'hubba/
|
7
|
-
require 'hubba/client'
|
24
|
+
require 'hubba/config'
|
8
25
|
require 'hubba/github'
|
26
|
+
require 'hubba/reposet'
|
27
|
+
|
28
|
+
### update stats (github data) machinery
|
9
29
|
require 'hubba/stats'
|
30
|
+
require 'hubba/update'
|
31
|
+
require 'hubba/update_traffic'
|
32
|
+
|
33
|
+
|
34
|
+
|
35
|
+
|
36
|
+
############
|
37
|
+
# add convenience alias for alternate spelling - why? why not?
|
38
|
+
module Hubba
|
39
|
+
GitHub = Github
|
40
|
+
end
|
10
41
|
|
11
42
|
|
12
43
|
# say hello
|
13
|
-
puts Hubba.banner
|
44
|
+
puts Hubba.banner
|
45
|
+
|
46
|
+
|
data/lib/hubba/config.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
module Hubba
|
2
|
+
|
3
|
+
class Configuration
|
4
|
+
def data_dir() @data_dir || './data'; end
|
5
|
+
def data_dir=( value ) @data_dir = value; end
|
6
|
+
|
7
|
+
### todo/check: rename to/use tmp_dir - why? why not?
|
8
|
+
def cache_dir() @cache_dir || './cache'; end
|
9
|
+
def cache_dir=( value ) @cache_dir = value; end
|
10
|
+
|
11
|
+
|
12
|
+
# try default setup via ENV variables
|
13
|
+
def token() @token || ENV[ 'HUBBA_TOKEN' ]; end
|
14
|
+
def token=( value ) @token = value; end
|
15
|
+
|
16
|
+
# todo/check: use HUBBA_LOGIN - why? why not?
|
17
|
+
def user() @user || ENV[ 'HUBBA_USER' ]; end
|
18
|
+
def password() @password || ENV[ 'HUBBA_PASSWORD' ]; end
|
19
|
+
def user=( value ) @user = value; end
|
20
|
+
def password=( value ) @password = value; end
|
21
|
+
|
22
|
+
end # class Configuration
|
23
|
+
|
24
|
+
|
25
|
+
## lets you use
|
26
|
+
## Hubba.configure do |config|
|
27
|
+
## config.token = 'secret'
|
28
|
+
## -or-
|
29
|
+
## config.user = 'testdada'
|
30
|
+
## config.password = 'secret'
|
31
|
+
## end
|
32
|
+
##
|
33
|
+
## move configure block to GitHub class - why? why not?
|
34
|
+
## e.g. GitHub.configure do |config|
|
35
|
+
## ...
|
36
|
+
## end
|
37
|
+
|
38
|
+
|
39
|
+
def self.configuration
|
40
|
+
@configuration ||= Configuration.new
|
41
|
+
end
|
42
|
+
class << self
|
43
|
+
alias_method :config, :configuration
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
def self.configure
|
48
|
+
yield( configuration )
|
49
|
+
end
|
50
|
+
|
51
|
+
end # module Hubba
|
data/lib/hubba/github.rb
CHANGED
@@ -1,88 +1,57 @@
|
|
1
1
|
module Hubba
|
2
2
|
|
3
|
-
class Configuration
|
4
|
-
attr_accessor :token
|
5
3
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
end
|
17
|
-
|
18
|
-
## lets you use
|
19
|
-
## Hubba.configure do |config|
|
20
|
-
## config.token = 'secret'
|
21
|
-
## #-or-
|
22
|
-
## config.user = 'testdada'
|
23
|
-
## config.password = 'secret'
|
24
|
-
## end
|
25
|
-
##
|
26
|
-
## move configure block to GitHub class - why? why not?
|
27
|
-
## e.g. GitHub.configure do |config|
|
28
|
-
## ...
|
29
|
-
## end
|
30
|
-
|
31
|
-
|
32
|
-
def self.configuration
|
33
|
-
@configuration ||= Configuration.new
|
34
|
-
end
|
35
|
-
|
36
|
-
def self.configure
|
37
|
-
yield( configuration )
|
38
|
-
end
|
39
|
-
|
40
|
-
|
41
|
-
class Resource
|
42
|
-
attr_reader :data
|
43
|
-
def initialize( data )
|
44
|
-
@data = data
|
4
|
+
class Github
|
5
|
+
BASE_URL = 'https://api.github.com'
|
6
|
+
|
7
|
+
###############
|
8
|
+
## (nested) classes for "wrapped" response (parsed json body)
|
9
|
+
class Resource
|
10
|
+
attr_reader :data
|
11
|
+
def initialize( data )
|
12
|
+
@data = data
|
13
|
+
end
|
45
14
|
end
|
46
|
-
end
|
47
15
|
|
48
|
-
class Repos < Resource
|
49
|
-
|
50
|
-
|
51
|
-
|
16
|
+
class Repos < Resource
|
17
|
+
def names
|
18
|
+
## sort by name
|
19
|
+
data.map { |item| item['name'] }.sort
|
20
|
+
end
|
52
21
|
end
|
53
|
-
end
|
54
22
|
|
55
|
-
class Orgs < Resource
|
56
|
-
|
57
|
-
|
58
|
-
|
23
|
+
class Orgs < Resource
|
24
|
+
def logins
|
25
|
+
## sort by name
|
26
|
+
data.map { |item| item['login'] }.sort
|
27
|
+
end
|
28
|
+
alias_method :names, :logins ## add name alias - why? why not?
|
59
29
|
end
|
60
|
-
end
|
61
|
-
|
62
|
-
|
63
30
|
|
64
|
-
class Github
|
65
|
-
|
66
|
-
def initialize( cache_dir: './cache' )
|
67
|
-
@cache = Cache.new( cache_dir )
|
68
31
|
|
69
|
-
@client = if Hubba.configuration.token
|
70
|
-
Client.new( token: Hubba.configuration.token )
|
71
|
-
elsif Hubba.configuration.user &&
|
72
|
-
Hubba.configuration.password
|
73
|
-
Client.new( user: Hubba.configuration.user,
|
74
|
-
password: Hubba.configuration.password )
|
75
|
-
else
|
76
|
-
Client.new
|
77
|
-
end
|
78
32
|
|
79
|
-
|
33
|
+
def initialize( token: nil,
|
34
|
+
user: nil,
|
35
|
+
password: nil )
|
36
|
+
@token = nil
|
37
|
+
@user = nil
|
38
|
+
@password = nil
|
39
|
+
|
40
|
+
if token ## 1a) give preference to passed in token
|
41
|
+
@token = token
|
42
|
+
elsif user && password ## 1b) or passed in user/password credentials
|
43
|
+
@user = user
|
44
|
+
@password = password
|
45
|
+
elsif Hubba.config.token ## 2a) followed by configured (or env) token
|
46
|
+
@token = Hubba.config.token
|
47
|
+
elsif Hubba.config.user && Hubba.config.password ## 2b)
|
48
|
+
@user = Hubba.config.user
|
49
|
+
@password = Hubba.config.password
|
50
|
+
else ## 3)
|
51
|
+
## no token or credentials passed in or configured
|
52
|
+
end
|
80
53
|
end
|
81
54
|
|
82
|
-
def offline!() @offline = true; end ## switch to offline - todo: find a "better" way - why? why not?
|
83
|
-
def online!() @offline = false; end
|
84
|
-
def offline?() @offline == true; end
|
85
|
-
def online?() @offline == false; end
|
86
55
|
|
87
56
|
|
88
57
|
def user( name )
|
@@ -107,6 +76,7 @@ def user_orgs( name )
|
|
107
76
|
end
|
108
77
|
|
109
78
|
|
79
|
+
|
110
80
|
def org( name )
|
111
81
|
Resource.new( get "/orgs/#{name}" )
|
112
82
|
end
|
@@ -121,20 +91,120 @@ def repo( full_name ) ## full_name (handle) e.g. henrythemes/jekyll-starter-th
|
|
121
91
|
Resource.new( get "/repos/#{full_name}" )
|
122
92
|
end
|
123
93
|
|
94
|
+
def repo_languages( full_name )
|
95
|
+
Resource.new( get "/repos/#{full_name}/languages" )
|
96
|
+
end
|
97
|
+
|
98
|
+
def repo_topics( full_name )
|
99
|
+
## note: requires "api preview" accept headers (overwrites default v3+json)
|
100
|
+
## e.g. application/vnd.github.mercy-preview+json
|
101
|
+
Resource.new( get( "/repos/#{full_name}/topics", preview: 'mercy' ) )
|
102
|
+
end
|
103
|
+
|
104
|
+
|
124
105
|
def repo_commits( full_name )
|
125
106
|
Resource.new( get "/repos/#{full_name}/commits" )
|
126
107
|
end
|
127
108
|
|
128
109
|
|
110
|
+
def repo_traffic_clones( full_name )
|
111
|
+
# Get repository clones
|
112
|
+
# Get the total number of clones and breakdown per day or week
|
113
|
+
# for the last 14 days.
|
114
|
+
# Timestamps are aligned to UTC midnight of the beginning of the day or week.
|
115
|
+
# Week begins on Monday.
|
116
|
+
Resource.new( get "/repos/#{full_name}/traffic/clones" )
|
117
|
+
end
|
118
|
+
|
119
|
+
def repo_traffic_views( full_name )
|
120
|
+
# Get page views
|
121
|
+
# Get the total number of views and breakdown per day or week
|
122
|
+
# for the last 14 days.
|
123
|
+
# Timestamps are aligned to UTC midnight of the beginning of the day or week.
|
124
|
+
# Week begins on Monday.
|
125
|
+
Resource.new( get "/repos/#{full_name}/traffic/views" )
|
126
|
+
end
|
127
|
+
|
128
|
+
|
129
|
+
def repo_traffic_popular_paths( full_name )
|
130
|
+
# Get top referral paths
|
131
|
+
# Get the top 10 popular contents over the last 14 days.
|
132
|
+
Resource.new( get "/repos/#{full_name}/traffic/popular/paths" )
|
133
|
+
end
|
134
|
+
|
135
|
+
def repo_traffic_popular_referrers( full_name )
|
136
|
+
# Get top referral sources
|
137
|
+
# Get the top 10 referrers over the last 14 days.
|
138
|
+
Resource.new( get "/repos/#{full_name}/traffic/popular/referrers" )
|
139
|
+
end
|
140
|
+
|
141
|
+
|
142
|
+
|
143
|
+
|
129
144
|
private
|
130
|
-
def get( request_uri )
|
131
|
-
|
132
|
-
|
145
|
+
def get( request_uri, preview: nil )
|
146
|
+
|
147
|
+
puts "GET #{request_uri}"
|
148
|
+
|
149
|
+
## note: request_uri ALWAYS starts with leading /, thus use + for now!!!
|
150
|
+
# e.g. /users/geraldb
|
151
|
+
# /users/geraldb/repos
|
152
|
+
url = BASE_URL + request_uri
|
153
|
+
|
154
|
+
|
155
|
+
headers = {}
|
156
|
+
## add default headers if nothing (custom) set / passed-in
|
157
|
+
headers['User-Agent'] = "ruby/hubba v#{VERSION}" ## required by GitHub API
|
158
|
+
headers['Accept'] = if preview # e.g. mercy or ???
|
159
|
+
"application/vnd.github.#{preview}-preview+json"
|
160
|
+
else
|
161
|
+
'application/vnd.github.v3+json' ## default - recommend by GitHub API
|
162
|
+
end
|
163
|
+
|
164
|
+
auth = []
|
165
|
+
## check if credentials (user/password) present - if yes, use basic auth
|
166
|
+
if @token
|
167
|
+
puts " using (personal access) token - starting with: #{@token[0..6]}**********"
|
168
|
+
headers['Authorization'] = "token #{@token}"
|
169
|
+
## token works like:
|
170
|
+
## curl -H 'Authorization: token my_access_token' https://api.github.com/user/repos
|
171
|
+
elsif @user && @password
|
172
|
+
puts " using basic auth - user: #{@user}, password: ***"
|
173
|
+
## use credential auth "tuple" (that is, array with two string items) for now
|
174
|
+
## or use Webclient::HttpBasicAuth or something - why? why not?
|
175
|
+
auth = [@user, @password]
|
176
|
+
# req.basic_auth( @user, @password )
|
133
177
|
else
|
134
|
-
|
178
|
+
puts " using no credentials (no token, no user/password)"
|
135
179
|
end
|
136
|
-
end
|
137
180
|
|
138
|
-
|
181
|
+
res = Webclient.get( url,
|
182
|
+
headers: headers,
|
183
|
+
auth: auth )
|
184
|
+
|
185
|
+
# Get specific header
|
186
|
+
# response["content-type"]
|
187
|
+
# => "text/html; charset=UTF-8"
|
139
188
|
|
140
|
-
|
189
|
+
# Iterate all response headers.
|
190
|
+
# puts "HTTP HEADERS:"
|
191
|
+
# res.headers.each do |key, value|
|
192
|
+
# puts " #{key}: >#{value}<"
|
193
|
+
# end
|
194
|
+
# puts
|
195
|
+
|
196
|
+
# => "location => http://www.google.com/"
|
197
|
+
# => "content-type => text/html; charset=UTF-8"
|
198
|
+
# ...
|
199
|
+
|
200
|
+
if res.status.ok?
|
201
|
+
res.json
|
202
|
+
else
|
203
|
+
puts "!! HTTP ERROR: #{res.status.code} #{res.status.message}:"
|
204
|
+
pp res.raw
|
205
|
+
exit 1
|
206
|
+
end
|
207
|
+
end # method get
|
208
|
+
|
209
|
+
end # class Github
|
210
|
+
end # module Hubba
|