elasticrepo 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/elasticrepo/extractor.rb +7 -8
- data/lib/elasticrepo/indexer.rb +31 -34
- data/lib/elasticrepo/version.rb +1 -1
- data/spec/elasticrepo/extractor_spec.rb +32 -48
- data/spec/elasticrepo/indexer_spec.rb +11 -11
- data/spec/elasticrepo/repo_subset_spec.rb +36 -27
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6133947dc64cdd43a54c157b7634045c72a4ef00
|
|
4
|
+
data.tar.gz: 7888a81625224d20237307fd4ecbe1683559cfe4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c81cfb59b733fb4c6928c8df1f088b445962ece19847fe5a1b9a87afa5f4e0ec22bc5e0b054a2ac1071080b768a40c4b33672bfd3e4e3bef6b464be9f292598d
|
|
7
|
+
data.tar.gz: 1c7a1a023f8f2fa38d4b8f359c5c7873c6f450caa8e98ff5ef17a11d0e20ccdcaf71b67ccc10157064ea70e0401ffe25711d38be29c4749b51ac8c8032dfef68
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
module Elasticrepo
|
|
2
|
-
|
|
3
|
-
attr_reader :repos
|
|
2
|
+
module Extractor
|
|
4
3
|
|
|
5
|
-
# GET /users/:user/starred
|
|
6
|
-
# extract fields subset from each repo
|
|
7
|
-
# then
|
|
8
|
-
|
|
4
|
+
# GET /users/:user/starred --> Octokit.starred owner
|
|
5
|
+
# extract fields subset from each repo --> Elasticrepo::RepoSubset
|
|
6
|
+
# then returns an array of lighter starred repositories by user --> .repos
|
|
7
|
+
#
|
|
8
|
+
def self.repos owner
|
|
9
9
|
repos = Octokit.starred owner
|
|
10
10
|
@repos = repos.map!{|repo|Elasticrepo::RepoSubset.new(repo)}
|
|
11
11
|
end
|
|
12
12
|
end
|
|
13
|
-
end
|
|
14
|
-
|
|
13
|
+
end
|
data/lib/elasticrepo/indexer.rb
CHANGED
|
@@ -1,20 +1,17 @@
|
|
|
1
1
|
module Elasticrepo
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
def initialize
|
|
5
|
-
@idx = "repositories"
|
|
6
|
-
end
|
|
2
|
+
module Indexer
|
|
3
|
+
INDEX = 'repositories'
|
|
7
4
|
|
|
8
5
|
# given a user, extract his starred repositories, then index the list with Tire
|
|
9
6
|
#
|
|
10
7
|
# GET /users/:user/starred
|
|
11
8
|
# http://developer.github.com/v3/repos/#list-user-repositories
|
|
12
9
|
#
|
|
13
|
-
def starred(owner)
|
|
14
|
-
repos = Elasticrepo::Extractor.
|
|
10
|
+
def self.starred(owner)
|
|
11
|
+
repos = Elasticrepo::Extractor.repos(owner)
|
|
15
12
|
# index the extraction
|
|
16
13
|
#
|
|
17
|
-
Tire::Index.new(
|
|
14
|
+
Tire::Index.new(INDEX) do
|
|
18
15
|
# Import documents
|
|
19
16
|
#
|
|
20
17
|
import repos
|
|
@@ -24,43 +21,43 @@ module Elasticrepo
|
|
|
24
21
|
end
|
|
25
22
|
end
|
|
26
23
|
|
|
27
|
-
def import_with_map
|
|
28
|
-
|
|
29
|
-
|
|
24
|
+
#def import_with_map
|
|
25
|
+
# repos = Elasticrepo::Extractor.new(@owner).repos
|
|
26
|
+
# Tire::Index.new(INDEX) do
|
|
30
27
|
# Create the index with proper mapping (if doesn not exist already)
|
|
31
28
|
#
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
end # end import
|
|
29
|
+
# create :mappings => {
|
|
30
|
+
# :question => {
|
|
31
|
+
# :properties => {
|
|
32
|
+
# :id => { :type => 'integer', :analyzer => 'keyword' },
|
|
33
|
+
# :owner => { :type => 'string', :analyzer => 'keyword' },
|
|
34
|
+
# :name => { :type => 'string', :analyzer => 'keyword' },
|
|
35
|
+
# #:full_name => { :type => 'string', :analyzer => 'keyword' },
|
|
36
|
+
# :url => { :type => 'string', :analyzer => 'snowball' },
|
|
37
|
+
# :description => { :type => 'string', :analyzer => 'snowball' },
|
|
38
|
+
# :organization => { :type => 'string', :analyzer => 'keyword' },
|
|
39
|
+
# :language => { :type => 'string', :analyzer => 'keyword' },
|
|
40
|
+
# :created_at => { :type => 'date', :analyzer => 'keyword' },
|
|
41
|
+
# :pushed_at => { :type => 'date', :analyzer => 'keyword' },
|
|
42
|
+
# :updated_at => { :type => 'date', :analyzer => 'keyword' }
|
|
43
|
+
# }
|
|
44
|
+
# }
|
|
45
|
+
# }
|
|
46
|
+
# import repos
|
|
47
|
+
# refresh
|
|
48
|
+
# end
|
|
49
|
+
#end # end import
|
|
53
50
|
|
|
54
51
|
def update
|
|
55
52
|
# Just refresh the index as is
|
|
56
53
|
#
|
|
57
|
-
Tire::Index.new(
|
|
54
|
+
Tire::Index.new(INDEX).refresh
|
|
58
55
|
end
|
|
59
56
|
|
|
60
57
|
def delete
|
|
61
58
|
# delete 'repositories' index
|
|
62
59
|
#
|
|
63
|
-
Tire::Index.new(
|
|
60
|
+
Tire::Index.new(INDEX).delete
|
|
64
61
|
end
|
|
65
62
|
end # end class
|
|
66
63
|
end # end module
|
data/lib/elasticrepo/version.rb
CHANGED
|
@@ -1,50 +1,34 @@
|
|
|
1
1
|
require "spec_helper"
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
its("repos[0]['id']") { extracted.repos[0].id.should eq 5392501 }
|
|
36
|
-
its("repos[0]['owner']") { extracted.repos[0].owner.should eq "cainlevy" }
|
|
37
|
-
its("repos[0]['url']") { extracted.repos[0].url.should eq "https://api.github.com/repos/cainlevy/photor" }
|
|
38
|
-
its("repos[0]['description']") { extracted.repos[0].description.should eq "Photo Organizer (in Ruby)" }
|
|
39
|
-
its("repos[0]['name']") { extracted.repos[0].name.should eq "photor" }
|
|
40
|
-
its("repos[0]['created_at']") { extracted.repos[0].created_at.should eq "2012-08-12T22:26:08Z" }
|
|
41
|
-
its("repos[0]['pushed_at']") { extracted.repos[0].pushed_at.should eq "2013-02-19T03:11:10Z" }
|
|
42
|
-
its("repos[0]['organization']") { extracted.repos[0].organization.should eq "User" }
|
|
43
|
-
its("repos[0]['full_name']") { extracted.repos[0].full_name.should eq "cainlevy/photor" }
|
|
44
|
-
its("repos[0]['language']") { extracted.repos[0].language.should eq "Ruby" }
|
|
45
|
-
its("repos[0]['updated_at']") { extracted.repos[0].updated_at.should eq "2013-03-13T02:05:33Z" }
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
end
|
|
3
|
+
# API ref.: GET /users/:user/starred
|
|
4
|
+
# http://developer.github.com/v3/repos/
|
|
5
|
+
# https://github.com/pengwynn/octokit/blob/master/lib/octokit/client/users.rb
|
|
6
|
+
#
|
|
7
|
+
describe Elasticrepo::Extractor, :vcr => { :cassette_name => "v3/users/_get/lapaty/starred" } do
|
|
8
|
+
subject(:extracted) { Elasticrepo::Extractor.repos("lapaty") }
|
|
9
|
+
|
|
10
|
+
describe "#new" do
|
|
11
|
+
context "debugging specs print out" do
|
|
12
|
+
it "print the content of running specs" do
|
|
13
|
+
print "extracted.class: #{extracted.class} \n \n"
|
|
14
|
+
print "extracted.inspect: #{extracted.inspect} \n \n"
|
|
15
|
+
print "extracted[0].id: #{extracted[0].id} \n \n"
|
|
16
|
+
print "extracted[0].owner #{extracted[0].owner} \n \n"
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
context "extract fields subset from each repo" do
|
|
20
|
+
its(:id) { extracted[0].id.should eq 5392501 }
|
|
21
|
+
its(:owner) { extracted[0].owner.should eq "cainlevy" }
|
|
22
|
+
its(:url) { extracted[0].url.should eq "https://api.github.com/repos/cainlevy/photor" }
|
|
23
|
+
its(:description) { extracted[0].description.should eq "Photo Organizer (in Ruby)" }
|
|
24
|
+
its(:name) { extracted[0].name.should eq "photor" }
|
|
25
|
+
its(:created_at) { extracted[0].created_at.should eq "2012-08-12T22:26:08Z" }
|
|
26
|
+
its(:pushed_at) { extracted[0].pushed_at.should eq "2013-02-19T03:11:10Z" }
|
|
27
|
+
its(:organization) { extracted[0].organization.should eq "User" }
|
|
28
|
+
its(:full_name) { extracted[0].full_name.should eq "cainlevy/photor" }
|
|
29
|
+
its(:language) { extracted[0].language.should eq "Ruby" }
|
|
30
|
+
its(:updated_at) { extracted[0].updated_at.should eq "2013-03-13T02:05:33Z" }
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
end
|
|
@@ -2,9 +2,7 @@ require "spec_helper"
|
|
|
2
2
|
|
|
3
3
|
describe Elasticrepo::Indexer, :vcr => { :cassette_name => "v3/users/_get/lapaty/starred" } do
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
subject(:starred_import) { indexer.starred("lapaty") }
|
|
7
|
-
|
|
5
|
+
subject(:starred_import) { Elasticrepo::Indexer.starred("lapaty") }
|
|
8
6
|
|
|
9
7
|
describe "#starred" do
|
|
10
8
|
context "print starred_import instance for debugging purposes" do
|
|
@@ -23,14 +21,16 @@ describe Elasticrepo::Indexer, :vcr => { :cassette_name => "v3/users/_get/lapaty
|
|
|
23
21
|
end
|
|
24
22
|
end
|
|
25
23
|
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
# this test doesn't work
|
|
25
|
+
#
|
|
26
|
+
#describe "#delete" do
|
|
27
|
+
# subject { starred_import.delete }
|
|
28
28
|
# see also: https://github.com/ferpetrelli/elasticsearch_tire_test
|
|
29
|
-
|
|
29
|
+
# let(:search) { Tire::Search::Search.new("repositories") }
|
|
30
30
|
# subject { starred_import }
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
end
|
|
31
|
+
# it "cannot find a repo matching 'Image Gall' description anymore" do
|
|
32
|
+
# search.query { string('description:*mage Gall') }
|
|
33
|
+
# search.results.count.should == 1
|
|
34
|
+
# end
|
|
35
|
+
#end
|
|
36
36
|
end
|
|
@@ -1,38 +1,47 @@
|
|
|
1
1
|
require "spec_helper"
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
# GET /repos/:owner/:repo
|
|
5
|
-
# http://developer.github.com/v3/repos/
|
|
6
|
-
describe Elasticrepo::RepoSubset, :vcr => { :cassette_name => "v3/repos/_get/lgs/elasticrepo" } do
|
|
3
|
+
describe Elasticrepo::RepoSubset, :vcr => { :cassette_name => "v3/repos/_get/lgs/elasticrepo" } do
|
|
7
4
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
let(:repo) { Faraday.get('https://api.github.com/repos/lgs/elasticrepo').body }
|
|
6
|
+
let(:parsed) { Yajl::Parser.parse(repo) }
|
|
7
|
+
subject { Elasticrepo::RepoSubset.new(parsed) }
|
|
11
8
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
9
|
+
describe "#new" do
|
|
10
|
+
context "checks the full repo subset of extracted fields" do
|
|
11
|
+
its(:id) { should eq(9416045) }
|
|
12
|
+
its(:id) { should_not eq(94160450) }
|
|
16
13
|
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
its(:owner) { should eq("lgs") }
|
|
15
|
+
its(:owner) { should_not eq("l-g-s") }
|
|
19
16
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
17
|
+
its(:name) { should eq("elasticrepo") }
|
|
18
|
+
its(:url){ should eq("https://api.github.com/repos/lgs/elasticrepo") }
|
|
19
|
+
its(:description) { should eq("Indexing GitHub repository with ElasticSearch") }
|
|
23
20
|
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
its(:created_at) { should eq("2013-04-13T15:57:03Z") }
|
|
22
|
+
its(:created_at) { should_not eq("2013-05-13T15:57:03Z") }
|
|
26
23
|
|
|
27
|
-
|
|
28
|
-
|
|
24
|
+
its(:pushed_at) { should eq("2013-06-02T09:33:42Z") }
|
|
25
|
+
its(:pushed_at) { should_not eq("2013-07-02T09:33:42Z") }
|
|
29
26
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
27
|
+
its(:organization) { should eq("User") }
|
|
28
|
+
its(:full_name) { should eq("lgs/elasticrepo") }
|
|
29
|
+
its(:language) { should eq("Ruby") }
|
|
33
30
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
end
|
|
31
|
+
its(:updated_at) { should eq("2013-06-02T09:33:42Z") }
|
|
32
|
+
its(:updated_at) { should_not eq("2013-07-02T09:33:42Z") }
|
|
37
33
|
end
|
|
38
|
-
end
|
|
34
|
+
end
|
|
35
|
+
# describe "GET /repos/:owner/:repo" do
|
|
36
|
+
# it 'gets "lgs/elasticrepo" repository (JSON)' do
|
|
37
|
+
# VCR.use_cassette('v3/repos/#get/lgs/elasticrepo') do
|
|
38
|
+
# response = Faraday.get 'https://api.github.com/repos/lgs/elasticrepo'
|
|
39
|
+
# response.body.should include("1573")
|
|
40
|
+
# response.body.should include("9416045")
|
|
41
|
+
# response.body.should include("elasticrepo")
|
|
42
|
+
# response.body.should include("lgs/elasticrepo")
|
|
43
|
+
# response.body.should include("Indexing GitHub repository with ElasticSearch")
|
|
44
|
+
# end
|
|
45
|
+
# end
|
|
46
|
+
# end
|
|
47
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: elasticrepo
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Luca G. Soave
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2013-06-
|
|
11
|
+
date: 2013-06-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: yajl-ruby
|