elasticrepo 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/elasticrepo/extractor.rb +7 -8
- data/lib/elasticrepo/indexer.rb +31 -34
- data/lib/elasticrepo/version.rb +1 -1
- data/spec/elasticrepo/extractor_spec.rb +32 -48
- data/spec/elasticrepo/indexer_spec.rb +11 -11
- data/spec/elasticrepo/repo_subset_spec.rb +36 -27
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6133947dc64cdd43a54c157b7634045c72a4ef00
|
|
4
|
+
data.tar.gz: 7888a81625224d20237307fd4ecbe1683559cfe4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c81cfb59b733fb4c6928c8df1f088b445962ece19847fe5a1b9a87afa5f4e0ec22bc5e0b054a2ac1071080b768a40c4b33672bfd3e4e3bef6b464be9f292598d
|
|
7
|
+
data.tar.gz: 1c7a1a023f8f2fa38d4b8f359c5c7873c6f450caa8e98ff5ef17a11d0e20ccdcaf71b67ccc10157064ea70e0401ffe25711d38be29c4749b51ac8c8032dfef68
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
module Elasticrepo
|
|
2
|
-
|
|
3
|
-
attr_reader :repos
|
|
2
|
+
module Extractor
|
|
4
3
|
|
|
5
|
-
# GET /users/:user/starred
|
|
6
|
-
# extract fields subset from each repo
|
|
7
|
-
# then
|
|
8
|
-
|
|
4
|
+
# GET /users/:user/starred --> Octokit.starred owner
|
|
5
|
+
# extract fields subset from each repo --> Elasticrepo::RepoSubset
|
|
6
|
+
# then returns an array of lighter starred repositories by user --> .repos
|
|
7
|
+
#
|
|
8
|
+
def self.repos owner
|
|
9
9
|
repos = Octokit.starred owner
|
|
10
10
|
@repos = repos.map!{|repo|Elasticrepo::RepoSubset.new(repo)}
|
|
11
11
|
end
|
|
12
12
|
end
|
|
13
|
-
end
|
|
14
|
-
|
|
13
|
+
end
|
data/lib/elasticrepo/indexer.rb
CHANGED
|
@@ -1,20 +1,17 @@
|
|
|
1
1
|
module Elasticrepo
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
def initialize
|
|
5
|
-
@idx = "repositories"
|
|
6
|
-
end
|
|
2
|
+
module Indexer
|
|
3
|
+
INDEX = 'repositories'
|
|
7
4
|
|
|
8
5
|
# given a user, extract his starred repositories, then index the list with Tire
|
|
9
6
|
#
|
|
10
7
|
# GET /users/:user/starred
|
|
11
8
|
# http://developer.github.com/v3/repos/#list-user-repositories
|
|
12
9
|
#
|
|
13
|
-
def starred(owner)
|
|
14
|
-
repos = Elasticrepo::Extractor.
|
|
10
|
+
def self.starred(owner)
|
|
11
|
+
repos = Elasticrepo::Extractor.repos(owner)
|
|
15
12
|
# index the extraction
|
|
16
13
|
#
|
|
17
|
-
Tire::Index.new(
|
|
14
|
+
Tire::Index.new(INDEX) do
|
|
18
15
|
# Import documents
|
|
19
16
|
#
|
|
20
17
|
import repos
|
|
@@ -24,43 +21,43 @@ module Elasticrepo
|
|
|
24
21
|
end
|
|
25
22
|
end
|
|
26
23
|
|
|
27
|
-
def import_with_map
|
|
28
|
-
|
|
29
|
-
|
|
24
|
+
#def import_with_map
|
|
25
|
+
# repos = Elasticrepo::Extractor.new(@owner).repos
|
|
26
|
+
# Tire::Index.new(INDEX) do
|
|
30
27
|
# Create the index with proper mapping (if doesn not exist already)
|
|
31
28
|
#
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
end # end import
|
|
29
|
+
# create :mappings => {
|
|
30
|
+
# :question => {
|
|
31
|
+
# :properties => {
|
|
32
|
+
# :id => { :type => 'integer', :analyzer => 'keyword' },
|
|
33
|
+
# :owner => { :type => 'string', :analyzer => 'keyword' },
|
|
34
|
+
# :name => { :type => 'string', :analyzer => 'keyword' },
|
|
35
|
+
# #:full_name => { :type => 'string', :analyzer => 'keyword' },
|
|
36
|
+
# :url => { :type => 'string', :analyzer => 'snowball' },
|
|
37
|
+
# :description => { :type => 'string', :analyzer => 'snowball' },
|
|
38
|
+
# :organization => { :type => 'string', :analyzer => 'keyword' },
|
|
39
|
+
# :language => { :type => 'string', :analyzer => 'keyword' },
|
|
40
|
+
# :created_at => { :type => 'date', :analyzer => 'keyword' },
|
|
41
|
+
# :pushed_at => { :type => 'date', :analyzer => 'keyword' },
|
|
42
|
+
# :updated_at => { :type => 'date', :analyzer => 'keyword' }
|
|
43
|
+
# }
|
|
44
|
+
# }
|
|
45
|
+
# }
|
|
46
|
+
# import repos
|
|
47
|
+
# refresh
|
|
48
|
+
# end
|
|
49
|
+
#end # end import
|
|
53
50
|
|
|
54
51
|
def update
|
|
55
52
|
# Just refresh the index as is
|
|
56
53
|
#
|
|
57
|
-
Tire::Index.new(
|
|
54
|
+
Tire::Index.new(INDEX).refresh
|
|
58
55
|
end
|
|
59
56
|
|
|
60
57
|
def delete
|
|
61
58
|
# delete 'repositories' index
|
|
62
59
|
#
|
|
63
|
-
Tire::Index.new(
|
|
60
|
+
Tire::Index.new(INDEX).delete
|
|
64
61
|
end
|
|
65
62
|
end # end class
|
|
66
63
|
end # end module
|
data/lib/elasticrepo/version.rb
CHANGED
|
@@ -1,50 +1,34 @@
|
|
|
1
1
|
require "spec_helper"
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
its("repos[0]['id']") { extracted.repos[0].id.should eq 5392501 }
|
|
36
|
-
its("repos[0]['owner']") { extracted.repos[0].owner.should eq "cainlevy" }
|
|
37
|
-
its("repos[0]['url']") { extracted.repos[0].url.should eq "https://api.github.com/repos/cainlevy/photor" }
|
|
38
|
-
its("repos[0]['description']") { extracted.repos[0].description.should eq "Photo Organizer (in Ruby)" }
|
|
39
|
-
its("repos[0]['name']") { extracted.repos[0].name.should eq "photor" }
|
|
40
|
-
its("repos[0]['created_at']") { extracted.repos[0].created_at.should eq "2012-08-12T22:26:08Z" }
|
|
41
|
-
its("repos[0]['pushed_at']") { extracted.repos[0].pushed_at.should eq "2013-02-19T03:11:10Z" }
|
|
42
|
-
its("repos[0]['organization']") { extracted.repos[0].organization.should eq "User" }
|
|
43
|
-
its("repos[0]['full_name']") { extracted.repos[0].full_name.should eq "cainlevy/photor" }
|
|
44
|
-
its("repos[0]['language']") { extracted.repos[0].language.should eq "Ruby" }
|
|
45
|
-
its("repos[0]['updated_at']") { extracted.repos[0].updated_at.should eq "2013-03-13T02:05:33Z" }
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
end
|
|
3
|
+
# API ref.: GET /users/:user/starred
|
|
4
|
+
# http://developer.github.com/v3/repos/
|
|
5
|
+
# https://github.com/pengwynn/octokit/blob/master/lib/octokit/client/users.rb
|
|
6
|
+
#
|
|
7
|
+
describe Elasticrepo::Extractor, :vcr => { :cassette_name => "v3/users/_get/lapaty/starred" } do
|
|
8
|
+
subject(:extracted) { Elasticrepo::Extractor.repos("lapaty") }
|
|
9
|
+
|
|
10
|
+
describe "#new" do
|
|
11
|
+
context "debugging specs print out" do
|
|
12
|
+
it "print the content of running specs" do
|
|
13
|
+
print "extracted.class: #{extracted.class} \n \n"
|
|
14
|
+
print "extracted.inspect: #{extracted.inspect} \n \n"
|
|
15
|
+
print "extracted[0].id: #{extracted[0].id} \n \n"
|
|
16
|
+
print "extracted[0].owner #{extracted[0].owner} \n \n"
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
context "extract fields subset from each repo" do
|
|
20
|
+
its(:id) { extracted[0].id.should eq 5392501 }
|
|
21
|
+
its(:owner) { extracted[0].owner.should eq "cainlevy" }
|
|
22
|
+
its(:url) { extracted[0].url.should eq "https://api.github.com/repos/cainlevy/photor" }
|
|
23
|
+
its(:description) { extracted[0].description.should eq "Photo Organizer (in Ruby)" }
|
|
24
|
+
its(:name) { extracted[0].name.should eq "photor" }
|
|
25
|
+
its(:created_at) { extracted[0].created_at.should eq "2012-08-12T22:26:08Z" }
|
|
26
|
+
its(:pushed_at) { extracted[0].pushed_at.should eq "2013-02-19T03:11:10Z" }
|
|
27
|
+
its(:organization) { extracted[0].organization.should eq "User" }
|
|
28
|
+
its(:full_name) { extracted[0].full_name.should eq "cainlevy/photor" }
|
|
29
|
+
its(:language) { extracted[0].language.should eq "Ruby" }
|
|
30
|
+
its(:updated_at) { extracted[0].updated_at.should eq "2013-03-13T02:05:33Z" }
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
end
|
|
@@ -2,9 +2,7 @@ require "spec_helper"
|
|
|
2
2
|
|
|
3
3
|
describe Elasticrepo::Indexer, :vcr => { :cassette_name => "v3/users/_get/lapaty/starred" } do
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
subject(:starred_import) { indexer.starred("lapaty") }
|
|
7
|
-
|
|
5
|
+
subject(:starred_import) { Elasticrepo::Indexer.starred("lapaty") }
|
|
8
6
|
|
|
9
7
|
describe "#starred" do
|
|
10
8
|
context "print starred_import instance for debugging purposes" do
|
|
@@ -23,14 +21,16 @@ describe Elasticrepo::Indexer, :vcr => { :cassette_name => "v3/users/_get/lapaty
|
|
|
23
21
|
end
|
|
24
22
|
end
|
|
25
23
|
|
|
26
|
-
|
|
27
|
-
|
|
24
|
+
# this test doesn't work
|
|
25
|
+
#
|
|
26
|
+
#describe "#delete" do
|
|
27
|
+
# subject { starred_import.delete }
|
|
28
28
|
# see also: https://github.com/ferpetrelli/elasticsearch_tire_test
|
|
29
|
-
|
|
29
|
+
# let(:search) { Tire::Search::Search.new("repositories") }
|
|
30
30
|
# subject { starred_import }
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
end
|
|
31
|
+
# it "cannot find a repo matching 'Image Gall' description anymore" do
|
|
32
|
+
# search.query { string('description:*mage Gall') }
|
|
33
|
+
# search.results.count.should == 1
|
|
34
|
+
# end
|
|
35
|
+
#end
|
|
36
36
|
end
|
|
@@ -1,38 +1,47 @@
|
|
|
1
1
|
require "spec_helper"
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
# GET /repos/:owner/:repo
|
|
5
|
-
# http://developer.github.com/v3/repos/
|
|
6
|
-
describe Elasticrepo::RepoSubset, :vcr => { :cassette_name => "v3/repos/_get/lgs/elasticrepo" } do
|
|
3
|
+
describe Elasticrepo::RepoSubset, :vcr => { :cassette_name => "v3/repos/_get/lgs/elasticrepo" } do
|
|
7
4
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
let(:repo) { Faraday.get('https://api.github.com/repos/lgs/elasticrepo').body }
|
|
6
|
+
let(:parsed) { Yajl::Parser.parse(repo) }
|
|
7
|
+
subject { Elasticrepo::RepoSubset.new(parsed) }
|
|
11
8
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
9
|
+
describe "#new" do
|
|
10
|
+
context "checks the full repo subset of extracted fields" do
|
|
11
|
+
its(:id) { should eq(9416045) }
|
|
12
|
+
its(:id) { should_not eq(94160450) }
|
|
16
13
|
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
its(:owner) { should eq("lgs") }
|
|
15
|
+
its(:owner) { should_not eq("l-g-s") }
|
|
19
16
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
17
|
+
its(:name) { should eq("elasticrepo") }
|
|
18
|
+
its(:url){ should eq("https://api.github.com/repos/lgs/elasticrepo") }
|
|
19
|
+
its(:description) { should eq("Indexing GitHub repository with ElasticSearch") }
|
|
23
20
|
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
its(:created_at) { should eq("2013-04-13T15:57:03Z") }
|
|
22
|
+
its(:created_at) { should_not eq("2013-05-13T15:57:03Z") }
|
|
26
23
|
|
|
27
|
-
|
|
28
|
-
|
|
24
|
+
its(:pushed_at) { should eq("2013-06-02T09:33:42Z") }
|
|
25
|
+
its(:pushed_at) { should_not eq("2013-07-02T09:33:42Z") }
|
|
29
26
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
27
|
+
its(:organization) { should eq("User") }
|
|
28
|
+
its(:full_name) { should eq("lgs/elasticrepo") }
|
|
29
|
+
its(:language) { should eq("Ruby") }
|
|
33
30
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
end
|
|
31
|
+
its(:updated_at) { should eq("2013-06-02T09:33:42Z") }
|
|
32
|
+
its(:updated_at) { should_not eq("2013-07-02T09:33:42Z") }
|
|
37
33
|
end
|
|
38
|
-
end
|
|
34
|
+
end
|
|
35
|
+
# describe "GET /repos/:owner/:repo" do
|
|
36
|
+
# it 'gets "lgs/elasticrepo" repository (JSON)' do
|
|
37
|
+
# VCR.use_cassette('v3/repos/#get/lgs/elasticrepo') do
|
|
38
|
+
# response = Faraday.get 'https://api.github.com/repos/lgs/elasticrepo'
|
|
39
|
+
# response.body.should include("1573")
|
|
40
|
+
# response.body.should include("9416045")
|
|
41
|
+
# response.body.should include("elasticrepo")
|
|
42
|
+
# response.body.should include("lgs/elasticrepo")
|
|
43
|
+
# response.body.should include("Indexing GitHub repository with ElasticSearch")
|
|
44
|
+
# end
|
|
45
|
+
# end
|
|
46
|
+
# end
|
|
47
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: elasticrepo
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Luca G. Soave
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2013-06-
|
|
11
|
+
date: 2013-06-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: yajl-ruby
|