utopian_solrizer 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/utopian_solrizer.rb +72 -27
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c29d98e15ca4a899758346438daf1f019a25cd14729db438d7fb97fa33d53a49
|
|
4
|
+
data.tar.gz: a74f060f161335fa5f09c4ba92cd930fe97d30970d70b454ad5158e496cc581c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6750a5b85d1fb43fc483fdecd24aae8d482a5b14a105829d4cc30297765acc14faaeb18a1d14f879fcfa930025ab027267b738ce26a18f0c4b11f5751bd7c560
|
|
7
|
+
data.tar.gz: b8cd9f812ae80cb6c3f08cb8de3dba755331929402cbf803c714ce8a6765823648111f8d6b7bc6e732377daa558b9375dc7336f074c8ac368003a623d9239771
|
data/lib/utopian_solrizer.rb
CHANGED
|
@@ -7,7 +7,7 @@ module UtopianSolrizer
|
|
|
7
7
|
|
|
8
8
|
class << self
|
|
9
9
|
# define utopian solr fileds by following solr dynamic fields conventions
|
|
10
|
-
@@
|
|
10
|
+
@@utopian_solr_field_mapping = {
|
|
11
11
|
'id' => 'id',
|
|
12
12
|
'author' => 'author_ssi',
|
|
13
13
|
'moderator' => 'moderator_ssim',
|
|
@@ -17,58 +17,103 @@ module UtopianSolrizer
|
|
|
17
17
|
'title' => 'title_tsim',
|
|
18
18
|
'body' => 'body_tsim',
|
|
19
19
|
'created' => 'created_dts',
|
|
20
|
-
'repository' => 'repository_ssi'
|
|
20
|
+
'repository' => 'repository_ssi',
|
|
21
|
+
'flagged' => 'flagged_ssi',
|
|
22
|
+
'reviewed' => 'reviewed_ssi',
|
|
23
|
+
'pending' => 'pending_ssi',
|
|
24
|
+
'last_update'=> 'last_update_dts',
|
|
25
|
+
'active' => 'active_ssi'
|
|
21
26
|
}
|
|
22
27
|
|
|
28
|
+
# set default utopian fields to index
|
|
29
|
+
@@default_fields = ['author',
|
|
30
|
+
'moderator',
|
|
31
|
+
'permlink',
|
|
32
|
+
'category',
|
|
33
|
+
'tags',
|
|
34
|
+
'title',
|
|
35
|
+
'body',
|
|
36
|
+
'created',
|
|
37
|
+
'repository'
|
|
38
|
+
]
|
|
23
39
|
|
|
24
40
|
# Add/update a post in solr
|
|
25
|
-
def solrize_post(post, solr_options,
|
|
41
|
+
def solrize_post(post, solr_options, solr_fields=nil)
|
|
26
42
|
rsolr = RSolr.connect solr_options
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
@@fields['body'] => post.body,
|
|
37
|
-
@@fields['created'] => post.created,
|
|
38
|
-
@@fields['repository'] => post.json_metadata['repository']['html_url']
|
|
39
|
-
)
|
|
40
|
-
rsolr.commit
|
|
43
|
+
rsolr.add post_to_json(post, solr_fields=nil)
|
|
44
|
+
rsolr.commit
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Add/update posts in solr
|
|
48
|
+
def solrize_posts(posts, solr_options, solr_fields=nil)
|
|
49
|
+
posts_json = []
|
|
50
|
+
posts.each do |post|
|
|
51
|
+
posts_json.append post_to_json(post, solr_fields=nil)
|
|
41
52
|
end
|
|
53
|
+
rsolr = RSolr.connect solr_options
|
|
54
|
+
rsolr.add posts_json
|
|
55
|
+
rsolr.commit
|
|
42
56
|
end
|
|
43
57
|
|
|
44
|
-
#
|
|
45
|
-
def
|
|
46
|
-
|
|
47
|
-
|
|
58
|
+
# convert post fields to json format
|
|
59
|
+
def post_to_json(post, solr_fields=nil)
|
|
60
|
+
if solr_fields.nil? or solr_fields.length==0
|
|
61
|
+
solr_fields = @@default_fields
|
|
62
|
+
end
|
|
63
|
+
solr_json = {}
|
|
64
|
+
|
|
65
|
+
solr_fields.each do |f|
|
|
66
|
+
solr_json[@@utopian_solr_field_mapping[f]] = post_value(post, f)
|
|
48
67
|
end
|
|
68
|
+
solr_json
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# return post value for a particular field_name
|
|
72
|
+
def post_value(post, field_name)
|
|
73
|
+
post.id if field_name=='id'
|
|
74
|
+
post.author if field_name=='author'
|
|
75
|
+
post.moderator if field_name=='moderator'
|
|
76
|
+
post.permlink if field_name=='permlink'
|
|
77
|
+
post.json_metadata['type'] if field_name=='category'
|
|
78
|
+
post.json_metadata['tags'] if field_name=='tags'
|
|
79
|
+
post.title if field_name=='title'
|
|
80
|
+
post.body if field_name=='body'
|
|
81
|
+
post.created if field_name=='created'
|
|
82
|
+
post.json_metadata['repository']['html_url'] if field_name=='repository'
|
|
83
|
+
post.flagged if field_name=='flagged'
|
|
84
|
+
post.reviewed if field_name=='reviewed'
|
|
85
|
+
post.pending if field_name=='pending'
|
|
86
|
+
post.last_update if field_name=='last_update'
|
|
87
|
+
post.active if field_name=='active'
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Add posts by criterias
|
|
91
|
+
def solrize_posts_by_criterias(criterias, solr_options, solr_fields)
|
|
92
|
+
posts = UtopianRuby::UtopianRubyAPI.get_posts_obj(criterias)
|
|
93
|
+
solrize_posts(posts, solr_options, solr_fields=nil)
|
|
94
|
+
posts.length
|
|
49
95
|
end
|
|
50
96
|
|
|
51
97
|
# Add posts within minutes to Solr
|
|
52
|
-
def solrize_posts_within_minutes(criterias, solr_options, minutes)
|
|
98
|
+
def solrize_posts_within_minutes(criterias, solr_options, solr_fields, minutes)
|
|
53
99
|
total_updated = 0
|
|
54
100
|
limit = 100
|
|
55
101
|
skip = 0
|
|
56
102
|
reached_end = false
|
|
103
|
+
posts = Set.new
|
|
57
104
|
unless reached_end==true
|
|
58
105
|
criterias = {"limit":limit,"skip":skip}
|
|
59
106
|
UtopianRuby::UtopianRubyAPI.get_posts_obj(criterias).each do |post|
|
|
60
107
|
if (Time.parse(Time.now.to_s)-Time.parse(post.created)) <= minutes*60
|
|
61
|
-
|
|
62
|
-
total_updated = total_updated + 1
|
|
63
|
-
solrize_post(post, solr_options)
|
|
108
|
+
posts << post
|
|
64
109
|
else
|
|
65
110
|
reached_end=true
|
|
66
111
|
break
|
|
67
112
|
end
|
|
68
113
|
end
|
|
69
|
-
skip = skip + limit
|
|
70
114
|
end
|
|
71
|
-
|
|
115
|
+
solrize_posts(posts, solr_options, solr_fields=nil)
|
|
116
|
+
posts.length
|
|
72
117
|
end
|
|
73
118
|
|
|
74
119
|
# search particular posts
|