utopian_solrizer 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/utopian_solrizer.rb +72 -27
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c29d98e15ca4a899758346438daf1f019a25cd14729db438d7fb97fa33d53a49
|
|
4
|
+
data.tar.gz: a74f060f161335fa5f09c4ba92cd930fe97d30970d70b454ad5158e496cc581c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6750a5b85d1fb43fc483fdecd24aae8d482a5b14a105829d4cc30297765acc14faaeb18a1d14f879fcfa930025ab027267b738ce26a18f0c4b11f5751bd7c560
|
|
7
|
+
data.tar.gz: b8cd9f812ae80cb6c3f08cb8de3dba755331929402cbf803c714ce8a6765823648111f8d6b7bc6e732377daa558b9375dc7336f074c8ac368003a623d9239771
|
data/lib/utopian_solrizer.rb
CHANGED
|
@@ -7,7 +7,7 @@ module UtopianSolrizer
|
|
|
7
7
|
|
|
8
8
|
class << self
|
|
9
9
|
# define utopian solr fileds by following solr dynamic fields conventions
|
|
10
|
-
@@
|
|
10
|
+
@@utopian_solr_field_mapping = {
|
|
11
11
|
'id' => 'id',
|
|
12
12
|
'author' => 'author_ssi',
|
|
13
13
|
'moderator' => 'moderator_ssim',
|
|
@@ -17,58 +17,103 @@ module UtopianSolrizer
|
|
|
17
17
|
'title' => 'title_tsim',
|
|
18
18
|
'body' => 'body_tsim',
|
|
19
19
|
'created' => 'created_dts',
|
|
20
|
-
'repository' => 'repository_ssi'
|
|
20
|
+
'repository' => 'repository_ssi',
|
|
21
|
+
'flagged' => 'flagged_ssi',
|
|
22
|
+
'reviewed' => 'reviewed_ssi',
|
|
23
|
+
'pending' => 'pending_ssi',
|
|
24
|
+
'last_update'=> 'last_update_dts',
|
|
25
|
+
'active' => 'active_ssi'
|
|
21
26
|
}
|
|
22
27
|
|
|
28
|
+
# set default utopian fields to index
|
|
29
|
+
@@default_fields = ['author',
|
|
30
|
+
'moderator',
|
|
31
|
+
'permlink',
|
|
32
|
+
'category',
|
|
33
|
+
'tags',
|
|
34
|
+
'title',
|
|
35
|
+
'body',
|
|
36
|
+
'created',
|
|
37
|
+
'repository'
|
|
38
|
+
]
|
|
23
39
|
|
|
24
40
|
# Add/update a post in solr
|
|
25
|
-
def solrize_post(post, solr_options,
|
|
41
|
+
def solrize_post(post, solr_options, solr_fields=nil)
|
|
26
42
|
rsolr = RSolr.connect solr_options
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
@@fields['body'] => post.body,
|
|
37
|
-
@@fields['created'] => post.created,
|
|
38
|
-
@@fields['repository'] => post.json_metadata['repository']['html_url']
|
|
39
|
-
)
|
|
40
|
-
rsolr.commit
|
|
43
|
+
rsolr.add post_to_json(post, solr_fields=nil)
|
|
44
|
+
rsolr.commit
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Add/update posts in solr
|
|
48
|
+
def solrize_posts(posts, solr_options, solr_fields=nil)
|
|
49
|
+
posts_json = []
|
|
50
|
+
posts.each do |post|
|
|
51
|
+
posts_json.append post_to_json(post, solr_fields=nil)
|
|
41
52
|
end
|
|
53
|
+
rsolr = RSolr.connect solr_options
|
|
54
|
+
rsolr.add posts_json
|
|
55
|
+
rsolr.commit
|
|
42
56
|
end
|
|
43
57
|
|
|
44
|
-
#
|
|
45
|
-
def
|
|
46
|
-
|
|
47
|
-
|
|
58
|
+
# convert post fields to json format
|
|
59
|
+
def post_to_json(post, solr_fields=nil)
|
|
60
|
+
if solr_fields.nil? or solr_fields.length==0
|
|
61
|
+
solr_fields = @@default_fields
|
|
62
|
+
end
|
|
63
|
+
solr_json = {}
|
|
64
|
+
|
|
65
|
+
solr_fields.each do |f|
|
|
66
|
+
solr_json[@@utopian_solr_field_mapping[f]] = post_value(post, f)
|
|
48
67
|
end
|
|
68
|
+
solr_json
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# return post value for a particular field_name
|
|
72
|
+
def post_value(post, field_name)
|
|
73
|
+
post.id if field_name=='id'
|
|
74
|
+
post.author if field_name=='author'
|
|
75
|
+
post.moderator if field_name=='moderator'
|
|
76
|
+
post.permlink if field_name=='permlink'
|
|
77
|
+
post.json_metadata['type'] if field_name=='category'
|
|
78
|
+
post.json_metadata['tags'] if field_name=='tags'
|
|
79
|
+
post.title if field_name=='title'
|
|
80
|
+
post.body if field_name=='body'
|
|
81
|
+
post.created if field_name=='created'
|
|
82
|
+
post.json_metadata['repository']['html_url'] if field_name=='repository'
|
|
83
|
+
post.flagged if field_name=='flagged'
|
|
84
|
+
post.reviewed if field_name=='reviewed'
|
|
85
|
+
post.pending if field_name=='pending'
|
|
86
|
+
post.last_update if field_name=='last_update'
|
|
87
|
+
post.active if field_name=='active'
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Add posts by criterias
|
|
91
|
+
def solrize_posts_by_criterias(criterias, solr_options, solr_fields)
|
|
92
|
+
posts = UtopianRuby::UtopianRubyAPI.get_posts_obj(criterias)
|
|
93
|
+
solrize_posts(posts, solr_options, solr_fields=nil)
|
|
94
|
+
posts.length
|
|
49
95
|
end
|
|
50
96
|
|
|
51
97
|
# Add posts within minutes to Solr
|
|
52
|
-
def solrize_posts_within_minutes(criterias, solr_options, minutes)
|
|
98
|
+
def solrize_posts_within_minutes(criterias, solr_options, solr_fields, minutes)
|
|
53
99
|
total_updated = 0
|
|
54
100
|
limit = 100
|
|
55
101
|
skip = 0
|
|
56
102
|
reached_end = false
|
|
103
|
+
posts = Set.new
|
|
57
104
|
unless reached_end==true
|
|
58
105
|
criterias = {"limit":limit,"skip":skip}
|
|
59
106
|
UtopianRuby::UtopianRubyAPI.get_posts_obj(criterias).each do |post|
|
|
60
107
|
if (Time.parse(Time.now.to_s)-Time.parse(post.created)) <= minutes*60
|
|
61
|
-
|
|
62
|
-
total_updated = total_updated + 1
|
|
63
|
-
solrize_post(post, solr_options)
|
|
108
|
+
posts << post
|
|
64
109
|
else
|
|
65
110
|
reached_end=true
|
|
66
111
|
break
|
|
67
112
|
end
|
|
68
113
|
end
|
|
69
|
-
skip = skip + limit
|
|
70
114
|
end
|
|
71
|
-
|
|
115
|
+
solrize_posts(posts, solr_options, solr_fields=nil)
|
|
116
|
+
posts.length
|
|
72
117
|
end
|
|
73
118
|
|
|
74
119
|
# search particular posts
|