feedcellar 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9702dd39c1fc809f99edbc966c15b775417246d8
4
- data.tar.gz: aa5d0688709824dbd2fd7663b8b2ed35233381de
3
+ metadata.gz: 51ea5cd90472416d725cd56331eed41d118068a7
4
+ data.tar.gz: 49485354cc51a62bd6de68d3415d40829dd7e76d
5
5
  SHA512:
6
- metadata.gz: 6d94b13da24e5f1a2007f3ef03fc9a8e23f0bbec5654090c3e673a8d4c9ee18880bfdf26f01f794e310b158190b53585ef6e102df79f0666321e957d1119e779
7
- data.tar.gz: f25da1c3fbfc51520f42c8c5ab4b54b601721791375feb9d23e025a45a56a4f2c5a59e166367920a0b1de0ef7c0f3f9c9a1ad0a81c0347fc82252721c761d78c
6
+ metadata.gz: 49fb61eb5b7dd8b6dd09cff3d81b8e7ba03dab46ec7e78b9b378432cea67816660edecd7c76bbef3db5dea2d39773052e8779a706955cbaf2a3078944bd05017
7
+ data.tar.gz: ca2e4666fefdf69db4badd5da346fd254a45fd74d697e04b63310906c9d90a30e31edad8d7083e8514fa495de28299a5eecffea987ce41eefdacd182b7279649
@@ -1,6 +1,8 @@
1
1
  rvm:
2
- - 2.0.0
3
2
  - 2.1
4
3
  - 2.2
4
+ - 2.3.0
5
5
  before_install:
6
6
  - curl --silent --location https://github.com/groonga/groonga/raw/master/data/travis/setup.sh | sh
7
+ - gem update bundler
8
+ - export TZ=Asia/Tokyo
data/NEWS.md CHANGED
@@ -1,5 +1,17 @@
1
1
  # NEWS
2
2
 
3
+ ## 0.7.0: 2016-02-17
4
+
5
+ Speed up.
6
+
7
+ ### Changes
8
+
9
+ #### Improvements
10
+
11
+ * Accepted multiple URLs for register command.
12
+ * Parallelized collect command.
13
+ * Added "parallel" option to "collect" command.
14
+
3
15
  ## 0.6.0: 2015-04-06
4
16
 
5
17
  Support date related columns such as month.
@@ -39,10 +39,12 @@ Gem::Specification.new do |spec|
39
39
 
40
40
  spec.add_runtime_dependency("rroonga", ">= 5.0.0")
41
41
  spec.add_runtime_dependency("thor")
42
+ spec.add_runtime_dependency("parallel")
42
43
 
43
44
  spec.add_development_dependency("test-unit")
44
45
  spec.add_development_dependency("test-unit-notify")
45
46
  spec.add_development_dependency("test-unit-rr")
47
+ spec.add_development_dependency("webmock")
46
48
  spec.add_development_dependency("bundler")
47
49
  spec.add_development_dependency("rake")
48
50
  end
@@ -1,6 +1,6 @@
1
1
  # class Feedcellar::Command
2
2
  #
3
- # Copyright (C) 2013-2015 Masafumi Yokoyama <myokoym@gmail.com>
3
+ # Copyright (C) 2013-2016 Masafumi Yokoyama <myokoym@gmail.com>
4
4
  #
5
5
  # This library is free software; you can redistribute it and/or
6
6
  # modify it under the terms of the GNU Lesser General Public
@@ -17,6 +17,7 @@
17
17
  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18
18
 
19
19
  require "thor"
20
+ require "parallel"
20
21
  require "feedcellar/version"
21
22
  require "feedcellar/groonga_database"
22
23
  require "feedcellar/groonga_searcher"
@@ -42,14 +43,15 @@ module Feedcellar
42
43
  puts Feedcellar::VERSION
43
44
  end
44
45
 
45
- desc "register URL", "Register a URL."
46
- def register(url)
47
- resource = Resource.parse(url)
48
- return 1 unless resource
49
- return 1 if resource["xmlUrl"].empty?
50
-
46
+ desc "register URL...", "Register URLs."
47
+ def register(*urls)
51
48
  GroongaDatabase.new.open(@database_dir) do |database|
52
- database.register(resource["xmlUrl"], resource)
49
+ urls.each do |url|
50
+ resource = Resource.parse(url)
51
+ next unless resource
52
+ next if resource["xmlUrl"].empty?
53
+ database.register(resource["xmlUrl"], resource)
54
+ end
53
55
  end
54
56
  end
55
57
 
@@ -88,9 +90,11 @@ module Feedcellar
88
90
  end
89
91
 
90
92
  desc "collect", "Collect feeds from WWW."
93
+ option :parallel, :type => :boolean, :desc => "run on multiple processes"
91
94
  def collect
92
95
  GroongaDatabase.new.open(@database_dir) do |database|
93
- database.resources.each do |record|
96
+ resources = database.resources
97
+ collect_proc = lambda do |record|
94
98
  feed_url = record.xmlUrl
95
99
  next unless feed_url
96
100
 
@@ -105,6 +109,13 @@ module Feedcellar
105
109
  item.date)
106
110
  end
107
111
  end
112
+
113
+ if options[:parallel]
114
+ Parallel.each(resources,
115
+ &collect_proc)
116
+ else
117
+ resources.each(&collect_proc)
118
+ end
108
119
  end
109
120
  end
110
121
 
@@ -1,6 +1,6 @@
1
1
  # constant Feedcellar::VERSION
2
2
  #
3
- # Copyright (C) 2013-2015 Masafumi Yokoyama <myokoym@gmail.com>
3
+ # Copyright (C) 2013-2016 Masafumi Yokoyama <myokoym@gmail.com>
4
4
  #
5
5
  # This library is free software; you can redistribute it and/or
6
6
  # modify it under the terms of the GNU Lesser General Public
@@ -17,5 +17,5 @@
17
17
  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18
18
 
19
19
  module Feedcellar
20
- VERSION = "0.6.0"
20
+ VERSION = "0.7.0"
21
21
  end
@@ -0,0 +1,42 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <rss version="2.0"
3
+ xmlns:content="http://purl.org/rss/1.0/modules/content/"
4
+ xmlns:dc="http://purl.org/dc/elements/1.1/"
5
+ xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
6
+ xmlns:trackback="http://madskills.com/public/xml/rss/module/trackback/">
7
+ <channel>
8
+ <title>Feedcellar Test</title>
9
+ <link>http://localhost:20075</link>
10
+ <description>Test Site</description>
11
+ <item>
12
+ <title>Sample Article 1</title>
13
+ <link>http://localhost:20075/article1.html</link>
14
+ <description>Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do: once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, `and what is the use of a book,' thought Alice `without pictures or conversation?'
15
+
16
+ So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.
17
+ </description>
18
+ <pubDate>Mon, 01 Feb 2016 00:00:00 +0900</pubDate>
19
+ <dc:date>2016-02-01T00:00:00+09:00</dc:date>
20
+ </item>
21
+ <item>
22
+ <title>Sample Article 2</title>
23
+ <link>http://localhost:20075/article2.html</link>
24
+ <description>Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do: once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, `and what is the use of a book,' thought Alice `without pictures or conversation?'
25
+
26
+ So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.
27
+ </description>
28
+ <pubDate>Tue, 02 Feb 2016 00:00:00 +0900</pubDate>
29
+ <dc:date>2016-02-02T00:00:00+09:00</dc:date>
30
+ </item>
31
+ <item>
32
+ <title>Sample Article 3</title>
33
+ <link>http://localhost:20075/article3.html</link>
34
+ <description>Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do: once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, `and what is the use of a book,' thought Alice `without pictures or conversation?'
35
+
36
+ So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.
37
+ </description>
38
+ <pubDate>Wed, 03 Feb 2016 00:00:00 +0900</pubDate>
39
+ <dc:date>2016-02-03T00:00:00+09:00</dc:date>
40
+ </item>
41
+ </channel>
42
+ </rss>
@@ -1,17 +1,10 @@
1
1
  <?xml version="1.0" encoding="UTF-8"?>
2
2
  <opml version="1.0">
3
3
  <head>
4
- <title>M リーダー登録フィード</title>
4
+ <title>Feedcellar Test Feeds</title>
5
5
  </head>
6
6
  <body>
7
- <outline text="my_letter" title="my_letter" type="rss"
8
- xmlUrl="http://myokoym.github.io/entries.rss" htmlUrl="http://myokoym.github.com/"/>
9
- <outline text="Rubygems | Latest Versions for mister_fairy"
10
- title="Rubygems | Latest Versions for mister_fairy" type="rss"
11
- xmlUrl="https://rubygems.org/gems/mister_fairy/versions.atom" htmlUrl="https://rubygems.org/gems"/>
12
- <outline title="shogi" text="shogi">
13
- <outline text="函館将棋情報のブログ" title="函館将棋情報のブログ" type="rss"
14
- xmlUrl="http://blogs.yahoo.co.jp/mi807258/rss.xml" htmlUrl="http://rd.yahoo.co.jp/rss/l/blog/myblog/rss2/url/*http://blogs.yahoo.co.jp/mi807258"/>
15
- </outline>
7
+ <outline text="feedcellar-test" title="Feedcellar Test" type="rss"
8
+ xmlUrl="http://localhost:20075/feed.xml" htmlUrl="http://localhost:20075/"/>
16
9
  </body>
17
10
  </opml>
@@ -21,6 +21,7 @@
21
21
  require "test-unit"
22
22
  require "test/unit/notify"
23
23
  require "test/unit/rr"
24
+ require "webmock/test_unit"
24
25
 
25
26
  base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
26
27
  $LOAD_PATH.unshift(File.join(base_dir, "lib"))
@@ -2,7 +2,7 @@
2
2
  #
3
3
  # class CommandTest
4
4
  #
5
- # Copyright (C) 2013-2015 Masafumi Yokoyama <myokoym@gmail.com>
5
+ # Copyright (C) 2013-2016 Masafumi Yokoyama <myokoym@gmail.com>
6
6
  #
7
7
  # This library is free software; you can redistribute it and/or
8
8
  # modify it under the terms of the GNU Lesser General Public
@@ -32,6 +32,10 @@ class CommandTest < Test::Unit::TestCase
32
32
  ENV["FEEDCELLAR_HOME"] = @tmpdir
33
33
  @command = Feedcellar::Command.new
34
34
  @database_dir = @command.database_dir
35
+
36
+ rss = File.new(File.join(fixtures_dir, "feed.xml"))
37
+ stub_request(:get, "http://localhost:20075/feed.xml").
38
+ to_return(:status => 200, :body => rss, :headers => {})
35
39
  end
36
40
 
37
41
  def teardown
@@ -48,46 +52,15 @@ class CommandTest < Test::Unit::TestCase
48
52
  end
49
53
 
50
54
  def test_command
51
- # confirm register command if invalid URL
52
- s = ""
53
- io = StringIO.new(s)
54
- $stderr = io
55
- assert_equal(1, @command.register("hoge"))
56
- assert_equal("ERROR: Invalid URL\n", s)
57
- $stderr = STDERR
58
-
59
- # confirm register command
60
- resources = nil
61
- resources_path = File.join(fixtures_dir, "resources.dump")
62
- File.open(resources_path, "rb") do |file|
63
- resources = Marshal.load(file)
64
- end
65
- mock(Feedcellar::Resource).parse("http://myokoym.github.io/entries.rss") {resources[0]}
66
- mock(Feedcellar::Resource).parse("https://rubygems.org/gems/mister_fairy/versions.atom") {resources[1]}
67
- @command.register("http://myokoym.github.io/entries.rss")
68
- @command.register("https://rubygems.org/gems/mister_fairy/versions.atom")
69
- Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
70
- assert_equal(2, database.resources.size)
71
- end
72
-
73
55
  # confirm import command
74
56
  file = File.join(fixtures_dir, "subscriptions.xml")
75
57
  @command.import(file)
76
-
77
- feeds = nil
78
- feeds_path = File.join(fixtures_dir, "feeds.dump")
79
- File.open(feeds_path, "rb") do |file|
80
- feeds = Marshal.load(file)
81
- end
82
- mock(Feedcellar::Feed).parse("http://myokoym.github.io/entries.rss") {feeds[0]}
83
- mock(Feedcellar::Feed).parse("https://rubygems.org/gems/mister_fairy/versions.atom") {feeds[1]}
84
- mock(Feedcellar::Feed).parse("http://blogs.yahoo.co.jp/mi807258/rss.xml") {feeds[2]}
85
58
  @command.collect
86
59
 
87
60
  Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
88
61
  # NOTE: a tag of outline is not register.
89
- assert_equal(3, database.resources.size)
90
- assert_true(database.feeds.count > 0)
62
+ assert_equal(1, database.resources.size)
63
+ assert_equal(3, database.feeds.count)
91
64
  end
92
65
 
93
66
  # confirm export command
@@ -96,34 +69,36 @@ class CommandTest < Test::Unit::TestCase
96
69
  $stdout = io
97
70
  @command.export
98
71
  assert_equal(1, s.scan(/<opml/).size)
99
- assert_equal(3, s.scan(/<outline/).size)
72
+ assert_equal(1, s.scan(/<outline/).size)
100
73
  $stdout = STDOUT
101
74
 
102
75
  # confirm search command
103
76
  s = ""
104
77
  io = StringIO.new(s)
105
78
  $stdout = io
106
- @command.search("ruby")
107
- assert_true(s.size > 100)
79
+ @command.search("Alice")
80
+ assert_equal(<<-RESULT, s)
81
+ 2016/02/03 Sample Article 3
82
+ 2016/02/02 Sample Article 2
83
+ 2016/02/01 Sample Article 1
84
+ RESULT
108
85
  $stdout = STDOUT
109
86
 
110
- # confirm unregister command
111
- @command.unregister("my_letter")
112
- Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
113
- assert_equal(2, database.resources.size)
114
- end
115
- @command.unregister("https://rubygems.org/gems/mister_fairy/versions.atom")
116
- Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
117
- assert_equal(1, database.resources.size)
118
- end
119
-
120
87
  # confirm latest command
121
88
  s = ""
122
89
  io = StringIO.new(s)
123
90
  $stdout = io
124
91
  @command.latest
125
- assert_true(s.size > 0)
92
+ assert_equal(<<-FEEDS, s)
93
+ 2016/02/01 Sample Article 1 - Feedcellar Test
94
+ FEEDS
126
95
  $stdout = STDOUT
96
+
97
+ # confirm unregister command
98
+ @command.unregister("Feedcellar Test")
99
+ Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
100
+ assert_equal(0, database.resources.size)
101
+ end
127
102
  end
128
103
 
129
104
  def test_reset
@@ -153,22 +128,29 @@ class CommandTest < Test::Unit::TestCase
153
128
  File.join(File.dirname(__FILE__), "fixtures")
154
129
  end
155
130
 
131
+ class RegisterTest < self
132
+ def test_single
133
+ @command.register("http://localhost:20075/feed.xml")
134
+ Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
135
+ assert_equal(1, database.resources.size)
136
+ end
137
+ end
138
+
139
+ def test_multiple
140
+ rss = File.new(File.join(fixtures_dir, "feed.xml"))
141
+ stub_request(:get, "http://localhost:20076/feed.xml").
142
+ to_return(:status => 200, :body => rss, :headers => {})
143
+ @command.register("http://localhost:20075/feed.xml", "http://localhost:20076/feed.xml")
144
+ Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
145
+ assert_equal(2, database.resources.size)
146
+ end
147
+ end
148
+ end
149
+
156
150
  class DeleteTest < self
157
151
  def setup
158
152
  super
159
- resources = nil
160
- resources_path = File.join(fixtures_dir, "resources.dump")
161
- File.open(resources_path, "rb") do |file|
162
- resources = Marshal.load(file)
163
- end
164
- mock(Feedcellar::Resource).parse("http://myokoym.github.io/entries.rss") {resources[0]}
165
- @command.register("http://myokoym.github.io/entries.rss")
166
- feeds = nil
167
- feeds_path = File.join(fixtures_dir, "feeds.dump")
168
- File.open(feeds_path, "rb") do |file|
169
- feeds = Marshal.load(file)
170
- end
171
- mock(Feedcellar::Feed).parse("http://myokoym.github.io/entries.rss") {feeds[0]}
153
+ @command.register("http://localhost:20075/feed.xml")
172
154
  @command.collect
173
155
  end
174
156
 
@@ -181,28 +163,28 @@ class CommandTest < Test::Unit::TestCase
181
163
  @str = ""
182
164
  io = StringIO.new(@str)
183
165
  $stdout = io
184
- @command.search("ruby")
166
+ @command.search("Alice")
185
167
  $stdout = STDOUT
186
- assert_equal(13, @str.lines.size)
168
+ assert_equal(3, @str.lines.size)
187
169
 
188
- @command.delete("http://myokoym.github.com/entries/20131201/a0.html")
170
+ @command.delete("http://localhost:20075/article2.html")
189
171
 
190
172
  @str = ""
191
173
  io = StringIO.new(@str)
192
174
  $stdout = io
193
- @command.search("ruby")
194
- assert_equal(12, @str.lines.size)
175
+ @command.search("Alice")
176
+ assert_equal(2, @str.lines.size)
195
177
  end
196
178
 
197
179
  def test_by_resource_key
198
180
  @str = ""
199
181
  io = StringIO.new(@str)
200
182
  $stdout = io
201
- @command.search("ruby")
183
+ @command.search("Alice")
202
184
  $stdout = STDOUT
203
- assert_equal(13, @str.lines.size)
185
+ assert_equal(3, @str.lines.size)
204
186
 
205
- @command.delete(:resource_key => "http://myokoym.github.io/entries.rss")
187
+ @command.delete(:resource_key => "localhost:20075/feed.xml")
206
188
 
207
189
  @str = ""
208
190
  io = StringIO.new(@str)
@@ -25,6 +25,7 @@ class OpmlTest < Test::Unit::TestCase
25
25
  def test_parse
26
26
  file = File.join(File.dirname(__FILE__), "fixtures", "subscriptions.xml")
27
27
  outlines = Feedcellar::Opml.parse(file)
28
- assert_equal(4, outlines.size) # FIXME (feed * 3) + (tag * 1)
28
+ # TODO: Support tag
29
+ assert_equal(1, outlines.size)
29
30
  end
30
31
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: feedcellar
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masafumi Yokoyama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-05 00:00:00.000000000 Z
11
+ date: 2016-02-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rroonga
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: parallel
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: test-unit
43
57
  requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
80
94
  - - ">="
81
95
  - !ruby/object:Gem::Version
82
96
  version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: webmock
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
83
111
  - !ruby/object:Gem::Dependency
84
112
  name: bundler
85
113
  requirement: !ruby/object:Gem::Requirement
@@ -136,8 +164,7 @@ files:
136
164
  - lib/feedcellar/opml.rb
137
165
  - lib/feedcellar/resource.rb
138
166
  - lib/feedcellar/version.rb
139
- - test/fixtures/feeds.dump
140
- - test/fixtures/resources.dump
167
+ - test/fixtures/feed.xml
141
168
  - test/fixtures/subscriptions.xml
142
169
  - test/run-test.rb
143
170
  - test/test-command.rb
@@ -164,16 +191,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
164
191
  version: '0'
165
192
  requirements: []
166
193
  rubyforge_project:
167
- rubygems_version: 2.4.5
194
+ rubygems_version: 2.5.1
168
195
  signing_key:
169
196
  specification_version: 4
170
197
  summary: Full-Text Searchable RSS Feed Reader by Groonga
171
198
  test_files:
172
- - test/fixtures/feeds.dump
173
- - test/fixtures/resources.dump
199
+ - test/fixtures/feed.xml
174
200
  - test/fixtures/subscriptions.xml
175
201
  - test/run-test.rb
176
202
  - test/test-command.rb
177
203
  - test/test-groonga_database.rb
178
204
  - test/test-groonga_searcher.rb
179
205
  - test/test-opml.rb
206
+ has_rdoc:
Binary file
Binary file