feedcellar 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9702dd39c1fc809f99edbc966c15b775417246d8
-  data.tar.gz: aa5d0688709824dbd2fd7663b8b2ed35233381de
+  metadata.gz: 51ea5cd90472416d725cd56331eed41d118068a7
+  data.tar.gz: 49485354cc51a62bd6de68d3415d40829dd7e76d
 SHA512:
-  metadata.gz: 6d94b13da24e5f1a2007f3ef03fc9a8e23f0bbec5654090c3e673a8d4c9ee18880bfdf26f01f794e310b158190b53585ef6e102df79f0666321e957d1119e779
-  data.tar.gz: f25da1c3fbfc51520f42c8c5ab4b54b601721791375feb9d23e025a45a56a4f2c5a59e166367920a0b1de0ef7c0f3f9c9a1ad0a81c0347fc82252721c761d78c
+  metadata.gz: 49fb61eb5b7dd8b6dd09cff3d81b8e7ba03dab46ec7e78b9b378432cea67816660edecd7c76bbef3db5dea2d39773052e8779a706955cbaf2a3078944bd05017
+  data.tar.gz: ca2e4666fefdf69db4badd5da346fd254a45fd74d697e04b63310906c9d90a30e31edad8d7083e8514fa495de28299a5eecffea987ce41eefdacd182b7279649

data/.travis.yml

@@ -1,6 +1,8 @@
 rvm:
-  - 2.0.0
   - 2.1
   - 2.2
+  - 2.3.0
 before_install:
   - curl --silent --location https://github.com/groonga/groonga/raw/master/data/travis/setup.sh | sh
+  - gem update bundler
+  - export TZ=Asia/Tokyo

data/NEWS.md

@@ -1,5 +1,17 @@
 # NEWS
 
+## 0.7.0: 2016-02-17
+
+Speed up.
+
+### Changes
+
+#### Improvements
+
+  * Accepted multiple URLs for register command.
+  * Parallelized collect command.
+  * Added "parallel" option to "collect" command.
+
 ## 0.6.0: 2015-04-06
 
 Support date related columns such as month.

data/feedcellar.gemspec

@@ -39,10 +39,12 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency("rroonga", ">= 5.0.0")
   spec.add_runtime_dependency("thor")
+  spec.add_runtime_dependency("parallel")
 
   spec.add_development_dependency("test-unit")
   spec.add_development_dependency("test-unit-notify")
   spec.add_development_dependency("test-unit-rr")
+  spec.add_development_dependency("webmock")
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("rake")
 end

data/lib/feedcellar/command.rb

@@ -1,6 +1,6 @@
 # class Feedcellar::Command
 #
-# Copyright (C) 2013-2015  Masafumi Yokoyama <myokoym@gmail.com>
+# Copyright (C) 2013-2016  Masafumi Yokoyama <myokoym@gmail.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,6 +17,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "thor"
+require "parallel"
 require "feedcellar/version"
 require "feedcellar/groonga_database"
 require "feedcellar/groonga_searcher"
@@ -42,14 +43,15 @@ module Feedcellar
       puts Feedcellar::VERSION
     end
 
-    desc "register URL", "Register a URL."
-    def register(url)
-      resource = Resource.parse(url)
-      return 1 unless resource
-      return 1 if resource["xmlUrl"].empty?
-
+    desc "register URL...", "Register URLs."
+    def register(*urls)
       GroongaDatabase.new.open(@database_dir) do |database|
-        database.register(resource["xmlUrl"], resource)
+        urls.each do |url|
+          resource = Resource.parse(url)
+          next unless resource
+          next if resource["xmlUrl"].empty?
+          database.register(resource["xmlUrl"], resource)
+        end
       end
     end
 
@@ -88,9 +90,11 @@ module Feedcellar
     end
 
     desc "collect", "Collect feeds from WWW."
+    option :parallel, :type => :boolean, :desc => "run on multiple processes"
     def collect
       GroongaDatabase.new.open(@database_dir) do |database|
-        database.resources.each do |record|
+        resources = database.resources
+        collect_proc = lambda do |record|
           feed_url = record.xmlUrl
           next unless feed_url
 
@@ -105,6 +109,13 @@ module Feedcellar
                          item.date)
           end
         end
+
+        if options[:parallel]
+          Parallel.each(resources,
+                        &collect_proc)
+        else
+          resources.each(&collect_proc)
+        end
       end
     end
 

data/lib/feedcellar/version.rb

@@ -1,6 +1,6 @@
 # constant Feedcellar::VERSION
 #
-# Copyright (C) 2013-2015  Masafumi Yokoyama <myokoym@gmail.com>
+# Copyright (C) 2013-2016  Masafumi Yokoyama <myokoym@gmail.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,5 +17,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module Feedcellar
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/test/fixtures/feed.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0"
+  xmlns:content="http://purl.org/rss/1.0/modules/content/"
+  xmlns:dc="http://purl.org/dc/elements/1.1/"
+  xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
+  xmlns:trackback="http://madskills.com/public/xml/rss/module/trackback/">
+  <channel>
+    <title>Feedcellar Test</title>
+    <link>http://localhost:20075</link>
+    <description>Test Site</description>
+    <item>
+      <title>Sample Article 1</title>
+      <link>http://localhost:20075/article1.html</link>
+      <description>Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do: once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, `and what is the use of a book,' thought Alice `without pictures or conversation?'
+
+So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.
+</description>
+      <pubDate>Mon, 01 Feb 2016 00:00:00 +0900</pubDate>
+      <dc:date>2016-02-01T00:00:00+09:00</dc:date>
+    </item>
+    <item>
+      <title>Sample Article 2</title>
+      <link>http://localhost:20075/article2.html</link>
+      <description>Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do: once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, `and what is the use of a book,' thought Alice `without pictures or conversation?'
+
+So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.
+</description>
+      <pubDate>Tue, 02 Feb 2016 00:00:00 +0900</pubDate>
+      <dc:date>2016-02-02T00:00:00+09:00</dc:date>
+    </item>
+    <item>
+      <title>Sample Article 3</title>
+      <link>http://localhost:20075/article3.html</link>
+      <description>Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do: once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, `and what is the use of a book,' thought Alice `without pictures or conversation?'
+
+So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.
+</description>
+      <pubDate>Wed, 03 Feb 2016 00:00:00 +0900</pubDate>
+      <dc:date>2016-02-03T00:00:00+09:00</dc:date>
+    </item>
+  </channel>
+</rss>

data/test/fixtures/subscriptions.xml

@@ -1,17 +1,10 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <opml version="1.0">
     <head>
-        <title>M の リーダー登録フィード</title>
+        <title>Feedcellar Test Feeds</title>
     </head>
     <body>
-        <outline text="my_letter" title="my_letter" type="rss"
-            xmlUrl="http://myokoym.github.io/entries.rss" htmlUrl="http://myokoym.github.com/"/>
-        <outline text="Rubygems | Latest Versions for mister_fairy"
-            title="Rubygems | Latest Versions for mister_fairy" type="rss"
-            xmlUrl="https://rubygems.org/gems/mister_fairy/versions.atom" htmlUrl="https://rubygems.org/gems"/>
-        <outline title="shogi" text="shogi">
-            <outline text="函館将棋情報のブログ" title="函館将棋情報のブログ" type="rss"
-                xmlUrl="http://blogs.yahoo.co.jp/mi807258/rss.xml" htmlUrl="http://rd.yahoo.co.jp/rss/l/blog/myblog/rss2/url/*http://blogs.yahoo.co.jp/mi807258"/>
-        </outline>
+        <outline text="feedcellar-test" title="Feedcellar Test" type="rss"
+          xmlUrl="http://localhost:20075/feed.xml" htmlUrl="http://localhost:20075/"/>
     </body>
 </opml>

data/test/run-test.rb

@@ -21,6 +21,7 @@
 require "test-unit"
 require "test/unit/notify"
 require "test/unit/rr"
+require "webmock/test_unit"
 
 base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
 $LOAD_PATH.unshift(File.join(base_dir, "lib"))

data/test/test-command.rb

@@ -2,7 +2,7 @@
 #
 # class CommandTest
 #
-# Copyright (C) 2013-2015  Masafumi Yokoyama <myokoym@gmail.com>
+# Copyright (C) 2013-2016  Masafumi Yokoyama <myokoym@gmail.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -32,6 +32,10 @@ class CommandTest < Test::Unit::TestCase
     ENV["FEEDCELLAR_HOME"] = @tmpdir
     @command = Feedcellar::Command.new
     @database_dir = @command.database_dir
+
+    rss = File.new(File.join(fixtures_dir, "feed.xml"))
+    stub_request(:get, "http://localhost:20075/feed.xml").
+      to_return(:status => 200, :body => rss, :headers => {})
   end
 
   def teardown
@@ -48,46 +52,15 @@ class CommandTest < Test::Unit::TestCase
   end
 
   def test_command
-    # confirm register command if invalid URL
-    s = ""
-    io = StringIO.new(s)
-    $stderr = io
-    assert_equal(1, @command.register("hoge"))
-    assert_equal("ERROR: Invalid URL\n", s)
-    $stderr = STDERR
-
-    # confirm register command
-    resources = nil
-    resources_path = File.join(fixtures_dir, "resources.dump")
-    File.open(resources_path, "rb") do |file|
-      resources = Marshal.load(file)
-    end
-    mock(Feedcellar::Resource).parse("http://myokoym.github.io/entries.rss") {resources[0]}
-    mock(Feedcellar::Resource).parse("https://rubygems.org/gems/mister_fairy/versions.atom") {resources[1]}
-    @command.register("http://myokoym.github.io/entries.rss")
-    @command.register("https://rubygems.org/gems/mister_fairy/versions.atom")
-    Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
-      assert_equal(2, database.resources.size)
-    end
-
     # confirm import command
     file = File.join(fixtures_dir, "subscriptions.xml")
     @command.import(file)
-
-    feeds = nil
-    feeds_path = File.join(fixtures_dir, "feeds.dump")
-    File.open(feeds_path, "rb") do |file|
-      feeds = Marshal.load(file)
-    end
-    mock(Feedcellar::Feed).parse("http://myokoym.github.io/entries.rss") {feeds[0]}
-    mock(Feedcellar::Feed).parse("https://rubygems.org/gems/mister_fairy/versions.atom") {feeds[1]}
-    mock(Feedcellar::Feed).parse("http://blogs.yahoo.co.jp/mi807258/rss.xml") {feeds[2]}
     @command.collect
 
     Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
       # NOTE: a tag of outline is not register.
-      assert_equal(3, database.resources.size)
-      assert_true(database.feeds.count > 0)
+      assert_equal(1, database.resources.size)
+      assert_equal(3, database.feeds.count)
     end
 
     # confirm export command
@@ -96,34 +69,36 @@ class CommandTest < Test::Unit::TestCase
     $stdout = io
     @command.export
     assert_equal(1, s.scan(/<opml/).size)
-    assert_equal(3, s.scan(/<outline/).size)
+    assert_equal(1, s.scan(/<outline/).size)
     $stdout = STDOUT
 
     # confirm search command
     s = ""
     io = StringIO.new(s)
     $stdout = io
-    @command.search("ruby")
-    assert_true(s.size > 100)
+    @command.search("Alice")
+    assert_equal(<<-RESULT, s)
+2016/02/03 Sample Article 3
+2016/02/02 Sample Article 2
+2016/02/01 Sample Article 1
+    RESULT
     $stdout = STDOUT
 
-    # confirm unregister command
-    @command.unregister("my_letter")
-    Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
-      assert_equal(2, database.resources.size)
-    end
-    @command.unregister("https://rubygems.org/gems/mister_fairy/versions.atom")
-    Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
-      assert_equal(1, database.resources.size)
-    end
-
     # confirm latest command
     s = ""
     io = StringIO.new(s)
     $stdout = io
     @command.latest
-    assert_true(s.size > 0)
+    assert_equal(<<-FEEDS, s)
+2016/02/01 Sample Article 1 - Feedcellar Test
+    FEEDS
     $stdout = STDOUT
+
+    # confirm unregister command
+    @command.unregister("Feedcellar Test")
+    Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
+      assert_equal(0, database.resources.size)
+    end
   end
 
   def test_reset
@@ -153,22 +128,29 @@ class CommandTest < Test::Unit::TestCase
     File.join(File.dirname(__FILE__), "fixtures")
   end
 
+  class RegisterTest < self
+    def test_single
+      @command.register("http://localhost:20075/feed.xml")
+      Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
+        assert_equal(1, database.resources.size)
+      end
+    end
+
+    def test_multiple
+      rss = File.new(File.join(fixtures_dir, "feed.xml"))
+      stub_request(:get, "http://localhost:20076/feed.xml").
+        to_return(:status => 200, :body => rss, :headers => {})
+      @command.register("http://localhost:20075/feed.xml", "http://localhost:20076/feed.xml")
+      Feedcellar::GroongaDatabase.new.open(@database_dir) do |database|
+        assert_equal(2, database.resources.size)
+      end
+    end
+  end
+
   class DeleteTest < self
     def setup
       super
-      resources = nil
-      resources_path = File.join(fixtures_dir, "resources.dump")
-      File.open(resources_path, "rb") do |file|
-        resources = Marshal.load(file)
-      end
-      mock(Feedcellar::Resource).parse("http://myokoym.github.io/entries.rss") {resources[0]}
-      @command.register("http://myokoym.github.io/entries.rss")
-      feeds = nil
-      feeds_path = File.join(fixtures_dir, "feeds.dump")
-      File.open(feeds_path, "rb") do |file|
-        feeds = Marshal.load(file)
-      end
-      mock(Feedcellar::Feed).parse("http://myokoym.github.io/entries.rss") {feeds[0]}
+      @command.register("http://localhost:20075/feed.xml")
       @command.collect
     end
 
@@ -181,28 +163,28 @@ class CommandTest < Test::Unit::TestCase
       @str = ""
       io = StringIO.new(@str)
       $stdout = io
-      @command.search("ruby")
+      @command.search("Alice")
       $stdout = STDOUT
-      assert_equal(13, @str.lines.size)
+      assert_equal(3, @str.lines.size)
 
-      @command.delete("http://myokoym.github.com/entries/20131201/a0.html")
+      @command.delete("http://localhost:20075/article2.html")
 
       @str = ""
       io = StringIO.new(@str)
       $stdout = io
-      @command.search("ruby")
-      assert_equal(12, @str.lines.size)
+      @command.search("Alice")
+      assert_equal(2, @str.lines.size)
     end
 
     def test_by_resource_key
       @str = ""
       io = StringIO.new(@str)
       $stdout = io
-      @command.search("ruby")
+      @command.search("Alice")
       $stdout = STDOUT
-      assert_equal(13, @str.lines.size)
+      assert_equal(3, @str.lines.size)
 
-      @command.delete(:resource_key => "http://myokoym.github.io/entries.rss")
+      @command.delete(:resource_key => "localhost:20075/feed.xml")
 
       @str = ""
       io = StringIO.new(@str)

data/test/test-opml.rb

@@ -25,6 +25,7 @@ class OpmlTest < Test::Unit::TestCase
   def test_parse
     file = File.join(File.dirname(__FILE__), "fixtures", "subscriptions.xml")
     outlines = Feedcellar::Opml.parse(file)
-    assert_equal(4, outlines.size) # FIXME (feed * 3) + (tag * 1)
+    # TODO: Support tag
+    assert_equal(1, outlines.size)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedcellar
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Masafumi Yokoyama
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-05 00:00:00.000000000 Z
+date: 2016-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rroonga
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -136,8 +164,7 @@ files:
 - lib/feedcellar/opml.rb
 - lib/feedcellar/resource.rb
 - lib/feedcellar/version.rb
-- test/fixtures/feeds.dump
-- test/fixtures/resources.dump
+- test/fixtures/feed.xml
 - test/fixtures/subscriptions.xml
 - test/run-test.rb
 - test/test-command.rb
@@ -164,16 +191,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Full-Text Searchable RSS Feed Reader by Groonga
 test_files:
-- test/fixtures/feeds.dump
-- test/fixtures/resources.dump
+- test/fixtures/feed.xml
 - test/fixtures/subscriptions.xml
 - test/run-test.rb
 - test/test-command.rb
 - test/test-groonga_database.rb
 - test/test-groonga_searcher.rb
 - test/test-opml.rb
+has_rdoc: