automatic 14.5.0 → 14.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f758274920596a6780c0d8976cc76a5a0344f84e
-  data.tar.gz: 1611da3699eaf8e52c0405c8a46491e719c10637
+  metadata.gz: c4f4a46d1a0cd3682792e621973c10dc7c537bd2
+  data.tar.gz: e33c52947ef78c95c46ccf3a32df3153ee37606c
 SHA512:
-  metadata.gz: 57cb29eaff31e98564961be7271cd3bd6f4591237fd13cd99098e693f41bc8238ba86a648e510bc911e2fa28bacaa3bc207c68221bf200cb6743e3a945842171
-  data.tar.gz: 677ebbbd71a1b4396ef571ca05878a144890841f7f52cfa2635c05f194a79acf7a5a0e5ca2964b0c7f20b7f8433cdd1efd561669d29380ad1f0c28bb1bf843da
+  metadata.gz: d6540fcf48d19633a23ef3e0a56da11482c9be88a3a97225f4103547f783e68fc9a5313780269c82d492aad8327876e2f7f280d73d67e05a9c940a42812eae73
+  data.tar.gz: 764359815342acb9cdc60b7fa927ef3b84981333f8c064f64a2a663b72cdfa0a9183df8972d1062161839f27f6807855324e806754c6a80491dbc8cb76cae1ed

data/VERSION

@@ -1 +1 @@
-14.5.0
+14.10.1

data/automatic.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: automatic 14.5.0 ruby lib
+# stub: automatic 14.10.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "automatic"
-  s.version = "14.5.0"
+  s.version = "14.10.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["id774"]
-  s.date = "2014-05-31"
+  s.date = "2014-10-23"
   s.description = "Ruby framework for the general-purpose automatic processing"
   s.email = "idnanashi@gmail.com"
   s.executables = ["automatic"]
@@ -47,8 +47,11 @@ Gem::Specification.new do |s|
     "plugins/custom_feed/svn_log.rb",
     "plugins/filter/absolute_uri.rb",
     "plugins/filter/accept.rb",
+    "plugins/filter/clear.rb",
+    "plugins/filter/description_link.rb",
     "plugins/filter/full_feed.rb",
     "plugins/filter/github_feed.rb",
+    "plugins/filter/google_news.rb",
     "plugins/filter/ignore.rb",
     "plugins/filter/image.rb",
     "plugins/filter/image_source.rb",
@@ -97,8 +100,11 @@ Gem::Specification.new do |s|
     "spec/plugins/custom_feed/svn_log_spec.rb",
     "spec/plugins/filter/absolute_uri_spec.rb",
     "spec/plugins/filter/accept_spec.rb",
+    "spec/plugins/filter/clear_spec.rb",
+    "spec/plugins/filter/description_link_spec.rb",
     "spec/plugins/filter/full_feed_spec.rb",
     "spec/plugins/filter/github_feed_spec.rb",
+    "spec/plugins/filter/google_news_spec.rb",
     "spec/plugins/filter/ignore_spec.rb",
     "spec/plugins/filter/image_source_spec.rb",
     "spec/plugins/filter/image_spec.rb",
@@ -141,8 +147,10 @@ Gem::Specification.new do |s|
     "test/integration/test_activerecord.yml",
     "test/integration/test_add_pocket.yml",
     "test/integration/test_chan_toru.yml",
+    "test/integration/test_descriptionlink.yml",
     "test/integration/test_fluentd.yml",
     "test/integration/test_fulltext.yml",
+    "test/integration/test_google_news.yml",
     "test/integration/test_googlealert.yml",
     "test/integration/test_hatenabookmark.yml",
     "test/integration/test_ignore.yml",

data/doc/ChangeLog

@@ -1,3 +1,18 @@
+=== 14.10.0 / 2014-10-23
+
+* Using Logger
+
+* Improvement File Saving Plugin
+
+  * Store::FullText will ignore duplicated titled entry.
+
+* Added new plugins.
+
+  * Filter::Clear
+  * Filter::GoogleNews
+  * Filter::DescriptionLink
+
+
 === 14.5.0 / 2014-05-31
 
 * Update to activesupport/record 4.1.

data/doc/PLUGINS

@@ -216,6 +216,18 @@ SubscriptionChanToru
       retry:    RETRY_COUNT
 
 
+FilterClear
+-----------
+[Path]
+ /plugins/filter/clear.rb
+
+[Abstract]
+ Clear Pipeline.
+
+[Syntax]
+  - module: FilterClear
+
+
 FilterSort
 ----------
 [Path]
@@ -381,6 +393,34 @@ FilterTumblrResize
   - module: FilterTumblrResize
 
 
+FilterDescriptionLink
+---------------------
+[Path]
+ /plugins/filter/description_link.rb
+
+[Abstract]
+ Pickup http or https URL from description.
+ Re-write link to the URL.
+
+[Syntax]
+  - module: FilterDescriptionLink
+    config:
+      clear_description: 1 # Set empty to description.
+      get_title: 1 # Get title after re-write link.
+
+
+FilterGoogleNews
+----------------
+[Path]
+ /plugins/filter/google_news.rb
+
+[Abstract]
+ Re-write Google News link to a new link of the redirect.
+
+[Syntax]
+  - module: FilterGoogleNews
+
+
 StorePermalink
 --------------
 [Path]

data/doc/PLUGINS.ja

@@ -216,6 +216,18 @@ SubscriptionChanToru
       retry: エラー時のリトライ回数 (回数, 省略時 0)
 
 
+FilterClear
+-----------
+[パス]
+ /plugins/filter/clear.rb
+
+[概要]
+ パイプラインをカラにする
+
+[レシピ記法]
+  - module: FilterClear
+
+
 FilterSort
 ----------
 [パス]
@@ -382,6 +394,36 @@ FilterTumblrResize
   - module: FilterTumblrResize
 
 
+FilterDescriptionLink
+---------------------
+[パス]
+ /plugins/filter/description_link.rb
+
+[概要]
+ 日付の昇順にソートする
+ もし config で sort に desc が指定されると
+ 昇順ではなく降順にソートする
+
+[レシピ記法]
+  - module: FilterDescriptionLink
+    config:
+      clear_description: 1 # Set empty to description.
+      get_title: 1 # Get title after re-write link.
+
+
+FilterGoogleNews
+----------------
+[パス]
+ /plugins/filter/google_news.rb
+
+[概要]
+ Google News のリンクを
+ リダイレクト先のものに書き換える
+
+[レシピ記法]
+  - module: FilterGoogleNews
+
+
 StorePermalink
 --------------
 [パス]

data/lib/automatic/log.rb

@@ -2,10 +2,12 @@
 # Name::      Automatic::Log
 # Author::    774 <http://id774.net>
 # Created::   Feb 20, 2012
-# Updated::   Mar 11, 2012
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Oct 09, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
+require 'logger'
+
 module Automatic
   module Log
     LOG_LEVELS = ['info', 'warn', 'error', 'none']
@@ -14,13 +16,16 @@ module Automatic
       @level = level
     end
 
-    def self.puts(level, message)
+    def self.logger
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    def self.puts(level = :info, message)
       if LOG_LEVELS.index(@level).to_i > LOG_LEVELS.index(level).to_i
         return
       end
-      t = Time.now.strftime("%Y/%m/%d %X")
-      print log = "#{t} [#{level}] #{message}\n"
-      return log
+      logger.send(level, message)
     end
+
   end
 end

data/lib/automatic/version.rb

@@ -1,3 +1,3 @@
 module Automatic
-  VERSION = "14.5.0"
+  VERSION = "14.10.1"
 end

data/plugins/filter/clear.rb

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Name::      Automatic::Plugin::Filter::Clear
+# Author::    774 <http://id774.net>
+# Created::   Oct 20, 2014
+# Updated::   Oct 20, 2014
+# Copyright:: Copyright (c) 2014 Automatic Ruby Developers.
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+module Automatic::Plugin
+  class FilterClear
+    def initialize(config, pipeline=[])
+      @config = config
+      @pipeline = pipeline
+    end
+
+    def run
+      []
+    end
+  end
+end

data/plugins/filter/description_link.rb

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+# Name::      Automatic::Plugin::Filter::DescriptionLink
+# Author::    774 <http://id774.net>
+# Created::   Oct 03, 2014
+# Updated::   Oct 16, 2014
+# Copyright:: Copyright (c) 2014 Automatic Ruby Developers.
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+module Automatic::Plugin
+  class FilterDescriptionLink
+    require 'uri'
+    require 'nkf'
+
+    def initialize(config, pipeline=[])
+      @config = config
+      @pipeline = pipeline
+    end
+
+    def run
+      @return_feeds = []
+      @pipeline.each {|feeds|
+        new_feeds = []
+        unless feeds.nil?
+          feeds.items.each {|feed|
+            new_feeds << rewrite_link(feed)
+          }
+        end
+        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds)
+      }
+      @return_feeds
+    end
+
+    private
+
+    def get_title(url)
+      new_title = nil
+      if url.class == String
+        url.gsub!(Regexp.new("[^#{URI::PATTERN::ALNUM}\/\:\?\=&~,\.\(\)#]")) {|match| ERB::Util.url_encode(match)}
+        begin
+          read_data = NKF.nkf("--utf8", open(url).read)
+          get_text = Nokogiri::HTML.parse(read_data, nil, 'utf8').xpath('//title').text
+          new_title = get_text if get_text.class == String
+        rescue
+          Automatic::Log.puts("warn", "Failed in get title for: #{url}")
+        end
+      end
+
+      new_title
+    end
+
+    def rewrite_link(feed)
+      new_link = URI.extract(feed.description, %w{http https}).uniq.last
+      feed.link = new_link unless new_link.nil?
+
+      if @config.class == Hash
+        if @config['clear_description'] == 1
+          feed.description = ""
+        end
+
+        if @config['get_title'] == 1
+          begin
+            new_title = get_title(feed.link)
+            feed.title = new_title unless new_title.nil?
+          rescue OpenURI::HTTPError
+            Automatic::Log.puts("warn", "404 Not Found in get title process.")
+          end
+        end
+      end
+
+      feed
+    end
+  end
+end
+

data/plugins/filter/google_news.rb

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Name::      Automatic::Plugin::Filter::GoogleNews
+# Author::    774 <http://id774.net>
+# Created::   Oct 12, 2014
+# Updated::   Oct 14, 2014
+# Copyright:: Copyright (c) 2014 Automatic Ruby Developers.
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+module Automatic::Plugin
+  class FilterGoogleNews
+    require 'uri'
+    require 'nkf'
+
+    def initialize(config, pipeline=[])
+      @config = config
+      @pipeline = pipeline
+    end
+
+    def run
+      @return_feeds = []
+      @pipeline.each {|feeds|
+        new_feeds = []
+        unless feeds.nil?
+          feeds.items.each {|feed|
+            new_feeds << rewrite_link(feed) unless feed.link.nil?
+          }
+        end
+        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds)
+      }
+      @return_feeds
+    end
+
+    private
+
+    def rewrite_link(feed)
+      if feed.link.class == String
+        if feed.link.index("http://news.google.com")
+          matched = feed.link.match(/(&url=)/)
+          unless matched.nil?
+            new_link = matched.post_match
+            feed.link = new_link unless new_link.nil?
+          end
+        end
+      end
+
+      feed
+    end
+  end
+end
+

data/plugins/store/database.rb

@@ -4,7 +4,7 @@
 #             774 <http://id774.net>
 #             soramugi <http://soramugi.net>
 # Created::   Feb 27, 2012
-# Updated::   Feb 21, 2014
+# Updated::   Oct 09, 2014
 # Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -21,7 +21,12 @@ module Automatic::Plugin
           new_feeds = []
           feeds.items.each {|feed|
             unless feed.link.nil?
-              unless existing_records.detect {|b| b.try(unique_key) == feed.link }
+              if unique_keys.length > 1
+                detection = existing_records.detect {|b| b.try(unique_keys[0]) == feed.link || b.try(unique_keys[1]) == feed.title }
+              else
+                detection = existing_records.detect {|b| b.try(unique_keys[0]) == feed.link }
+              end
+              unless detection
                 yield(feed)
                 new_feeds << feed
               end

data/plugins/store/full_text.rb

@@ -2,8 +2,8 @@
 # Name::      Automatic::Plugin::Store::FullText
 # Author::    774 <http://id774.net>
 # Created::   Feb 26, 2012
-# Updated::   Jun 14, 2012
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Oct 09, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 require 'plugins/store/database'
@@ -30,8 +30,8 @@ module Automatic::Plugin
       }
     end
 
-    def unique_key
-      :link
+    def unique_keys
+      [:link, :title]
     end
 
     def model_class

data/plugins/store/permalink.rb

@@ -3,8 +3,8 @@
 # Name::      Automatic::Plugin::Store::Permalink
 # Author::    774 <http://id774.net>
 # Created::   Feb 22, 2012
-# Updated::   Sep 18, 2012
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Oct 09, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 require 'plugins/store/database'
@@ -28,8 +28,8 @@ module Automatic::Plugin
       }
     end
 
-    def unique_key
-      :url
+    def unique_keys
+      [:url]
     end
 
     def model_class

data/spec/lib/automatic/log_spec.rb

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 # Name::      Auaotmatic::Log
 # Author::    soramugi
+#             774 <http://id774.net>
 # Created::   May 19, 2013
-# Updated::   May 19, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Oct 09, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 require File.expand_path(File.join(File.dirname(__FILE__) ,'../../spec_helper'))

data/spec/lib/automatic_spec.rb

@@ -3,7 +3,7 @@
 # Author::    kzgs
 #             774 <http://id774.net>
 # Created::   Mar  9, 2012
-# Updated::   May 31, 2014
+# Updated::   Oct 23, 2014
 # Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -27,7 +27,7 @@ describe Automatic do
   describe "#version" do
     subject { Automatic.const_get(:VERSION) }
 
-    it { expect(subject).to eq "14.5.0" }
+    it { expect(subject).to eq "14.10.1" }
   end
 
   describe "#(root)_dir" do

data/spec/plugins/filter/clear_spec.rb

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+# Name::      Automatic::Plugin::Filter::Clear
+# Author::    774 <http://id774.net>
+# Created::   Oct 20, 2014
+# Updated::   Oct 20, 2014
+# Copyright:: Copyright (c) 2014 Automatic Ruby Developers.
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
+
+require 'filter/clear'
+
+describe Automatic::Plugin::FilterClear do
+
+  context "should be cleared" do
+    subject {
+      Automatic::Plugin::FilterClear.new({
+      },
+        AutomaticSpec.generate_pipeline {
+          feed {
+            item "http://hogefuga.com", "",
+            "aaaabbbccc"
+          }
+          feed {
+            item "http://aaaabbbccc.com", "",
+            "hogefugahoge"
+          }
+          feed {
+            item "http://aaabbbccc.com", "",
+            "aaaaaaaaaacccdd"
+          }
+          feed {
+            item "http://aaccc.com", "",
+            "aaaabbbccc"
+            item "http://aabbccc.com", "",
+            "aabbbccc"
+          }
+          feed {
+            item "http://cccddd.com", "",
+            "aabbbcccdd"
+          }
+        }
+      )}
+
+    describe "#run" do
+      its(:run) { should have(0).feeds }
+    end
+  end
+end

data/spec/plugins/filter/description_link_spec.rb

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+# Name::      Automatic::Plugin::Filter::DescriptionLink
+# Author::    774 <http://id774.net>
+# Created::   Oct 03, 2014
+# Updated::   Oct 03, 2014
+# Copyright:: Copyright (c) 2014 Automatic Ruby Developers.
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
+
+require 'filter/description_link'
+
+describe Automatic::Plugin::FilterDescriptionLink do
+
+  context "It should be rewrite link based on the description" do
+
+    subject {
+      Automatic::Plugin::FilterDescriptionLink.new(
+        {},
+        AutomaticSpec.generate_pipeline {
+          feed {
+            item "http://test1.id774.net",
+            "dummy title",
+            "aaa bbb ccc http://test2.id774.net ddd eee",
+            "Mon, 07 Mar 2011 15:54:11 +0900"
+          }
+        }
+      )
+    }
+
+    describe "#run" do
+      its(:run) { should have(1).feeds }
+
+      specify {
+        subject.run
+        subject.instance_variable_get(:@pipeline)[0].items[0].link.
+        should == "http://test2.id774.net"
+        subject.instance_variable_get(:@pipeline)[0].items[0].description.
+        should == "aaa bbb ccc http://test2.id774.net ddd eee"
+      }
+    end
+  end
+
+  context "It should be empty description if clear_description specified" do
+
+    subject {
+      Automatic::Plugin::FilterDescriptionLink.new({
+          'clear_description' => 1,
+        },
+        AutomaticSpec.generate_pipeline {
+          feed {
+            item "http://test1.id774.net",
+            "dummy title",
+            "aaa bbb ccc http://test2.id774.net ddd eee",
+            "Mon, 07 Mar 2011 15:54:11 +0900"
+          }
+        }
+      )
+    }
+
+    describe "#run" do
+      its(:run) { should have(1).feeds }
+
+      specify {
+        subject.run
+        subject.instance_variable_get(:@pipeline)[0].items[0].link.
+        should == "http://test2.id774.net"
+        subject.instance_variable_get(:@pipeline)[0].items[0].description.
+        should == ""
+      }
+    end
+  end
+
+  context "It should be got title if get_title specified" do
+
+    subject {
+      Automatic::Plugin::FilterDescriptionLink.new({
+          'get_title' => 1,
+        },
+        AutomaticSpec.generate_pipeline {
+          feed {
+            item "http://test1.id774.net",
+            "dummy title",
+            "aaa bbb ccc http://blog.id774.net/post/2014/10/01/531/ ddd eee",
+            "Mon, 07 Mar 2011 15:54:11 +0900"
+          }
+        }
+      )
+    }
+
+    describe "#run" do
+      its(:run) { should have(1).feeds }
+
+      specify {
+        subject.run
+        subject.instance_variable_get(:@pipeline)[0].items[0].link.
+        should == "http://blog.id774.net/post/2014/10/01/531/"
+        subject.instance_variable_get(:@pipeline)[0].items[0].title.
+        should == "二穂様は俺の嫁 | 774::Blog"
+        subject.instance_variable_get(:@pipeline)[0].items[0].description.
+        should == "aaa bbb ccc http://blog.id774.net/post/2014/10/01/531/ ddd eee"
+      }
+    end
+  end
+
+  context "It should be handling error if 404 Not Found" do
+
+    subject {
+      Automatic::Plugin::FilterDescriptionLink.new({
+          'get_title' => 1,
+        },
+        AutomaticSpec.generate_pipeline {
+          feed {
+            item "http://test1.id774.net",
+            "dummy title",
+            "aaa bbb ccc http://blog.id774.net/post/2014/10/01/532/ ddd eee",
+            "Mon, 07 Mar 2011 15:54:11 +0900"
+          }
+        }
+      )
+    }
+
+    describe "#run" do
+      its(:run) { should have(1).feeds }
+
+      specify {
+        subject.run
+        subject.instance_variable_get(:@pipeline)[0].items[0].link.
+        should == "http://blog.id774.net/post/2014/10/01/532/"
+        subject.instance_variable_get(:@pipeline)[0].items[0].title.
+        should == "dummy title"
+        subject.instance_variable_get(:@pipeline)[0].items[0].description.
+        should == "aaa bbb ccc http://blog.id774.net/post/2014/10/01/532/ ddd eee"
+      }
+    end
+  end
+
+end

data/spec/plugins/filter/google_news_spec.rb

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Name::      Automatic::Plugin::Filter::GoogleNews
+# Author::    774 <http://id774.net>
+# Created::   Oct 12, 2014
+# Updated::   Oct 12, 2014
+# Copyright:: Copyright (c) 2014 Automatic Ruby Developers.
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
+
+require 'filter/google_news'
+
+describe Automatic::Plugin::FilterGoogleNews do
+
+  context "It should be not rewrite link other urls" do
+
+    subject {
+      Automatic::Plugin::FilterGoogleNews.new(
+        {},
+        AutomaticSpec.generate_pipeline {
+          feed {
+            item "http://test1.id774.net",
+            "dummy title",
+            "aaa bbb ccc http://test2.id774.net ddd eee",
+            "Mon, 07 Mar 2011 15:54:11 +0900"
+          }
+        }
+      )
+    }
+
+    describe "#run" do
+      its(:run) { should have(1).feeds }
+
+      specify {
+        subject.run
+        subject.instance_variable_get(:@pipeline)[0].items[0].link.
+        should == "http://test1.id774.net"
+      }
+    end
+  end
+
+  context "It should be rewrite link for google news urls" do
+
+    subject {
+      Automatic::Plugin::FilterGoogleNews.new(
+        {},
+        AutomaticSpec.generate_pipeline {
+          feed {
+            item "http://news.google.com/news/url?sa=t&fd=R&ct2=us&usg=AFQjCNGhIFo1illQ6jFyVGPZtfkttFaJYQ&clid=c3a7d30bb8a4878e06b80cf16b898331&cid=52779138313507&ei=Pts3VLDAD4X7kgWO9oGIBg&url=http://www.yomiuri.co.jp/world/20141010-OYT1T50090.html",
+            "dummy title",
+            "aaa bbb ccc http://test2.id774.net ddd eee",
+            "Mon, 07 Mar 2011 15:54:11 +0900"
+          }
+        }
+      )
+    }
+
+    describe "#run" do
+      its(:run) { should have(1).feeds }
+
+      specify {
+        subject.run
+        subject.instance_variable_get(:@pipeline)[0].items[0].link.
+        should == "http://www.yomiuri.co.jp/world/20141010-OYT1T50090.html"
+      }
+    end
+  end
+
+end

data/spec/plugins/store/full_text_spec.rb

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 # Name::      Automatic::Plugin::Store::FullText
 # Author::    774 <http://id774.net>
-# Updated::   Jun 14, 2012
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Oct 16, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
@@ -26,13 +26,20 @@ describe Automatic::Plugin::StoreFullText do
   it "should store 1 record for the new blog entry" do
     instance = Automatic::Plugin::StoreFullText.new({"db" => @db_filename},
       AutomaticSpec.generate_pipeline {
-        feed { item "http://blog.id774.net/blogs/feed/" }
+        feed {
+          item "http://blog.id774.net/blogs/feed/",
+          "dummy title",
+          "aaa bbb ccc http://test2.id774.net ddd eee",
+          "Mon, 07 Mar 2011 15:54:11 +0900"
+        }
       }
     )
 
+    Automatic::Plugin::Blog.count.should eq 0
     lambda {
       instance.run.should have(1).feed
     }.should change(Automatic::Plugin::Blog, :count).by(1)
+    Automatic::Plugin::Blog.count.should eq 1
   end
 
   it "should not store record for the existent blog entry" do
@@ -42,10 +49,104 @@ describe Automatic::Plugin::StoreFullText do
       }
     )
 
+    Automatic::Plugin::Blog.count.should eq 0
     instance.run.should have(1).feed
     lambda {
       instance.run.should have(0).feed
     }.should change(Automatic::Plugin::Blog, :count).by(0)
+    instance.run.should have(0).feed
+    Automatic::Plugin::Blog.count.should eq 1
+  end
+
+  it "should not store record for the existent blog link" do
+    instance = Automatic::Plugin::StoreFullText.new({"db" => @db_filename},
+      AutomaticSpec.generate_pipeline {
+        feed {
+          item "http://blog.id774.net/post/100",
+          "dummy title 1",
+          "aaa bbb ccc http://test2.id774.net ddd eee",
+          "Mon, 07 Mar 2011 15:54:11 +0900"
+        }
+      }
+    )
+    instance2 = Automatic::Plugin::StoreFullText.new({"db" => @db_filename},
+      AutomaticSpec.generate_pipeline {
+        feed {
+          item "http://blog.id774.net/post/100",
+          "dummy title 2",
+          "aaa bbb ccc http://test2.id774.net ddd eee",
+          "Mon, 07 Mar 2011 15:54:11 +0900"
+        }
+      }
+    )
+
+    Automatic::Plugin::Blog.count.should eq 0
+    instance.run.should have(1).feed
+    lambda {
+      instance2.run.should have(0).feed
+    }.should change(Automatic::Plugin::Blog, :count).by(0)
+    instance2.run.should have(0).feed
+    Automatic::Plugin::Blog.count.should eq 1
   end
 
+  it "should not store record for the existent blog title" do
+    instance = Automatic::Plugin::StoreFullText.new({"db" => @db_filename},
+      AutomaticSpec.generate_pipeline {
+        feed {
+          item "http://blog.id774.net/post/100",
+          "dummy title 1",
+          "aaa bbb ccc http://test2.id774.net ddd eee",
+          "Mon, 07 Mar 2011 15:54:11 +0900"
+        }
+      }
+    )
+    instance2 = Automatic::Plugin::StoreFullText.new({"db" => @db_filename},
+      AutomaticSpec.generate_pipeline {
+        feed {
+          item "http://blog.id774.net/post/200",
+          "dummy title 1",
+          "aaa bbb ccc http://test2.id774.net ddd eee",
+          "Mon, 07 Mar 2011 15:54:11 +0900"
+        }
+      }
+    )
+
+    Automatic::Plugin::Blog.count.should eq 0
+    instance.run.should have(1).feed
+    lambda {
+      instance2.run.should have(0).feed
+    }.should change(Automatic::Plugin::Blog, :count).by(0)
+    instance2.run.should have(0).feed
+    Automatic::Plugin::Blog.count.should eq 1
+  end
+
+  it "should store 2 records for the independent entries" do
+    instance = Automatic::Plugin::StoreFullText.new({"db" => @db_filename},
+      AutomaticSpec.generate_pipeline {
+        feed {
+          item "http://blog.id774.net/post/100",
+          "dummy title 1",
+          "aaa bbb ccc http://test2.id774.net ddd eee",
+          "Mon, 07 Mar 2011 15:54:11 +0900"
+        }
+      }
+    )
+    instance2 = Automatic::Plugin::StoreFullText.new({"db" => @db_filename},
+      AutomaticSpec.generate_pipeline {
+        feed {
+          item "http://blog.id774.net/post/200",
+          "dummy title 2",
+          "aaa bbb ccc http://test2.id774.net ddd eee",
+          "Mon, 07 Mar 2011 15:54:11 +0900"
+        }
+      }
+    )
+
+    Automatic::Plugin::Blog.count.should eq 0
+    instance.run.should have(1).feed
+    lambda {
+      instance2.run.should have(1).feed
+    }.should change(Automatic::Plugin::Blog, :count).by(1)
+    Automatic::Plugin::Blog.count.should eq 2
+  end
 end

data/test/integration/test_descriptionlink.yml

@@ -0,0 +1,21 @@
+global:
+  timezone: Asia/Tokyo
+  cache:
+    base: /tmp
+  log:
+    level: info
+
+plugins:
+  - module: SubscriptionFeed
+    config:
+      feeds:
+        - http://reblog.id774.net/rss
+
+  - module: FilterDescriptionLink
+
+  - module: StorePermalink
+    config:
+      db: test_description_link.db
+
+  #- module: PublishConsole
+

data/test/integration/test_google_news.yml

@@ -0,0 +1,21 @@
+global:
+  timezone: Asia/Tokyo
+  cache:
+    base: /tmp
+  log:
+    level: info
+
+plugins:
+  - module: SubscriptionFeed
+    config:
+      feeds:
+        - http://news.google.com/news?hl=ja&ned=us&ie=UTF-8&oe=UTF-8&output=rss
+
+  - module: FilterGoogleNews
+
+  - module: StorePermalink
+    config:
+      db: test_google_news.db
+
+  - module: PublishConsole
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: automatic
 version: !ruby/object:Gem::Version
-  version: 14.5.0
+  version: 14.10.1
 platform: ruby
 authors:
 - id774
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-31 00:00:00.000000000 Z
+date: 2014-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -354,8 +354,11 @@ files:
 - plugins/custom_feed/svn_log.rb
 - plugins/filter/absolute_uri.rb
 - plugins/filter/accept.rb
+- plugins/filter/clear.rb
+- plugins/filter/description_link.rb
 - plugins/filter/full_feed.rb
 - plugins/filter/github_feed.rb
+- plugins/filter/google_news.rb
 - plugins/filter/ignore.rb
 - plugins/filter/image.rb
 - plugins/filter/image_source.rb
@@ -404,8 +407,11 @@ files:
 - spec/plugins/custom_feed/svn_log_spec.rb
 - spec/plugins/filter/absolute_uri_spec.rb
 - spec/plugins/filter/accept_spec.rb
+- spec/plugins/filter/clear_spec.rb
+- spec/plugins/filter/description_link_spec.rb
 - spec/plugins/filter/full_feed_spec.rb
 - spec/plugins/filter/github_feed_spec.rb
+- spec/plugins/filter/google_news_spec.rb
 - spec/plugins/filter/ignore_spec.rb
 - spec/plugins/filter/image_source_spec.rb
 - spec/plugins/filter/image_spec.rb
@@ -448,8 +454,10 @@ files:
 - test/integration/test_activerecord.yml
 - test/integration/test_add_pocket.yml
 - test/integration/test_chan_toru.yml
+- test/integration/test_descriptionlink.yml
 - test/integration/test_fluentd.yml
 - test/integration/test_fulltext.yml
+- test/integration/test_google_news.yml
 - test/integration/test_googlealert.yml
 - test/integration/test_hatenabookmark.yml
 - test/integration/test_ignore.yml