automatic 14.1.0 → 14.2.0

data/plugins/publish/fluentd.rb

@@ -2,8 +2,8 @@
 # Name::      Automatic::Plugin::Publish::Fluentd
 # Author::    774 <http://id774.net>
 # Created::   Jun 21, 2013
-# Updated::   Jun 25, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 25, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
@@ -13,9 +13,10 @@ module Automatic::Plugin
     def initialize(config, pipeline=[])
       @config = config
       @pipeline = pipeline
+      @mode = @config['mode']
       @fluentd = Fluent::Logger::FluentLogger.open(nil,
         host = @config['host'],
-        port = @config['port'])
+        port = @config['port']) unless @mode == 'test'
     end
 
     def run
@@ -29,7 +30,7 @@ module Automatic::Plugin
                 :description => feed.description,
                 :content => feed.content_encoded,
                 :created_at => Time.now.strftime("%Y/%m/%d %X")
-              })
+              }) unless @mode == 'test'
             rescue
               Automatic::Log.puts("warn", "Skip feed due to fault in forward.")
             end

data/plugins/publish/google_calendar.rb

@@ -37,8 +37,8 @@ module Automatic::Plugin
         end
       end
 
-      Automatic::Log.puts("info", "日付     ： #{date}")
-      Automatic::Log.puts("info", "タイトル ： #{text}")
+      Automatic::Log.puts("info", "Date  ： #{date}")
+      Automatic::Log.puts("info", "Title ： #{text}")
 
       # Register to calendar
       require 'rubygems'

data/plugins/publish/hipchat.rb

@@ -29,7 +29,7 @@ module Automatic::Plugin
             retry_max = @config['retry'].to_i || 0
             begin
               @client.send(@config['username'], feed.description, @options)
-              Automatic::Log.puts("info", "post: #{feed.description.gsub(/[\r\n]/,'')[0..50]}...") rescue nil
+              Automatic::Log.puts("info", "Hipchat post: #{feed.description.gsub(/[\r\n]/,'')[0..50]}...") rescue nil
             rescue => e
               retries += 1
               Automatic::Log.puts("error", "ErrorCount: #{retries}, #{e.message}")

data/plugins/publish/twitter.rb

@@ -34,7 +34,7 @@ module Automatic::Plugin
       @pipeline.each {|feeds|
         unless feeds.nil?
           feeds.items.each {|feed|
-            Automatic::Log.puts("info", "tweet: #{feed.link}")
+            Automatic::Log.puts("info", "Publish Tweet: #{feed.link}")
             retries = 0
             retry_max = @config['retry'].to_i || 0
             begin

data/plugins/store/database.rb

@@ -4,8 +4,8 @@
 #             774 <http://id774.net>
 #             soramugi <http://soramugi.net>
 # Created::   Feb 27, 2012
-# Updated::   Jun 27, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 21, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 require 'active_record'
@@ -27,7 +27,7 @@ module Automatic::Plugin
               end
             end
           }
-          @return_feeds << Automatic::FeedParser.create(new_feeds) if new_feeds.length > 0
+          @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0
         end
       }
       @return_feeds
@@ -54,7 +54,7 @@ module Automatic::Plugin
 
     def prepare_database
       db = File.join(db_dir, @config['db'])
-      Automatic::Log.puts("info", "Database: #{db}")
+      Automatic::Log.puts("info", "Using Database: #{db}")
       ActiveRecord::Base.establish_connection(
         :adapter  => "sqlite3",
         :database => db)

data/plugins/store/file.rb

@@ -0,0 +1,95 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+# Name::      Automatic::Plugin::Store::File
+# Author::    774 <http://id774.net>
+# Created::   Feb 28, 2012
+# Updated::   Feb 25, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+require 'open-uri'
+require 'uri'
+require 'aws-sdk'
+
+module Automatic::Plugin
+  class StoreFile
+
+    def initialize(config, pipeline=[])
+      @config = config
+      @pipeline = pipeline
+      unless @config['bucket_name'].nil?
+        s3 = AWS::S3.new(
+          :access_key_id => @config['access_key'],
+          :secret_access_key => @config['secret_key']
+        )
+        @bucket = s3.buckets[@config['bucket_name']]
+      end
+      @return_feeds = []
+    end
+
+    def run
+      @pipeline.each {|feeds|
+        unless feeds.nil?
+          feeds.items.each {|feed|
+            unless feed.link.nil?
+              Automatic::Log.puts("info", "Downloading File: #{feed.link}")
+              FileUtils.mkdir_p(@config['path']) unless FileTest.exist?(@config['path'])
+              retries = 0
+              retry_max = @config['retry'].to_i || 0
+              begin
+                retries += 1
+                feed.link = get_file(feed.link)
+                sleep ||= @config['interval'].to_i
+                @return_feeds << feed
+              rescue
+                Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault during file download.")
+                sleep ||= @config['interval'].to_i
+                retry if retries <= retry_max
+              end
+            end
+          }
+        end
+      }
+      @pipeline = []
+      @pipeline << Automatic::FeedMaker.create_pipeline(@return_feeds) if @return_feeds.length > 0
+      @pipeline
+    end
+
+    private
+
+    def get_file(url)
+      uri = URI.parse(url)
+      case uri.scheme
+      when "s3n"
+        return_path = get_aws(uri)
+      else
+        return_path = wget(uri, url)
+      end
+      Automatic::Log.puts("info", "Saved File: #{return_path}")
+      "file://" + return_path
+    end
+
+    def wget(uri, url)
+      filename = File.basename(uri.path)
+      filepath = File.join(@config['path'], filename)
+      open(url) {|source|
+        open(filepath, "w+b") { |o|
+          o.print(source.read)
+        }
+      }
+      filepath
+    end
+
+    def get_aws(uri)
+      filename = File.basename(uri.path)
+      filepath = File.join(@config['path'], filename)
+      object = @bucket.objects[uri.path]
+      File.open(filepath, 'wb') do |file|
+        object.read do |chunk|
+           file.write(chunk)
+        end
+      end
+      filepath
+    end
+  end
+end

data/plugins/store/full_text.rb

@@ -40,7 +40,7 @@ module Automatic::Plugin
 
     def run
       for_each_new_feed {|feed|
-        Automatic::Log.puts("info", "Saving: #{feed.link}")
+        Automatic::Log.puts("info", "Saving FullText: #{feed.link}")
         begin
           Blog.create(
             :title => feed.title,

data/plugins/store/permalink.rb

@@ -42,7 +42,7 @@ module Automatic::Plugin
           Permalink.create(
             :url => feed.link,
             :created_at => Time.now.strftime("%Y/%m/%d %X"))
-          Automatic::Log.puts("info", "Saving: #{feed.link}")
+          Automatic::Log.puts("info", "Saving Permalink: #{feed.link}")
         end
       }
     end

data/plugins/subscription/feed.rb

@@ -2,7 +2,7 @@
 # Name::      Automatic::Plugin::SubscriptionFeed
 # Author::    774 <http://id774.net>
 # Created::   Feb 22, 2012
-# Updated::   Jan 15, 2014
+# Updated::   Feb 21, 2014
 # Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -18,7 +18,7 @@ module Automatic::Plugin
         retries = 0
         retry_max = @config['retry'].to_i || 0
         begin
-          rss = Automatic::FeedParser.get(feed)
+          rss = Automatic::FeedParser.get_url(feed)
           @pipeline << rss
         rescue
           retries += 1

data/plugins/subscription/g_guide.rb

@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
 # Name::      Automatic::Plugin::Subscription::GGuide
 # Author::    soramugi <http://soramugi.net>
+#             774 <http://id774.net>
 # Created::   Jun 28, 2013
-# Updated::   Jan 15, 2014
+# Updated::   Feb 21, 2014
 # Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -25,7 +26,7 @@ module Automatic::Plugin
       retry_max = @config['retry'].to_i || 0
       begin
         @keywords.each {|keyword|
-          @pipeline << Automatic::FeedParser.get(feed_url keyword)
+          @pipeline << Automatic::FeedParser.get_url(feed_url keyword)
         }
       rescue
         retries += 1

data/plugins/subscription/link.rb

@@ -2,7 +2,7 @@
 # Name::      Automatic::Plugin::Subscription::Link
 # Author::    774 <http://id774.net>
 # Created::   Sep 18, 2012
-# Updated::   Jan 15, 2014
+# Updated::   Feb 21, 2014
 # Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -34,11 +34,12 @@ module Automatic::Plugin
     end
 
     private
+
     def create_rss(url)
-      Automatic::Log.puts("info", "Parsing: #{url}")
+      Automatic::Log.puts("info", "Parsing Link: #{url}")
       html = open(url).read
       unless html.nil?
-        rss = Automatic::FeedParser.parse(html)
+        rss = Automatic::FeedParser.parse_html(html)
         sleep ||= @config['interval'].to_i
         @return_feeds << rss
       end

data/plugins/subscription/pocket.rb

@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
 # Name::      Automatic::Plugin::Subscription::Pocket
 # Author::    soramugi <http://soramugi.net>
+#             774 <http://id774.net>
 # Created::   May 21, 2013
-# Updated::   Jan 15, 2014
+# Updated::   Feb 21, 2014
 # Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -24,8 +25,8 @@ module Automatic::Plugin
       retries = 0
       retry_max = @config['retry'].to_i || 0
       begin
-        dummyfeeds = cleate_dummy_feed(@client.retrieve(@config['optional']))
-        @pipeline << Automatic::FeedParser.create(dummyfeeds)
+        return_feeds = generate_feed(@client.retrieve(@config['optional']))
+        @pipeline << Automatic::FeedMaker.create_pipeline(return_feeds)
       rescue
         retries += 1
         Automatic::Log.puts("error", "ErrorCount: #{retries}, Fault in parsing: #{retries}")
@@ -35,16 +36,16 @@ module Automatic::Plugin
       @pipeline
     end
 
-    def cleate_dummy_feed(retrieve)
-      dummyFeeds = []
+    def generate_feed(retrieve)
+      return_feeds = []
       retrieve['list'].each {|key,list|
-        dummy             = Hashie::Mash.new
-        dummy.title       = list['given_title']
-        dummy.link        = list['given_url']
-        dummy.description = list['excerpt']
-        dummyFeeds << dummy
+        hashie             = Hashie::Mash.new
+        hashie.title       = list['given_title']
+        hashie.link        = list['given_url']
+        hashie.description = list['excerpt']
+        return_feeds << hashie
       }
-      dummyFeeds
+      return_feeds
     end
   end
 end

data/plugins/subscription/text.rb

@@ -1,76 +1,67 @@
 # -*- coding: utf-8 -*-
 # Name::      Automatic::Plugin::Subscription::Text
 # Author::    soramugi <http://soramugi.net>
-# Created::   May 6, 2013
-# Updated::   May 6, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+#             774 <http://id774.net>
+# Created::   May  6, 2013
+# Updated::   Feb 21, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
-  class TextFeed
-    def initialize
-      @link     = 'http://dummy'
-      @title    = 'dummy'
-    end
-
-    def link
-      @link
-    end
-
-    def title
-      @title
-    end
-
-    def set_link(link)
-      @link = link
-    end
-
-    def set_title(title)
-      @title = title
-    end
-  end
-
   class SubscriptionText
-
     def initialize(config, pipeline=[])
       @config   = config
       @pipeline = pipeline
+      @return_feeds = []
+    end
 
+    def run
+      create_feed
+      @pipeline << Automatic::FeedMaker.create_pipeline(@return_feeds) if @return_feeds.length > 0
+      @pipeline
+    end
+
+    private
+
+    def create_feed
       unless @config.nil?
         @dummyfeeds = []
         unless @config['titles'].nil?
           @config['titles'].each {|title|
-            textFeed = TextFeed.new
-            textFeed.set_title(title)
-            @dummyfeeds << textFeed
+            feed = {}
+            feed['title'] = title
+            @return_feeds << Automatic::FeedMaker.generate_feed(feed)
           }
         end
 
         unless @config['urls'].nil?
           @config['urls'].each {|url|
-            textFeed = TextFeed.new
-            textFeed.set_link(url)
-            @dummyfeeds << textFeed
+            feed = {}
+            feed['url'] = url
+            @return_feeds << Automatic::FeedMaker.generate_feed(feed)
           }
         end
 
         unless @config['feeds'].nil?
           @config['feeds'].each {|feed|
-            textFeed = TextFeed.new
-            textFeed.set_title(feed['title']) unless feed['title'].nil?
-            textFeed.set_link(feed['url']) unless feed['url'].nil?
-            @dummyfeeds << textFeed
+            @return_feeds << Automatic::FeedMaker.generate_feed(feed)
           }
         end
-      end
 
-    end
-
-    def run
-      if @dummyfeeds != []
-        @pipeline << Automatic::FeedParser.create(@dummyfeeds)
+        unless @config['files'].nil?
+          @config['files'].each {|f|
+            open(File.expand_path(f)) do |file|
+              file.each_line do |line|
+                feed = {}
+                feed['title'], feed['url'], feed['description'], feed['author'],
+                feed['comments'] = line.force_encoding("utf-8").strip.split("\t")
+                @return_feeds << Automatic::FeedMaker.generate_feed(feed)
+              end
+            end
+          }
+        end
       end
-      @pipeline
     end
+
   end
 end

data/plugins/subscription/tumblr.rb

@@ -2,7 +2,7 @@
 # Name::      Automatic::Plugin::Subscription::Tumblr
 # Author::    774 <http://id774.net>
 # Created::   Oct 16, 2012
-# Updated::   Jan 15, 2014
+# Updated::   Feb 21, 2014
 # Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -42,11 +42,11 @@ module Automatic::Plugin
 
     private
     def create_rss(url)
-      Automatic::Log.puts("info", "Parsing: #{url}")
+      Automatic::Log.puts("info", "Parsing Tumblr: #{url}")
       html = open(url).read
       unless html.nil?
         uri = URI.parse(url)
-        rss = Automatic::FeedParser.parse(html)
+        rss = Automatic::FeedParser.parse_html(html)
         rss.items.each {|item|
            unless item.link =~ Regexp.new(uri.host)
              item.link = nil