automatic 14.1.0 → 14.2.0

data/lib/automatic.rb

@@ -2,7 +2,7 @@
 # Name::      Automatic::Ruby
 # Author::    774 <http://id774.net>
 # Created::   Feb 18, 2012
-# Updated::   Jan 24, 2014
+# Updated::   Feb 21, 2014
 # Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -12,6 +12,7 @@ module Automatic
   require 'automatic/pipeline'
   require 'automatic/log'
   require 'automatic/feed_parser'
+  require 'automatic/feed_maker'
   require 'automatic/version'
 
   USER_DIR = "/.automatic"

data/lib/automatic/feed_maker.rb

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+# Name::      Automatic::FeedMaker
+# Author::    774 <http://id774.net>
+# Created::   Feb 21, 2014
+# Updated::   Feb 26, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+module Automatic
+  module FeedMaker
+    require 'rss'
+    require 'uri'
+    require 'nokogiri'
+
+    class FeedObject
+      attr_accessor :title, :link, :description, :author, :comments
+      def initialize
+        @link        = 'http://dummy'
+        @title       = 'dummy'
+        @description = ''
+        @author      = ''
+        @comments    = ''
+      end
+    end
+
+    def self.generate_feed(feed)
+      feed_object = FeedObject.new
+      feed_object.title = feed['title'] unless feed['title'].nil?
+      feed_object.link = feed['url'] unless feed['url'].nil?
+      feed_object.description = feed['description'] unless feed['description'].nil?
+      feed_object.author = feed['author'] unless feed['author'].nil?
+      feed_object.comments = feed['comments'] unless feed['comments'].nil?
+      feed_object
+    end
+
+    def self.create_pipeline(feeds = [])
+      RSS::Maker.make("2.0") {|maker|
+        xss = maker.xml_stylesheets.new_xml_stylesheet
+        maker.channel.title = "Automatic Ruby"
+        maker.channel.description = "Automatic::FeedMaker"
+        maker.channel.link = "https://github.com/automaticruby/automaticruby"
+        maker.items.do_sort = true
+
+        unless feeds.nil?
+          feeds.each {|feed|
+            unless feed.link.nil?
+              Automatic::Log.puts("info", "Create Pipeline: #{feed.link}")
+              item = maker.items.new_item
+              item.title = feed.title
+              item.link = feed.link
+              begin
+                item.description = feed.description
+                item.author = feed.author
+                item.comments = feed.comments
+                item.date = feed.pubDate || Time.now
+              rescue NoMethodError
+                Automatic::Log.puts("warn", "Undefined field detected in feed.")
+              end
+            end
+          }
+        end
+      }
+    end
+
+    def self.content_provide(url, data)
+      RSS::Maker.make("2.0") {|maker|
+        xss = maker.xml_stylesheets.new_xml_stylesheet
+        maker.channel.title = "Automatic Ruby"
+        maker.channel.description = "Automatic::FeedMaker"
+        maker.channel.link = "https://github.com/automaticruby/automaticruby"
+        maker.items.do_sort = true
+        item = maker.items.new_item
+        item.title = "Automatic Ruby"
+        item.link = url
+        item.content_encoded = data
+        item.date = Time.now
+      }
+    end
+  end
+end

data/lib/automatic/feed_parser.rb

@@ -2,8 +2,8 @@
 # Name::      Automatic::FeedParser
 # Author::    774 <http://id774.net>
 # Created::   Feb 19, 2012
-# Updated::   Jul 12, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 26, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic
@@ -12,10 +12,10 @@ module Automatic
     require 'uri'
     require 'nokogiri'
 
-    def self.get(url)
+    def self.get_url(url)
       begin
         unless url.nil?
-          Automatic::Log.puts("info", "Parsing: #{url}")
+          Automatic::Log.puts("info", "Parsing Feed: #{url}")
           feed = URI.parse(url).normalize
           open(feed) {|http|
             response = http.read
@@ -27,40 +27,11 @@ module Automatic
       end
     end
 
-    def self.create(feeds = [])
+    def self.parse_html(html)
       RSS::Maker.make("2.0") {|maker|
         xss = maker.xml_stylesheets.new_xml_stylesheet
         maker.channel.title = "Automatic Ruby"
-        maker.channel.description = "Automatic Ruby"
-        maker.channel.link = "https://github.com/automaticruby/automaticruby"
-        maker.items.do_sort = true
-
-        unless feeds.nil?
-          feeds.each {|feed|
-            unless feed.link.nil?
-              Automatic::Log.puts("info", "Feed: #{feed.link}")
-              item = maker.items.new_item
-              item.title = feed.title
-              item.link = feed.link
-              begin
-                item.description = feed.description
-                item.author = feed.author
-                item.comments = feed.comments
-                item.date = feed.pubDate || Time.now
-              rescue NoMethodError
-                Automatic::Log.puts("warn", "Undefined field detected in feed.")
-              end
-            end
-          }
-        end
-      }
-    end
-
-    def self.parse(html)
-      RSS::Maker.make("2.0") {|maker|
-        xss = maker.xml_stylesheets.new_xml_stylesheet
-        maker.channel.title = "Automatic Ruby"
-        maker.channel.description = "Automatic Ruby"
+        maker.channel.description = "Automatic::FeedParser"
         maker.channel.link = "https://github.com/automaticruby/automaticruby"
         maker.items.do_sort = true
 
@@ -76,20 +47,5 @@ module Automatic
         }
       }
     end
-
-    def self.content_provide(url, data)
-      RSS::Maker.make("2.0") {|maker|
-        xss = maker.xml_stylesheets.new_xml_stylesheet
-        maker.channel.title = "Automatic Ruby"
-        maker.channel.description = "Automatic Ruby"
-        maker.channel.link = "https://github.com/automaticruby/automaticruby"
-        maker.items.do_sort = true
-        item = maker.items.new_item
-        item.title = "Automatic Ruby"
-        item.link = url
-        item.content_encoded = data
-        item.date = Time.now
-      }
-    end
   end
 end

data/lib/automatic/recipe.rb

@@ -24,7 +24,7 @@ module Automatic
       @procedure = Hashie::Mash.new(YAML.load(File.read(path)))
       log_level = @procedure.global && @procedure.global.log && @procedure.global.log.level
       Automatic::Log.level(log_level)
-      Automatic::Log.puts("info", "Loading: #{path}")
+      Automatic::Log.puts("info", "Loading Recipe: #{path}")
       @procedure
     end
 

data/lib/automatic/version.rb

@@ -1,3 +1,3 @@
 module Automatic
-  VERSION = "14.1.0"
+  VERSION = "14.2.0"
 end

data/plugins/filter/accept.rb

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 # Name::      Automatic::Plugin::Filter::Accept
 # Author::    soramugi <http://soramugi.net>
+#             774 <http://id774.net>
 # Created::   Jun  4, 2013
-# Updated::   Jun  4, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 21, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
@@ -22,7 +23,7 @@ module Automatic::Plugin
             new_feeds << items if contain(items) == true
           }
         end
-        @return_feeds << Automatic::FeedParser.create(new_feeds) if new_feeds.length > 0
+        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0
       }
       @return_feeds
     end

data/plugins/filter/github_feed.rb

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 # Name::      Automatic::Plugin::Filter::GithubFeed
 # Author::    Kohei Hasegawa <http://github.com/banyan>
-# Created::   Jun 6, 2013
-# Updated::   Jun 6, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+#             774 <http://id774.net>
+# Created::   Jun  6, 2013
+# Updated::   Feb 21, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
@@ -17,20 +18,20 @@ module Automatic::Plugin
     def run
       @return_feeds = []
       @pipeline.each {|feeds|
-        dummyFeeds = []
+        new_feeds = []
         unless feeds.nil?
           feeds.items.each {|feed|
             Automatic::Log.puts("info", "Invoked: FilterGithubFeed")
-            dummy = Hashie::Mash.new
-            dummy.title       = feed.title.content
-            dummy.link        = feed.id.content
-            dummy.description = feed.content.content
-            dummyFeeds << dummy
+            hashie = Hashie::Mash.new
+            hashie.title       = feed.title.content
+            hashie.link        = feed.id.content
+            hashie.description = feed.content.content
+            new_feeds << hashie
           }
         end
-        @return_feeds << Automatic::FeedParser.create(dummyFeeds)
+        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds)
       }
-      @pipeline = @return_feeds
+      @return_feeds
     end
   end
 end

data/plugins/filter/ignore.rb

@@ -2,8 +2,8 @@
 # Name::      Automatic::Plugin::Filter::Ignore
 # Author::    774 <http://id774.net>
 # Created::   Feb 22, 2012
-# Updated::   Oct 15, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 21, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
@@ -22,7 +22,7 @@ module Automatic::Plugin
             new_feeds << items if exclude(items) == false
           }
         end
-        @return_feeds << Automatic::FeedParser.create(new_feeds) if new_feeds.length > 0
+        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds) if new_feeds.length > 0
       }
       @return_feeds
     end

data/plugins/filter/image_source.rb

@@ -2,8 +2,8 @@
 # Name::      Automatic::Plugin::Filter::ImageSource
 # Author::    774 <http://id774.net>
 # Created::   Feb 28, 2012
-# Updated::   Apr  5, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 26, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
@@ -19,24 +19,24 @@ module Automatic::Plugin
     def run
       @return_feeds = []
       @pipeline.each {|feeds|
-        dummyFeeds = Array.new
+        new_feeds = Array.new
         unless feeds.nil?
           feeds.items.each {|feed|
             arr = rewrite_link(feed)
             if arr.length > 0
               arr.each {|link|
-                Automatic::Log.puts("info", "Parsing: #{link}")
-                dummy = Hashie::Mash.new
-                dummy.title = 'FilterImageSource'
-                dummy.link = link
-                dummyFeeds << dummy
+                Automatic::Log.puts("info", "Extract Image: #{link}")
+                hashie = Hashie::Mash.new
+                hashie.title = 'FilterImageSource'
+                hashie.link = link
+                new_feeds << hashie
               }
             end
           }
         end
-        @return_feeds << Automatic::FeedParser.create(dummyFeeds)
+        @return_feeds << Automatic::FeedMaker.create_pipeline(new_feeds)
       }
-      @pipeline = @return_feeds
+      @return_feeds
     end
 
     private

data/plugins/filter/one.rb

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 # Name::      Automatic::Plugin::Filter::One
 # Author::    soramugi <http://soramugi.net>
+#             774 <http://id774.net>
 # Created::   May  8, 2013
-# Updated::   May  8, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 21, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
@@ -27,7 +28,7 @@ module Automatic::Plugin
           if item.count == 0
             item << feed.items.shift
           end
-          @return_feeds << Automatic::FeedParser.create(item)
+          @return_feeds << Automatic::FeedMaker.create_pipeline(item)
         end
       }
       @return_feeds

data/plugins/filter/rand.rb

@@ -3,8 +3,8 @@
 # Author::    soramugi <http://soramugi.net>
 #             774 <http://id774.net>
 # Created::   Mar  6, 2013
-# Updated::   Mar  7, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 21, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
@@ -19,7 +19,7 @@ module Automatic::Plugin
       @return_feeds = []
       @pipeline.each {|feed|
         unless feed.nil?
-          @return_feeds << Automatic::FeedParser.create(feed.items.shuffle)
+          @return_feeds << Automatic::FeedMaker.create_pipeline(feed.items.shuffle)
         end
       }
       @return_feeds

data/plugins/filter/sanitize.rb

@@ -2,8 +2,8 @@
 # Name::      Automatic::Plugin::Filter::Sanitize
 # Author::    774 <http://id774.net>
 # Created::   Jun 20, 2013
-# Updated::   Jun 24, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 21, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
@@ -26,7 +26,6 @@ module Automatic::Plugin
     def run
       @return_feeds = []
       @pipeline.each {|feeds|
-        return_feed_items = []
         unless feeds.nil?
           feeds.items.each {|feed|
             feed = sanitize(feed)

data/plugins/provide/fluentd.rb

@@ -2,8 +2,8 @@
 # Name::      Automatic::Plugin::Provide::Fluentd
 # Author::    774 <http://id774.net>
 # Created::   Jul 12, 2013
-# Updated::   Jul 12, 2013
-# Copyright:: Copyright (c) 2012-2013 Automatic Ruby Developers.
+# Updated::   Feb 25, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 module Automatic::Plugin
@@ -14,9 +14,10 @@ module Automatic::Plugin
     def initialize(config, pipeline=[])
       @config = config
       @pipeline = pipeline
+      @mode = @config['mode']
       @fluentd = Fluent::Logger::FluentLogger.open(nil,
         host = @config['host'],
-        port = @config['port'])
+        port = @config['port']) unless @mode == 'test'
     end
 
     def run
@@ -24,7 +25,7 @@ module Automatic::Plugin
         unless feeds.nil?
           feeds.items.each {|feed|
             begin
-              @fluentd.post(@config['tag'], feed.content_encoded)
+              @fluentd.post(@config['tag'], feed.content_encoded) unless @mode == 'test'
             rescue
               Automatic::Log.puts("error", "Fluent::Logger.post failed, the content_encoded of item may be not kind of Hash.")
             end

data/plugins/publish/amazon_s3.rb

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# Name::      Automatic::Plugin::Publish::AmazonS3
+# Author::    774 <http://id774.net>
+# Created::   Feb 24, 2014
+# Updated::   Feb 24, 2014
+# Copyright:: Copyright (c) 2012-2014 Automatic Ruby Developers.
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+module Automatic::Plugin
+  class PublishAmazonS3
+    require 'uri'
+    require 'aws-sdk'
+
+    def initialize(config, pipeline=[])
+      @config = config
+      @pipeline = pipeline
+      s3 = AWS::S3.new(
+        :access_key_id => @config['access_key'],
+        :secret_access_key => @config['secret_key']
+      )
+      @bucket = s3.buckets[@config['bucket_name']]
+      @mode = @config['mode']
+    end
+
+    def run
+      @pipeline.each {|feeds|
+        unless feeds.nil?
+          feeds.items.each {|feed|
+            unless feed.link.nil?
+              begin
+                uri = URI.parse(feed.link)
+                if uri.scheme == 'file'
+                  upload_amazons3(uri.path, @config['target_path'])
+                else
+                  Automatic::Log.puts("warn", "Skip feed due to uri scheme is not file.")
+                end
+              rescue
+                Automatic::Log.puts("error", "Error detected with #{feed.link} in uploading AmazonS3.")
+              end
+            end
+          }
+        end
+      }
+      @pipeline
+    end
+
+    private
+
+    def upload_amazons3(filename, target_path)
+      target = File.join(target_path, File.basename(filename))
+      object = @bucket.objects[target]
+      source = Pathname.new(filename)
+      object.write(source) unless @mode == "test"
+      Automatic::Log.puts("info", "Uploaded: file #{source} to the bucket #{target} on #{@bucket.name}.")
+      return source, target
+    end
+  end
+end