yapra 0.1.0 → 0.1.1

data/lib-plugins/yapra/plugin/filter/apply_template.rb

@@ -0,0 +1,52 @@
+require 'yapra/plugin/base'
+
+module Yapra::Plugin::Filter
+  # = Filter::ApplyTemplate -- Yuanying
+  # 
+  # apply template and set to attribute.
+  #
+  # example:
+  # 
+  #     - module: Filter::ApplyTemplate
+  #       config:
+  #         content_encoded: '<div><%= title %></div>'
+  #
+  class ApplyTemplate < Yapra::Plugin::Base
+    def run(data)
+      regexp = nil
+      if config['regexp']
+        regexp = Regexp.new(config['regexp'])
+      else
+        regexp = /^(https?|ftp)(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)$/
+      end
+      
+      data.map! do |item|
+        url = item
+        if item.respond_to?('link')
+          url = item.link
+        end
+
+        if regexp =~ url
+          unless(item.instance_of?(RSS::RDF::Item))
+            new_item = RSS::RDF::Item.new
+            new_item.title = item.title rescue item.to_s
+            new_item.date = item.date rescue Time.now
+            new_item.description = item.description rescue item.to_s
+            new_item.link = item.link rescue '#'
+            item = new_item
+          end
+          
+          if plugin_config
+            plugin_config.each do |k, template|
+              value = apply_template template, binding
+              set_attribute_to item, k, value
+            end
+          end
+        end
+        item
+      end
+      
+      data
+    end
+  end
+end

data/lib-plugins/yapra/plugin/filter/entry_full_text.rb

@@ -1,21 +1,21 @@
-## Filter::EntryFullText -- Yuanying
-##
-## get the entry full text from page with WWW::Mechanize.
-## 
-## - module: Filter::EntryFullText
-##   config:
-##     regexp: http://www\.pixiv\.net/*
-##     extract_xpath:
-##       title: '//title/text()'
-##       dc_creator: "//div[@id='profile']/div/text()"
-##       author: "//div[@id='profile']/div/text()"
-##       description: "//div[@id='content2']"
-##     apply_template_after_extracted:
-##       content_encoded: '<div><%= title %></div>'
-##
 require 'yapra/plugin/mechanize_base'
 
 module Yapra::Plugin::Filter
+  # Filter::EntryFullText -- Yuanying
+  # 
+  # get the entry full text from page with WWW::Mechanize.
+  # 
+  #     - module: Filter::EntryFullText
+  #       config:
+  #         regexp: http://www\.pixiv\.net/*
+  #         extract_xpath:
+  #           title: '//title/text()'
+  #           dc_creator: "//div[@id='profile']/div/text()"
+  #           author: "//div[@id='profile']/div/text()"
+  #           description: "//div[@id='content2']"
+  #         apply_template_after_extracted:
+  #           content_encoded: '<div><%= title %></div>'
+  
   class EntryFullText < Yapra::Plugin::MechanizeBase
     def run(data)
       regexp = nil
@@ -34,9 +34,9 @@ module Yapra::Plugin::Filter
         end
 
         if regexp =~ url
+          logger.debug "Process: #{url}"
           page = agent.get(url)
           sleep wait
-          logger.info "Process: #{url}"
 
           unless(item.instance_of?(RSS::RDF::Item))
             new_item = RSS::RDF::Item.new

data/lib-plugins/yapra/plugin/publish/file_download.rb

@@ -1,72 +1,34 @@
-## Publish::FileDownload -- Yuanying
-##
-## download file with WWW::Mechanize.
-## 
-## example: 
-## 
-## - module: Publish::FileDownload
-##   config:
-##     regexp: http://www\.yahoo\.co\.jp/*
-##     dir: '/Users/yuanying/Downloads/'
-##     auto_suffix: true
-##     before_hook: "agent.get('http://www.yahoo.co.jp/'); sleep 1"
-##     url:
-##       attribute: link
-##       replace: index(\d*?)\.html
-##       to: file\1.zip
-##     filename:
-##       attribute: title
-##       replace: 'Yahoo'
-##       to: ''
-##     referrer:
-##       attribute: link
-##       replace: 'zip'
-##       to: 'html'
-##
-require 'yapra/plugin/mechanize_base'
+require 'yapra/plugin/publish/on_memory_download'
 
 module Yapra::Plugin::Publish
-  class FileDownload < Yapra::Plugin::MechanizeBase
-    def run(data)
-      regexp = nil
-      if config['regexp']
-        regexp = Regexp.new(config['regexp'])
-      else
-        regexp = /^(https?|ftp)(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)$/
-      end
-      
-      wait = config['wait'] || 3
-
-      data.each do |item|
-        url = construct_data(config['url'], item, item.respond_to?('link') ? item.link : item)
-
-        if regexp =~ url
-          referrer = construct_data(config['referrer'], item)
-          download(item, url, referrer)
-          sleep wait
-        end
-      end
-      
-      data
-    end
+  # = Publish::FileDownload -- Yuanying
+  # 
+  # download file with WWW::Mechanize.
+  # 
+  # example: 
+  # 
+  #     - module: Publish::FileDownload
+  #       config:
+  #         regexp: http://www\.yahoo\.co\.jp/*
+  #         dir: '/Users/yuanying/Downloads/'
+  #         auto_suffix: true
+  #         before_hook: "agent.get('http://www.yahoo.co.jp/'); sleep 1"
+  #         url:
+  #           attribute: link
+  #           replace: index(\d*?)\.html
+  #           to: file\1.zip
+  #         filename:
+  #           attribute: title
+  #           replace: 'Yahoo'
+  #           to: ''
+  #         referrer:
+  #           attribute: link
+  #           replace: 'zip'
+  #           to: 'html'
+  #
+  class FileDownload < Yapra::Plugin::Publish::OnMemoryDownload
     
     protected
-    def construct_data(config, item, original=nil)
-      if config && config['attribute']
-        if item.respond_to?(config['attribute'])
-          original = item.__send__(config['attribute'])
-        end
-      elsif config
-        original = item
-      end
-
-      if original && config && config['replace']
-        original = original.gsub(config['replace'], config['to'] || Time.now.to_s)
-      end
-
-      original
-    end
-    
     def discover_extensions page
       require 'mime/types'
       types = MIME::Types[page.content_type]
@@ -80,19 +42,8 @@ module Yapra::Plugin::Publish
       logger.warn 'require mime-types is failed.'
     end
     
-    def download(item, url, referrer)
-      dir = config['dir']
+    def save config, item, page
       filename = construct_data(config['filename'], item)
-      
-      if config['before_hook']
-        eval(config['before_hook'])
-      end
-      page = agent.get(url, referrer)
-      logger.info "Download: #{url}"
-      if config['after_hook']
-        eval(config['after_hook'])
-      end
-      
       filename = page.filename unless filename
       
       if config['auto_suffix']

data/lib-plugins/yapra/plugin/publish/gmail.rb

@@ -1,23 +1,23 @@
-# module: Publish::Gmail -- Yuanying
-#
-# publish entry to imap mail.
-#
-# example:
-#
-# - module: Publish::Gmail
-#   config:
-#     username: username
-#     password: password
-#     wait: 1
-#     mail:
-#       subject_prefix: '[Yapra]'
-#       from: 'test@example.com'
-#       to: 'test2@example.com'
-#
 require 'net/imap'
 require 'yapra/plugin/publish/imap'
 
 module Yapra::Plugin::Publish
+  # = module: Publish::Gmail -- Yuanying
+  #
+  # publish entry to imap mail.
+  #
+  # example:
+  #
+  #    - module: Publish::Gmail
+  #      config:
+  #        username: username
+  #        password: password
+  #        wait: 1
+  #        mail:
+  #          subject_prefix: '[Yapra]'
+  #          from: 'test@example.com'
+  #          to: 'test2@example.com'
+  #
   class Gmail < Yapra::Plugin::Publish::Imap
     protected
     def create_imap server, port, usessl

data/lib-plugins/yapra/plugin/publish/imap.rb

@@ -1,26 +1,28 @@
-# module: Publish::Imap -- Yuanying
-#
-# publish entry to imap mail.
-#
-# example:
-#
-# - module: Publish::Imap
-#   config:
-#     username: username
-#     password: password
-#     imap_server: imap.gmail.com
-#     port: 993
-#     ssl: on
-#     wait: 1
-#     mail:
-#       subject_prefix: '[Yapra]'
-#       from: 'test@example.com'
-#       to: 'test2@example.com'
-#
 require 'net/imap'
+require 'yapra/version'
 require 'yapra/plugin/base'
 
 module Yapra::Plugin::Publish
+  # = module: Publish::Imap -- Yuanying
+  # 
+  # publish entry to imap mail.
+  # 
+  # example:
+  # 
+  #     - module: Publish::Imap
+  #       config:
+  #         username: username
+  #         password: password
+  #         imap_server: imap.gmail.com
+  #         port: 993
+  #         ssl: on
+  #         wait: 1
+  #         mail:
+  #           subject_prefix: '[Yapra]'
+  #           from_template: <%=item.author%> <test@example.com>
+  #           #from: 'test@example.com'
+  #           to: 'test2@example.com'
+  #
   class Imap < Yapra::Plugin::Base
     def run(data)
       username  = config['username']
@@ -39,26 +41,28 @@ module Yapra::Plugin::Publish
       to              = config['mail']['to']   || 'me@localhost'
       
       imap = create_imap server, port, usessl
-      logger.info(imap.greeting)
+      logger.debug(imap.greeting)
       
       imap.login(username, password)
+      logger.info('imap login was succeed.')
       imap.examine(mailbox)
       data.each do |item|
         date = item.date || item.dc_date || Time.now
         content = item.content_encoded || item.description || 'from Yapra.'
         content = [content].pack('m')
-        imap.append("inbox", <<EOF.gsub(/¥n/, "¥r¥n"), nil, date)
-From: #{from}
-To: #{to}
-Date: #{date.rfc2822}
-MIME-Version: 1.0
-Content-type: text/html; charset=UTF-8
-Content-transfer-encoding: base64
-X-Mailer: Yapra 0.1
-Subject: #{encode_field(subject_prefix + item.title)}
+        if config['mail']['from_template']
+          from = apply_template(config['mail']['from_template'], binding)
+        end
+        if config['mail']['to_template']
+          to = apply_template(config['mail']['to_template'], binding)
+        end
+        subject = (subject_prefix + item.title).gsub(/\n/, '').chomp
+        logger.debug("try append item: #{subject}")
+        boundary = "----_____====#{Time.now.to_i}--BOUDARY"
+        attachments = create_attachments(item, config)
+        imap.append(mailbox, apply_template(mail_template, binding), nil, date)
+        # puts apply_template(mail_template, binding)
 
-#{content}
-EOF
         sleep wait
       end
       imap.disconnect
@@ -71,12 +75,62 @@ EOF
       logger.debug("server: #{server}:#{port}, usessl = #{usessl}")
       Net::IMAP.new(server, port, usessl)
     end
+    
     def encode_field field
-      #field.gsub(/[　-瑤]\S*\s*/) {|x|
-        field.scan(/.{1,10}/).map {|y|
+      field.gsub(/[^\x01-\x7f]*/) {|x|
+        x.scan(/.{1,10}/).map {|y|
           "=?UTF-8?B?" + y.to_a.pack('m').chomp + "?="
         }.join("\n ")
-      #}
+      }
+    end
+    
+    def create_attachments item, config
+      attachments = []
+      attachment_attributes = config['mail']['attachments']
+      if attachment_attributes.kind_of?(String)
+        file = item.__send__(attachment_attributes)
+        attachments << file if file.kind_of?(WWW::Mechanize::File)
+      elsif attachment_attributes.kind_of?(Array)
+        attachment_attributes.each do |atc|
+          file = item.__send__(atc)
+          attachments << file if file.kind_of?(WWW::Mechanize::File)
+        end
+      end
+      attachments
+    end
+    
+    def mail_template
+      return <<EOT
+From: <%=encode_field(from) %>
+To: <%=encode_field(to) %>
+Date: <%=date.rfc2822 %>
+MIME-Version: 1.0
+X-Mailer: Yapra <%=Yapra::VERSION::STRING %>
+Subject: <%=encode_field(subject) %>
+Content-Type: multipart/mixed; boundary="<%=boundary -%>"
+
+This is a multi-part message in MIME format.
+
+--<%=boundary %>
+Content-type: text/html; charset=UTF-8
+Content-transfer-encoding: base64
+
+<%=content %>
+
+--<%=boundary %>
+<% attachments.each do |file| -%>
+Content-Type: <%=file.header['Content-Type'] %>;
+	name="<%=encode_field(file.filename) %>"
+Content-Disposition: attachment;
+	filename="<%=encode_field(file.filename) %>"
+Content-Transfer-Encoding: base64
+
+<%=[file.body].pack('m') -%>
+
+--<%=boundary %>
+
+<% end -%>
+EOT
     end
   end
 end

data/lib-plugins/yapra/plugin/publish/on_memory_download.rb

@@ -0,0 +1,86 @@
+require 'yapra/plugin/mechanize_base'
+
+module Yapra::Plugin::Publish
+  # = Publish::OnMemoryDownload -- Yuanying
+  # 
+  # download web resource and set to item attribute.
+  # 
+  # example: 
+  # 
+  #     - module: Publish::OnMemoryDownload
+  #       config:
+  #         regexp: http://www\.yahoo\.co\.jp/*
+  #         attribute: binary_image
+  #         before_hook: "agent.get('http://www.yahoo.co.jp/'); sleep 1"
+  #         url:
+  #           attribute: link
+  #           replace: index(\d*?)\.html
+  #           to: file\1.zip
+  #         referrer:
+  #           attribute: link
+  #           replace: 'zip'
+  #           to: 'html'
+  #
+  class OnMemoryDownload < Yapra::Plugin::MechanizeBase
+    def run(data)
+      regexp = nil
+      if config['regexp']
+        regexp = Regexp.new(config['regexp'])
+      else
+        regexp = /^(https?|ftp)(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)$/
+      end
+      
+      wait = config['wait'] || 3
+
+      data.each do |item|
+        url = construct_data(config['url'], item, item.respond_to?('link') ? item.link : item)
+
+        if regexp =~ url
+          logger.debug "Download start: #{url}"
+          referrer = construct_data(config['referrer'], item)
+          download(item, url, referrer)
+          sleep wait
+        end
+      end
+      
+      data
+    end
+    
+    protected
+    def construct_data(config, item, original=nil)
+      if config && config['attribute']
+        if item.respond_to?(config['attribute'])
+          original = item.__send__(config['attribute'])
+        end
+      elsif config
+        original = item
+      end
+
+      if original && config && config['replace']
+        original = original.gsub(config['replace'], config['to'] || Time.now.to_s)
+      end
+
+      original
+    end
+    
+    def save config, item, page
+      set_attribute_to item, config['attribute'], page
+    end
+    
+    def download(item, url, referrer)
+      if config['before_hook']
+        eval(config['before_hook'])
+      end
+      
+      dir = config['dir']
+      
+      page = agent.get(url, referrer)
+      
+      save(config, item, page)
+      
+      if config['after_hook']
+        eval(config['after_hook'])
+      end
+    end
+  end
+end

data/lib-plugins/yapra/plugin/test/append_entry.rb

@@ -1,17 +1,17 @@
-# module: Test::AppendEntry -- Yuanying
-#
-# append entry to data array.
-#
-# example:
-#
-# - module: Test::AppendEntry
-#   config:
-#     title: 'title'
-#     description: 'description.'
-#
 require 'yapra/plugin/base'
 
 module Yapra::Plugin::Test
+  # module: Test::AppendEntry -- Yuanying
+  # 
+  # append entry to data array.
+  # 
+  # example:
+  # 
+  #     - module: Test::AppendEntry
+  #       config:
+  #         title: 'title'
+  #         description: 'description.'
+  
   class AppendEntry < Yapra::Plugin::Base
     def run(data)
       item = RSS::RDF::Item.new

data/plugins/Filter/grep.rb

@@ -0,0 +1,24 @@
+## Filter input by given regular expression -- IKeJI
+## 
+## Filter input by given regular expression.
+## The test will be done with the result of to_s method of the input.
+## invert option will invert results(-v option of UNIX grep command).
+##
+## - module: grep
+##   config:
+##     regex: "[あ-ん]"
+##     invert: false
+
+def grep(config,data)
+  regex     = Regexp.new(config["regex"])
+  invert    = config["invert"] || false
+  attribute = config['attribute']
+  
+  data.select do |i|
+    if attribute
+      invert ^ (regex =~ i.__send__(attribute).to_s)
+    else
+      invert ^ (regex =~ i.to_s)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -6,6 +6,8 @@ rescue LoadError
   require 'spec'
 end
 
+$fixture_dir = File.join(File.dirname(__FILE__), '..', 'fixtures') unless $fixture_dir
+
 $:.unshift(File.dirname(__FILE__) + '/../lib')
 $:.unshift(File.dirname(__FILE__) + '/../lib-plugins')
 require 'yapra'

data/spec/yapra/config_spec.rb

@@ -0,0 +1,72 @@
+require File.dirname(__FILE__) + '/../spec_helper.rb'
+
+require 'yaml'
+require 'yapra/config'
+
+describe Yapra::Config do
+  
+  it 'should create logger from env setting' do
+    config = Yapra::Config.new(YAML.load_file(File.join($fixture_dir, 'config', 'pragger_like.yml')))
+    config.env.update({
+      'log' => {
+        'out'   => 'stderr',
+        'level' => 'debug'
+      }
+    })
+    logger = config.create_logger
+    logger.should be_debug
+  end
+  
+  describe 'which is initialized by pragger like config file' do
+    before do
+      @config = Yapra::Config.new(YAML.load_file(File.join($fixture_dir, 'config', 'pragger_like.yml')))
+    end
+    
+    it 'should have empty env.' do
+      @config.env.should == {}
+    end
+    
+    it 'should have one pipeline command which is named "default".' do
+      @config.should have(1).pipeline_commands
+    end
+    
+    it 'should have pipeline command named "default"' do
+      @config.pipeline_commands.should have_key('default')
+    end
+  end
+  
+  describe 'which is initialized by python habu like config file' do
+    before do
+      @habu_like_file = YAML.load_file(File.join($fixture_dir, 'config', 'habu_like.yml'))
+      @config = Yapra::Config.new(@habu_like_file)
+    end
+    
+    it 'should have env hash from habu_like_config_file "global" section.' do
+      @config.env.should == @habu_like_file['global']
+    end
+    
+    it 'should have pipeline_commands from habu_like_config_file "pipeline" section.' do
+      @config.pipeline_commands.should == @habu_like_file['pipeline']
+    end
+  end
+  
+  describe 'which is initialized by mixed type config file' do
+    before do
+      @mixed_type_file = YAML.load_file(File.join($fixture_dir, 'config', 'mixed_type.yml'))
+      @config = Yapra::Config.new(@mixed_type_file)
+    end
+    
+    it 'should have env hash from mixed_type "global" section.' do
+      @config.env.should == @mixed_type_file['global']
+    end
+    
+    it 'should have one pipeline command which is named "default".' do
+      @config.should have(1).pipeline_commands
+    end
+    
+    it 'should have pipeline command named "default"' do
+      @config.pipeline_commands.should have_key('default')
+    end
+  end
+
+end

data/spec/yapra/legacy_plugin/base_spec.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/../../spec_helper.rb'
+
+require 'yapra/legacy_plugin/base'
+
+describe Yapra::LegacyPlugin::Base do
+  
+  before do
+    @pipeline = mock('pipeline', :null_object => true)
+  end
+  
+  describe 'which is initialized by "fixtures/legacy_test_plugin.rb"' do
+    before do
+      @path = File.join($fixture_dir, 'legacy_plugin', 'legacy_test_plugin.rb')
+      @plugin = Yapra::LegacyPlugin::Base.new(@pipeline, @path)
+    end
+    
+    it 'should have _yapra_run_method named "legacy_test_plugin".' do
+      @plugin._yapra_run_method.should == 'legacy_test_plugin'
+    end
+    
+    it 'should have plugin source.' do
+      @plugin.source.should == File.read(@path)
+    end
+    
+    it 'should recieve message "legacy_test_plugin" when _yapra_run_as_legacy_plugin is called.' do
+      config  = mock('config')
+      data    = []
+      @plugin.stub!(:legacy_test_plugin).and_return(:default_value)
+      @plugin.should_receive(:legacy_test_plugin).with(config, data)
+      @plugin._yapra_run_as_legacy_plugin(config, data)
+    end
+  end
+end

data/spec/yapra/legacy_plugin/registry_factory_spec.rb

@@ -0,0 +1,19 @@
+require File.dirname(__FILE__) + '/../../spec_helper.rb'
+
+require 'yapra/legacy_plugin/registry_factory'
+
+describe Yapra::LegacyPlugin::RegistryFactory do
+  before do
+    @pipeline = mock('pipeline', :null_object => true)
+  end
+  
+  it 'should create advance mode registry from string "advance"' do
+    factory = Yapra::LegacyPlugin::RegistryFactory.new([], 'advance')
+    factory.create(@pipeline).class.should == Yapra::LegacyPlugin::AdvanceModeRegistry
+  end
+  
+  it 'should create compatible mode registry from string "compatible"' do
+    factory = Yapra::LegacyPlugin::RegistryFactory.new([], 'compatible')
+    factory.create(@pipeline).class.should == Yapra::LegacyPlugin::CompatibleModeRegistry
+  end
+end