yapra 0.1.2 → 0.1.3

data/{History.txt → ChangeLog}

@@ -1,3 +1,11 @@
+== 0.1.3
+
+* 1 bug fix:
+
+* 2 majar enhancement:
+  * Publish::Mail plugin is added.
+  * Feed::Custom supports url list.
+
 == 0.1.2
 
 * 1 bug fix:

data/LICENCE

@@ -1,4 +1,4 @@
-Copyright (c) 2008 fraction.jp. Some rights reserved.
+Copyright (c) 2008 Yuanying Ohtsuka
 
 Original from 2008-06-12 http://pragger.ikejisoft.com/
 

data/README.mdown

@@ -0,0 +1,70 @@
+yapra
+====================================
+-   http://yapra.rubyforge.org/
+-   http://github.com/yuanying/yapra/tree/master
+
+DESCRIPTION:
+------------------------------------
+Yet Another Pragger(http://pragger.ikejisoft.com/) implementation.
+
+FEATURES/PROBLEMS:
+------------------------------------
+-   99% compatible of Pragger.
+-   Class-based plugin support.
+-   Loadpass based plugin loading strategy. (also support pragger's plugin in advance mode.)
+-   Support Python habu like config file.
+
+SYNOPSIS:
+------------------------------------
+
+### Use at command
+
+    $ yapra -c config_file.yml
+
+### Use in your application
+
+    require 'yapra/runtime'
+    require 'yapra/config'
+
+    config = YAML.load(config_file)
+    config = Yapra::Config.new(config)
+
+    Yapra::Runtime.logger = Logger.new(STDOUT)
+
+    yapra = Yapra::Runtime.new(config.env)
+    yapra.execute(config.pipeline_commands)
+
+### REQUIREMENTS:
+
+-   mechanize (>= 0.7.6)
+
+INSTALL:
+------------------------------------
+
+    sudo gem install yapra
+
+LICENSE:
+------------------------------------
+
+    (The MIT License)
+
+    Copyright (c) 2008 Yuanying Ohtsuka
+
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    'Software'), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+    IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+    CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+    TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+    SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -1,13 +1,119 @@
-require 'config/requirements'
-require 'config/hoe' # setup Hoe + all gem configuration
-
-Dir['tasks/**/*.rake'].each { |rake| load rake }
-
-##############################################################################
-# SVN
-##############################################################################
-
-desc "Add new files to subversion"
-task :svn_add do
-   system "svn status | grep '^\?' | sed -e 's/? *//' | sed -e 's/ /\ /g' | xargs svn add"
-end
+$:.unshift('lib')
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/testtask'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/contrib/rubyforgepublisher'
+require 'rake/contrib/sshpublisher'
+require 'fileutils'
+require 'yapra/version'
+include FileUtils
+NAME              = "yapra"
+AUTHOR            = "yuanying"
+EMAIL             = "yuanying at fraction dot jp"
+DESCRIPTION       = "Yet another pragger implementation."
+RUBYFORGE_PROJECT = "yapra"
+HOMEPATH          = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+BIN_FILES         = %w( yapra )
+VERS              = Yapra::VERSION::STRING
+
+REV = File.read(".svn/entries")[/committed-rev="(d+)"/, 1] rescue nil
+CLEAN.include ['**/.*.sw?', '*.gem', '.config' ,'pkg']
+RDOC_OPTS = [
+  '--title', "#{NAME} documentation",
+  "--charset", "utf-8",
+  "--opname", "index.html",
+  "--line-numbers",
+  "--main", "README.mdown",
+  "--inline-source",
+]
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }
+
+task :default => [:spec]
+task :package => [:clean]
+
+spec = Gem::Specification.new do |s|
+  s.name              = NAME
+  s.version           = VERS
+  s.platform          = Gem::Platform::RUBY
+  s.has_rdoc          = true
+  s.extra_rdoc_files  = ["README.mdown", "ChangeLog"]
+  s.rdoc_options     += RDOC_OPTS + ['--exclude', '^(examples|extras)/']
+  s.summary           = DESCRIPTION
+  s.description       = DESCRIPTION
+  s.author            = AUTHOR
+  s.email             = EMAIL
+  s.homepage          = HOMEPATH
+  s.executables       = BIN_FILES
+  s.rubyforge_project = RUBYFORGE_PROJECT
+  s.bindir            = "bin"
+  s.require_paths     << "lib-plugins"
+  s.autorequire       = ""
+  s.test_files        = Dir["test/test_*.rb"]
+
+  s.add_dependency('mechanize', '>=0.7.6')
+  #s.required_ruby_version = '>= 1.8.2'
+
+  s.files = %w(README.mdown ChangeLog Rakefile LICENCE) +
+    Dir.glob("{bin,doc,fixtures,legacy_plugins,lib,lib-plugins,plugins,spec,tasks,website,script}/**/*") + 
+    # Dir.glob("ext/**/*.{h,c,rb}") +
+    # Dir.glob("examples/**/*.rb") +
+    # Dir.glob("tools/*.rb")
+
+  s.extensions = FileList["ext/**/extconf.rb"].to_a
+end
+
+Rake::GemPackageTask.new(spec) do |p|
+  p.need_tar = true
+  p.gem_spec = spec
+end
+
+task :install do
+  name = "#{NAME}-#{VERS}.gem"
+  sh %{rake package}
+  sh %{sudo gem install pkg/#{name}}
+end
+
+task :uninstall => [:clean] do
+  sh %{sudo gem uninstall #{NAME}}
+end
+
+
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'html'
+  rdoc.options += RDOC_OPTS
+  rdoc.template = "resh"
+  #rdoc.template = "#{ENV['template']}.rb" if ENV['template']
+  if ENV['DOC_FILES']
+    rdoc.rdoc_files.include(ENV['DOC_FILES'].split(/,\s*/))
+  else
+    rdoc.rdoc_files.include('README', 'ChangeLog')
+    rdoc.rdoc_files.include('lib/**/*.rb')
+    rdoc.rdoc_files.include('ext/**/*.c')
+  end
+end
+
+desc "Publish to RubyForge"
+task :rubyforge => [:rdoc, :package] do
+  require 'rubyforge'
+  Rake::RubyForgePublisher.new(RUBYFORGE_PROJECT, 'yuanying').upload
+end
+
+desc 'Package and upload the release to gemcutter.'
+task :release => [:clean, :package] do |t|
+  v = ENV["VERSION"] or abort "Must supply VERSION=x.y.z"
+  abort "Versions don't match #{v} vs #{VERS}" unless v == VERS
+  pkg = "pkg/#{NAME}-#{VERS}"
+
+  files = [
+    "#{pkg}.tgz",
+    "#{pkg}.gem"
+  ].compact
+
+  puts "Releasing #{NAME} v. #{VERS}"
+  sh %{gem push #{pkg}.gem}
+end

data/bin/yapra

@@ -23,11 +23,13 @@ require 'yapra/runtime'
 require 'yapra/config'
 require 'yapra/legacy_plugin/registry_factory'
 
-Version       = Yapra::VERSION::STRING
 mode          = 'compatible'
 config_files  = []#"config.yaml"
 loglebel      = nil
+
 opt = OptionParser.new
+opt.version = Yapra::VERSION::STRING
+opt.release = nil
 opt.on("-c", "--configfile CONFIGFILE") {|v| config_files << v }
 opt.on("-d", "--configfile-directory CONFIGFILE_DIRECTORY") { |v|
   Dir.glob(File.join(v, '**/*.yml')).each do |file|

data/lib/yapra/pipeline.rb

@@ -1,31 +1,14 @@
 require 'yapra'
+require 'yapra/pipeline_base'
 require 'yapra/runtime'
 require 'yapra/inflector'
 require 'yapra/legacy_plugin/base'
 
-class Yapra::Pipeline
-  attr_reader :yapra, :context
-  attr_writer :logger
+class Yapra::Pipeline < Yapra::PipelineBase
   attr_accessor :legacy_plugin_registry
   
   UPPER_CASE = /[A-Z]/
   
-  def initialize pipeline_name, yapra=Yapra::Runtime.new
-    @logger             = nil
-    @yapra              = yapra
-    @context            = { 'pipeline_name' => pipeline_name }
-    
-    @module_name_prefix = construct_module_name_prefix yapra.env
-  end
-  
-  def name
-    self.context[ 'pipeline_name' ]
-  end
-  
-  def logger
-    return @logger || Yapra::Runtime.logger
-  end
-  
   # start pipeline from commands.
   # 
   # example:
@@ -87,20 +70,11 @@ class Yapra::Pipeline
   
   def run_class_based_plugin command, data
     self.logger.debug("evaluate plugin as class based")
-    load_error_stack = []
-    plugin_class = nil
-    @module_name_prefix.each do |prefix|
-      yapra_module_name = "#{prefix}#{command['module']}"
-      begin
-        plugin_class      = Yapra.load_class_constant(yapra_module_name)
-        break if plugin_class
-      rescue LoadError, NameError => ex
-        load_error_stack << ex
-      end
-    end
-    raise_load_error(load_error_stack, command) unless plugin_class
+    plugin = load(command['module'])
+    
+    # yml pipeline specific.
+    plugin.plugin_config  = command['config'] if plugin.respond_to?('plugin_config=')
     
-    plugin = initialize_plugin(plugin_class, command)
     @plugins << plugin
     data = plugin.run(data)
     return data
@@ -116,24 +90,4 @@ class Yapra::Pipeline
     load_error.set_backtrace(backtrace)
     raise load_error
   end
-  
-  def initialize_plugin plugin_class, command
-    plugin                = plugin_class.new
-    plugin.yapra          = yapra if plugin.respond_to?('yapra=')
-    plugin.pipeline       = self  if plugin.respond_to?('pipeline=')
-    plugin.plugin_config  = command['config'] if plugin.respond_to?('plugin_config=')
-    plugin
-  end
-  
-  def construct_module_name_prefix env
-    module_name_prefix = [ 'Yapra::Plugin::', '' ]
-    if env['module_name_prefix']
-      if env['module_name_prefix'].kind_of?(Array)
-        module_name_prefix = env['module_name_prefix']
-      else
-        module_name_prefix = [ env['module_name_prefix'] ]
-      end
-    end
-    module_name_prefix
-  end
 end

data/lib/yapra/pipeline_base.rb

@@ -0,0 +1,64 @@
+require 'yapra'
+require 'yapra/inflector'
+require 'yapra/legacy_plugin/base'
+
+class Yapra::PipelineBase
+  attr_reader :yapra, :context
+  attr_writer :logger
+  
+  def initialize pipeline_name, yapra=Yapra::Runtime.new
+    @logger             = nil
+    @yapra              = yapra
+    @context            = { 'pipeline_name' => pipeline_name }
+    
+    @module_name_prefix = construct_module_name_prefix yapra.env
+  end
+  
+  def name
+    self.context[ 'pipeline_name' ]
+  end
+  
+  def logger
+    return @logger || Yapra::Runtime.logger
+  end
+  
+  def load plugin_name
+    load_error_stack = []
+    plugin_class = nil
+    @module_name_prefix.each do |prefix|
+      yapra_module_name = "#{prefix}#{plugin_name}"
+      begin
+        plugin_class      = Yapra.load_class_constant(yapra_module_name)
+        break if plugin_class
+      rescue LoadError, NameError => ex
+        load_error_stack << ex
+      end
+    end
+    raise_load_error(load_error_stack, command) unless plugin_class
+    
+    plugin = initialize_plugin( plugin_class )
+    
+    plugin
+  end
+  
+  protected
+  def initialize_plugin plugin_class
+    plugin                = plugin_class.new
+    plugin.yapra          = yapra if plugin.respond_to?('yapra=')
+    plugin.pipeline       = self  if plugin.respond_to?('pipeline=')
+    
+    plugin
+  end
+  
+  def construct_module_name_prefix env
+    module_name_prefix = [ 'Yapra::Plugin::', '' ]
+    if env['module_name_prefix']
+      if env['module_name_prefix'].kind_of?(Array)
+        module_name_prefix = env['module_name_prefix']
+      else
+        module_name_prefix = [ env['module_name_prefix'] ]
+      end
+    end
+    module_name_prefix
+  end
+end

data/lib/yapra/plugin/mechanize_base.rb

@@ -8,7 +8,7 @@ class Yapra::Plugin::MechanizeBase < Yapra::Plugin::Base
     pipeline.context['mechanize_agent']
   end
   
-  def extract_attribute_from element, item
+  def extract_attribute_from element, item, binding=nil
     if plugin_config['extract_xpath']
       plugin_config['extract_xpath'].each do |k, v|
         value = nil

data/lib/yapra/rb_pipeline.rb

@@ -0,0 +1,7 @@
+require 'yapra'
+require 'yapra/inflector'
+require 'yapra/legacy_plugin/base'
+
+class Yapra::RbPipeline < Yapra::PipelineBase
+
+end

data/lib/yapra/version.rb

@@ -2,7 +2,7 @@ module Yapra
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 1
-    TINY  = 2
+    TINY  = 3
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/yapra.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2008 fraction.jp. Yuanying, Some rights reserved.
+# Copyright (c) 2008 Yuanying Ohtsuka
 # Original from 2008-06-12 http://pragger.ikejisoft.com/
 #
 # Permission is hereby granted, free of charge, to any person obtaining

data/lib-plugins/yapra/plugin/feed/custom.rb

@@ -20,24 +20,37 @@ module Yapra::Plugin::Feed
   #           content_encoded: '<div><%= title %></div>'
   class Custom < Yapra::Plugin::MechanizeBase
     def run(data)
-      page    = agent.get(config['url'])
-      root    = page.root
-      
+      urls = 
+        if config['url'].kind_of?(Array)
+          config['url']
+        else
+          [ config['url'] ]
+        end
       xconfig = config['extract_xpath']
+      wait    = config['wait'] || 1
+      capture = xconfig['capture']
+      split   = xconfig['split']
       
-      if xconfig['capture']
-        root = root.at(xconfig['capture'])
-      end
-      split = xconfig['split']
       xconfig.delete('capture')
       xconfig.delete('split')
       
-      root.search(split).each do |element|
-        item = RSS::RDF::Item.new
-        
-        extract_attribute_from element, item
+      urls.each do |url|
+        logger.debug("Process: #{url}")
+        page    = agent.get(url)
+        root    = page.root
+
+        if capture
+          root = root.at(capture)
+        end
+
+        root.search(split).each do |element|
+          item = RSS::RDF::Item.new
+
+          extract_attribute_from element, item, binding
 
-        data << item
+          data << item
+        end
+        sleep wait
       end
       
       data

data/lib-plugins/yapra/plugin/filter/entry_full_text.rb

@@ -47,7 +47,7 @@ module Yapra::Plugin::Filter
             item = new_item
           end
 
-          extract_attribute_from page.root, item
+          extract_attribute_from page.root, item, binding
 
         end
         item

data/lib-plugins/yapra/plugin/publish/gmail.rb

@@ -20,8 +20,11 @@ module Yapra::Plugin::Publish
   #
   class Gmail < Yapra::Plugin::Publish::Imap
     protected
-    def create_imap server, port, usessl
-      Net::IMAP.new('imap.gmail.com', 993, true)
+    def prepare
+      super
+      config['imap_server'] = 'imap.gmail.com'
+      config['port']        = 993
+      config['ssl']         = true
     end
   end
-end
+end

data/lib-plugins/yapra/plugin/publish/imap.rb

@@ -1,6 +1,5 @@
 require 'net/imap'
-require 'yapra/version'
-require 'yapra/plugin/base'
+require 'yapra/plugin/publish/mail'
 
 module Yapra::Plugin::Publish
   # = module: Publish::Imap -- Yuanying
@@ -23,114 +22,32 @@ module Yapra::Plugin::Publish
   #           #from: 'test@example.com'
   #           to: 'test2@example.com'
   #
-  class Imap < Yapra::Plugin::Base
-    def run(data)
-      username  = config['username']
-      password  = config['password']
-      server    = config['imap_server'] || 'imap.gmail.com'
-      port      = config['port'] || 993
-      usessl    = ('off' != config['ssl'])
-      mailbox   = config['mailbox'] || 'inbox'
-      wait      = config['wait'] || 1
-      
-      unless config['mail']
-        config['mail'] = {}
-      end
-      subject_prefix  = config['mail']['subject_prefix'] || ''
-      from            = config['mail']['from'] || 'yapra@localhost'
-      to              = config['mail']['to']   || 'me@localhost'
-      
-      imap = create_imap server, port, usessl
-      logger.debug(imap.greeting)
-      
-      imap.login(username, password)
-      logger.info('imap login was succeed.')
-      imap.examine(mailbox)
-      data.each do |item|
-        date = item.date || item.dc_date || Time.now
-        content = item.content_encoded || item.description || 'from Yapra.'
-        content = [content].pack('m')
-        if config['mail']['from_template']
-          from = apply_template(config['mail']['from_template'], binding)
-        end
-        if config['mail']['to_template']
-          to = apply_template(config['mail']['to_template'], binding)
-        end
-        subject = (subject_prefix + item.title).gsub(/\n/, '').chomp
-        logger.debug("try append item: #{subject}")
-        boundary = "----_____====#{Time.now.to_i}--BOUDARY"
-        attachments = create_attachments(item, config)
-        imap.append(mailbox, apply_template(mail_template, binding), nil, date)
-        # puts apply_template(mail_template, binding)
-
-        sleep wait
-      end
-      imap.disconnect
-      
-      data
-    end
-    
+  class Imap < Mail
     protected
-    def create_imap server, port, usessl
-      logger.debug("server: #{server}:#{port}, usessl = #{usessl}")
-      Net::IMAP.new(server, port, usessl)
+    def prepare
+      super
+      config['imap_server'] = config['imap_server'] || 'imap.gmail.com'
+      config['port']        = config['port'] || 993
+      config['ssl']         = ('off' != config['ssl'])
+      config['mailbox']     = config['mailbox'] || 'inbox'
     end
-    
-    def encode_field field
-      field.gsub(/[^\x01-\x7f]*/) {|x|
-        x.scan(/.{1,10}/).map {|y|
-          "=?UTF-8?B?" + y.to_a.pack('m').chomp + "?="
-        }.join("\n ")
-      }
-    end
-    
-    def create_attachments item, config
-      attachments = []
-      attachment_attributes = config['mail']['attachments']
-      if attachment_attributes.kind_of?(String)
-        file = item.__send__(attachment_attributes)
-        attachments << file if file.kind_of?(WWW::Mechanize::File)
-      elsif attachment_attributes.kind_of?(Array)
-        attachment_attributes.each do |atc|
-          file = item.__send__(atc)
-          attachments << file if file.kind_of?(WWW::Mechanize::File)
-        end
-      end
-      attachments
-    end
-    
-    def mail_template
-      return <<EOT
-From: <%=encode_field(from) %>
-To: <%=encode_field(to) %>
-Date: <%=date.rfc2822 %>
-MIME-Version: 1.0
-X-Mailer: Yapra <%=Yapra::VERSION::STRING %>
-Subject: <%=encode_field(subject) %>
-Content-Type: multipart/mixed; boundary="<%=boundary -%>"
-
-This is a multi-part message in MIME format.
 
---<%=boundary %>
-Content-type: text/html; charset=UTF-8
-Content-transfer-encoding: base64
-
-<%=content %>
-
---<%=boundary %>
-<% attachments.each do |file| -%>
-Content-Type: <%=file.header['Content-Type'] %>;
-	name="<%=encode_field(file.filename) %>"
-Content-Disposition: attachment;
-	filename="<%=encode_field(file.filename) %>"
-Content-Transfer-Encoding: base64
-
-<%=[file.body].pack('m') -%>
+    def open_session
+      logger.debug("server: #{config['imap_server']}:#{config['port']}, usessl = #{config['ssl']}")
+      imap = Net::IMAP.new(config['imap_server'], config['port'], config['ssl'])
+      logger.debug(imap.greeting)
+      imap.login(config['username'], config['password'])
+      logger.info('imap login was succeed.')
+      imap.examine(config['mailbox'])
+      @session = imap
+    end
 
---<%=boundary %>
+    def close_session
+      @session.disconnect
+    end
 
-<% end -%>
-EOT
+    def send_item(msg, opt)
+      @session.append(config['mailbox'], msg, nil, opt['date'])
     end
   end
-end
+end