yapra 0.1.1 → 0.1.2

data/History.txt

@@ -1,3 +1,12 @@
+== 0.1.2
+
+* 1 bug fix:
+  * rss 1.0 format bug is fixed. (RSS::save)
+
+* 2 majar enhancement:
+  * Error handling of plugin is added.
+  * Plugin load error has more detailed informations.
+
 == 0.1.1
 
 * 1 bug fix:

data/Manifest.txt

@@ -64,12 +64,16 @@ lib-plugins/yapra/plugin/config/web_post.rb
 lib-plugins/yapra/plugin/feed/custom.rb
 lib-plugins/yapra/plugin/feed/load.rb
 lib-plugins/yapra/plugin/filter/apply_template.rb
+lib-plugins/yapra/plugin/filter/deduped.rb
 lib-plugins/yapra/plugin/filter/entry_full_text.rb
 lib-plugins/yapra/plugin/publish/file_download.rb
 lib-plugins/yapra/plugin/publish/gmail.rb
 lib-plugins/yapra/plugin/publish/imap.rb
 lib-plugins/yapra/plugin/publish/on_memory_download.rb
 lib-plugins/yapra/plugin/test/append_entry.rb
+lib-plugins/yapra/plugin/test/raise_error.rb
+lib-plugins/yapra/plugin/test/test.rb
+lib-plugins/yapra/plugin/test/test2.rb
 lib/yapra.rb
 lib/yapra/config.rb
 lib/yapra/inflector.rb
@@ -101,6 +105,7 @@ spec/spec_helper.rb
 spec/yapra/config_spec.rb
 spec/yapra/legacy_plugin/base_spec.rb
 spec/yapra/legacy_plugin/registry_factory_spec.rb
+spec/yapra/pipeline_spec.rb
 spec/yapra_spec.rb
 tasks/deployment.rake
 tasks/environment.rake

data/bin/yapra

@@ -49,18 +49,27 @@ config_files << 'config.yaml' if config_files.size == 0
 
 legacy_plugin_registry_factory = Yapra::LegacyPlugin::RegistryFactory.new(legacy_plugin_directory_paths, mode)
 
+yapra = nil
 config_files.each do |config_file|
-  config = YAML.load(File.read(config_file).toutf8.gsub(/base64::([\w+\/]+=*)/){ Base64.decode64($1) })
-  config = Yapra::Config.new(config)
-  config.env.update({
-    'log' => {
-      'level' => loglebel
-    }
-  }) if loglebel
-  Yapra::Runtime.logger = config.create_logger
-  yapra = Yapra::Runtime.new(
-    config.env,
-    legacy_plugin_registry_factory
-  )
-  yapra.execute(config.pipeline_commands)
+  begin
+    config = YAML.load(File.read(config_file).toutf8.gsub(/base64::([\w+\/]+=*)/){ Base64.decode64($1) })
+    config = Yapra::Config.new(config)
+    config.env.update({
+      'log' => {
+        'level' => loglebel
+      }
+    }) if loglebel
+    Yapra::Runtime.logger = config.create_logger
+    yapra = Yapra::Runtime.new(
+      config.env,
+      legacy_plugin_registry_factory
+    )
+    yapra.execute(config.pipeline_commands)
+  rescue => ex
+    STDERR.puts("* Pipeline, '#{yapra.current_pipeline.name}'") if yapra && yapra.current_pipeline
+    STDERR.puts("#{ex.class.name} in '#{ex.message}'")
+    ex.backtrace.each do |t|
+      STDERR.puts(t)
+    end
+  end
 end

data/legacy_plugins/RSS/save.rb

@@ -18,7 +18,7 @@ def save(config,data)
   rss = RSS::Maker.make("1.0") do |maker|
     maker.channel.about = config['about'] || config['link'] || "http://example.net/"
     maker.channel.title = config['title'] || "Pragger output"
-    maker.channel.description = config['description'] || ""
+    maker.channel.description = config['description'] || "Generated by Yapra."
     maker.channel.link = config['link'] || "http://example.net/"
     
     data.each do |i|

data/lib/yapra/pipeline.rb

@@ -1,22 +1,29 @@
 require 'yapra'
+require 'yapra/runtime'
 require 'yapra/inflector'
 require 'yapra/legacy_plugin/base'
 
 class Yapra::Pipeline
   attr_reader :yapra, :context
+  attr_writer :logger
   attr_accessor :legacy_plugin_registry
   
   UPPER_CASE = /[A-Z]/
   
-  def initialize yapra, pipeline_name
+  def initialize pipeline_name, yapra=Yapra::Runtime.new
+    @logger             = nil
     @yapra              = yapra
     @context            = { 'pipeline_name' => pipeline_name }
     
     @module_name_prefix = construct_module_name_prefix yapra.env
   end
   
+  def name
+    self.context[ 'pipeline_name' ]
+  end
+  
   def logger
-    Yapra::Runtime.logger
+    return @logger || Yapra::Runtime.logger
   end
   
   # start pipeline from commands.
@@ -41,8 +48,20 @@ class Yapra::Pipeline
   #       }
   #     ])
   def run pipeline_command, data=[]
-    pipeline_command.inject(data) do |data, command|
-      execute_plugin(command, data.clone)
+    @plugins = []
+    begin
+      pipeline_command.inject(data) do |data, command|
+        execute_plugin(command, data.clone)
+      end
+    rescue => ex
+      @plugins.each do |plugin|
+        begin
+          plugin.on_error(ex) if plugin.respond_to?('on_error')
+        rescue => exx
+          self.logger.error("error is occured when error handling: #{exx.message}")
+        end
+      end
+      raise ex
     end
   end
   
@@ -62,23 +81,48 @@ class Yapra::Pipeline
   
   def run_legacy_plugin command, data
     self.logger.debug("evaluate plugin as legacy")
-    legacy_plugin_registry.get(command['module'])._yapra_run_as_legacy_plugin(command['config'], data)
+    data = legacy_plugin_registry.get(command['module'])._yapra_run_as_legacy_plugin(command['config'], data)
+    return data
   end
   
   def run_class_based_plugin command, data
     self.logger.debug("evaluate plugin as class based")
+    load_error_stack = []
     plugin_class = nil
     @module_name_prefix.each do |prefix|
       yapra_module_name = "#{prefix}#{command['module']}"
-      plugin_class      = Yapra.load_class_constant(yapra_module_name)
-      break if plugin_class
+      begin
+        plugin_class      = Yapra.load_class_constant(yapra_module_name)
+        break if plugin_class
+      rescue LoadError, NameError => ex
+        load_error_stack << ex
+      end
     end
-    raise LoadError unless plugin_class
+    raise_load_error(load_error_stack, command) unless plugin_class
+    
+    plugin = initialize_plugin(plugin_class, command)
+    @plugins << plugin
+    data = plugin.run(data)
+    return data
+  end
+  
+  def raise_load_error load_error_stack, command
+    load_error = LoadError.new("#{command['module']} module is not found.")
+    backtrace = load_error.backtrace || []
+    load_error_stack.each do |e|
+      backtrace << "#{e.class.name} in '#{e.message}'"
+      backtrace = backtrace + e.backtrace
+    end
+    load_error.set_backtrace(backtrace)
+    raise load_error
+  end
+  
+  def initialize_plugin plugin_class, command
     plugin                = plugin_class.new
     plugin.yapra          = yapra if plugin.respond_to?('yapra=')
     plugin.pipeline       = self  if plugin.respond_to?('pipeline=')
     plugin.plugin_config  = command['config'] if plugin.respond_to?('plugin_config=')
-    plugin.run(data)
+    plugin
   end
   
   def construct_module_name_prefix env

data/lib/yapra/plugin/mechanize_base.rb

@@ -11,7 +11,14 @@ class Yapra::Plugin::MechanizeBase < Yapra::Plugin::Base
   def extract_attribute_from element, item
     if plugin_config['extract_xpath']
       plugin_config['extract_xpath'].each do |k, v|
-        value = element.search(v).to_html.toutf8
+        value = nil
+        case v.class.to_s
+        when 'String'
+          value = element.search(v).to_html.toutf8
+        when 'Hash'
+          ele = element.at( v['first_node'] )
+          value = ( ele.nil? ) ? nil : ele.get_attribute( v['attr'] )
+        end
         set_attribute_to item, k, value
       end
     end

data/lib/yapra/runtime.rb

@@ -1,3 +1,4 @@
+require 'logger'
 require 'yapra'
 require 'yapra/pipeline'
 require 'yapra/config'
@@ -20,6 +21,9 @@ require 'yapra/inflector'
 class Yapra::Runtime
   attr_reader :env
   attr_reader :legacy_plugin_registry_factory
+  attr_reader :current_pipeline
+  
+  @@logger = Logger.new(STDOUT)
   
   def initialize env={}, legacy_plugin_registry_factory=nil
     @env    = env
@@ -37,9 +41,11 @@ class Yapra::Runtime
   # execute one pipeline.
   def execute_pipeline pipeline_name, command_array, data=[]
     self.class.logger.info("# pipeline '#{pipeline_name}' is started...")
-    pipeline = Yapra::Pipeline.new(self, pipeline_name)
+    pipeline = Yapra::Pipeline.new(pipeline_name, self)
+    @current_pipeline = pipeline
     legacy_plugin_registory = legacy_plugin_registry_factory.create(pipeline) if legacy_plugin_registry_factory
     pipeline.run(command_array, data)
+    @current_pipeline = nil
   end
   
   def self.logger

data/lib/yapra/version.rb

@@ -2,7 +2,7 @@ module Yapra
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 1
-    TINY  = 1
+    TINY  = 2
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/yapra.rb

@@ -30,11 +30,7 @@ module Yapra
   # TODO create util class, and move this method.
   def load_class_constant module_name
     require Yapra::Inflector.underscore(module_name)
-    Yapra::Inflector.constantize(module_name)
-  rescue LoadError
-    nil
-  rescue NameError
-    nil
+    return Yapra::Inflector.constantize(module_name)
   end
 end
 require 'rss/1.0'

data/lib-plugins/yapra/plugin/feed/load.rb

@@ -19,14 +19,14 @@ module Yapra::Plugin::Feed
           [ config['url'] ]
         end
       
-      urls.each.do |url|
+      urls.each do |url|
         logger.debug("Process: #{url}")
         source = agent.get(url).body
         rss = nil
         begin
-          rss = RSS::Parser.parse(cont)
+          rss = RSS::Parser.parse(source)
         rescue
-          rss = RSS::Parser.parse(cont, false)
+          rss = RSS::Parser.parse(source, false)
         end
         rss.items.each do |item|
           data << item

data/lib-plugins/yapra/plugin/filter/deduped.rb

@@ -0,0 +1,57 @@
+require 'yapra/plugin/base'
+require 'fileutils'
+require 'pathname'
+require 'digest/md5'
+
+module Yapra::Plugin::Filter
+  # Filter::Deduped - Plugin to get Deduped entries -- original by emergent
+  # 
+  # Plugin to get Deduped entries
+  # Cache path can be set.
+  #
+  # - module: Filter::Deduped
+  #   config:
+  #    path: /tmp/cache/hoge
+  #
+  class Deduped < Yapra::Plugin::Base
+    def run(data)
+      cacheroot = Pathname.new(File.join(File.dirname(__FILE__), '..', '..', '..', '..', 'cache'))
+      cachepath = Pathname.new(config['path']) || cacheroot
+      if cachepath.relative?
+        cachepath = cacheroot + cachepath
+      end
+      
+      unless File.exists?(cachepath)
+        FileUtils.mkdir_p(cachepath)
+      end
+      
+      attribute = config['attribute']
+      
+      @cache_paths = []
+      deduped_data = data.select {|d|
+        v = d.link rescue d.to_s
+        if attribute && d.respond_to?(attribute)
+          v = d.__send__(attribute).to_s
+        end
+        hashpath = File.join(cachepath.to_s, Digest::MD5.hexdigest(v))
+        if File.exists?(hashpath)
+          false
+        else
+          begin
+            File.open(hashpath, "wb").write(v)
+            @cache_paths << hashpath
+            true
+          rescue 
+            false
+          end
+        end
+      }
+      return deduped_data
+    end
+    
+    def on_error(ex)
+      logger.debug('error is occured.')
+      FileUtils.rm(@cache_paths, {:force => true})
+    end
+  end
+end

data/lib-plugins/yapra/plugin/test/raise_error.rb

@@ -0,0 +1,10 @@
+require 'yapra/plugin/base'
+
+module Yapra::Plugin::Test
+  class RaiseError < Yapra::Plugin::Base
+    def run(data)
+      message = config['message'] || 'Error.'
+      raise message
+    end
+  end
+end

data/lib-plugins/yapra/plugin/test/test.rb

@@ -0,0 +1,14 @@
+require 'yapra/plugin'
+
+module Yapra::Plugin::Test
+  class Test
+    def run(data)
+      puts 'test!!'
+      data
+    end
+    
+    def on_error(ex)
+      puts 'on error!!'
+    end
+  end
+end

data/lib-plugins/yapra/plugin/test/test2.rb

@@ -0,0 +1,14 @@
+require 'yapra/plugin/base'
+
+module Yapra::Plugin::Test
+  class Test2 < Yapra::Plugin::Base
+    def run(data)
+      logger.debug 'test2!!'
+      data
+    end
+    
+    def on_error(ex)
+      logger.debug 'on error2!!'
+    end
+  end
+end

data/plugins/Filter/deduped.rb

@@ -24,7 +24,7 @@ end
 
 def deduped config, data
   cacheroot = Pathname(__FILE__).parent.parent.parent.realpath + 'cache'
-  cachepath = Pathname.new(config['path']) || root
+  cachepath = Pathname.new(config['path']) || cacheroot
   if cachepath.relative?
     cachepath = cacheroot + cachepath
   end

data/spec/yapra/pipeline_spec.rb

@@ -0,0 +1,63 @@
+require File.dirname(__FILE__) + '/../spec_helper.rb'
+require 'yapra/pipeline'
+
+describe Yapra::Pipeline do
+  before do
+    require 'yapra/plugin/test/test'
+    @pipeline     = Yapra::Pipeline.new('spec test')
+    @test_plugin  = Yapra::Plugin::Test::Test.new
+  end
+  
+  it 'execute pipeline from commands.' do
+    @pipeline.run([
+      {
+        'module' => 'Test::Test'
+      }
+    ])
+  end
+  
+  it 'call run method of plugin, when pipeline is running.' do
+    data = []
+    Yapra::Plugin::Test::Test.should_receive(:new).and_return(@test_plugin)
+    @test_plugin.should_receive(:run).with(data).and_return([])
+    @pipeline.run([
+      {
+        'module' => 'Test::Test'
+      }
+    ], data)
+  end
+  
+  it 'transfer previous plugin return object to next plugin.' do
+    require 'yapra/plugin/test/test2'
+    @test_plugin2  = Yapra::Plugin::Test::Test2.new
+    data = []
+    modified_data = [ 'modified' ]
+    Yapra::Plugin::Test::Test.should_receive(:new).and_return(@test_plugin)
+    @test_plugin.should_receive(:run).with(data).and_return(modified_data)
+    Yapra::Plugin::Test::Test2.should_receive(:new).and_return(@test_plugin2)
+    @test_plugin2.should_receive(:run).with(modified_data)
+    @pipeline.run([
+      {
+        'module' => 'Test::Test'
+      },
+      {
+        'module' => 'Test::Test2'
+      }
+    ], data)
+  end
+  
+  it 'call on_error method of plugin, when Error has occured.' do
+    Yapra::Plugin::Test::Test.should_receive(:new).and_return(@test_plugin)
+    @test_plugin.should_receive(:on_error)
+    lambda {
+      @pipeline.run([
+        {
+          'module' => 'Test::Test'
+        },
+        {
+          'module' => 'Test::RaiseError'
+        }
+      ])
+    }.should raise_error
+  end
+end

data/spec/yapra_spec.rb

@@ -9,7 +9,9 @@ describe Yapra do
   end
   
   it 'can\'t load constant from invalid name.' do
-    Yapra.load_class_constant('_arheiuhri_333***').should be_nil
+    lambda {
+      Yapra.load_class_constant('_arheiuhri_333***').should be_nil
+    }.should raise_error(LoadError)
   end
   
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: yapra
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - Yuanying
@@ -9,10 +9,19 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-06-26 00:00:00 +09:00
+date: 2008-10-14 00:00:00 +09:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.7.0
+    version: 
 description: Yet another pragger implementation.
 email: 
 - yuanying at fraction dot jp
@@ -94,12 +103,16 @@ files:
 - lib-plugins/yapra/plugin/feed/custom.rb
 - lib-plugins/yapra/plugin/feed/load.rb
 - lib-plugins/yapra/plugin/filter/apply_template.rb
+- lib-plugins/yapra/plugin/filter/deduped.rb
 - lib-plugins/yapra/plugin/filter/entry_full_text.rb
 - lib-plugins/yapra/plugin/publish/file_download.rb
 - lib-plugins/yapra/plugin/publish/gmail.rb
 - lib-plugins/yapra/plugin/publish/imap.rb
 - lib-plugins/yapra/plugin/publish/on_memory_download.rb
 - lib-plugins/yapra/plugin/test/append_entry.rb
+- lib-plugins/yapra/plugin/test/raise_error.rb
+- lib-plugins/yapra/plugin/test/test.rb
+- lib-plugins/yapra/plugin/test/test2.rb
 - lib/yapra.rb
 - lib/yapra/config.rb
 - lib/yapra/inflector.rb
@@ -131,6 +144,7 @@ files:
 - spec/yapra/config_spec.rb
 - spec/yapra/legacy_plugin/base_spec.rb
 - spec/yapra/legacy_plugin/registry_factory_spec.rb
+- spec/yapra/pipeline_spec.rb
 - spec/yapra_spec.rb
 - tasks/deployment.rake
 - tasks/environment.rake