yapra 0.1.0

data/lib-plugins/yapra/plugin/feed/load.rb

@@ -0,0 +1,29 @@
+require 'yapra/plugin/mechanize_base'
+
+module Yapra::Plugin::Feed
+  class Load < Yapra::Plugin::MechanizeBase
+    def run(data)
+      urls = 
+        if config['url'].kind_of?(Array)
+          config['url']
+        else
+          [ config['url'] ]
+        end
+      
+      urls.each.do |url|
+        source = agent.get(url).body
+        rss = nil
+        begin
+          rss = RSS::Parser.parse(cont)
+        rescue
+          rss = RSS::Parser.parse(cont, false)
+        end
+        rss.items.each do |item|
+          data << item
+        end
+      end
+      
+      data
+    end
+  end
+end

data/lib-plugins/yapra/plugin/filter/entry_full_text.rb

@@ -0,0 +1,59 @@
+## Filter::EntryFullText -- Yuanying
+##
+## get the entry full text from page with WWW::Mechanize.
+## 
+## - module: Filter::EntryFullText
+##   config:
+##     regexp: http://www\.pixiv\.net/*
+##     extract_xpath:
+##       title: '//title/text()'
+##       dc_creator: "//div[@id='profile']/div/text()"
+##       author: "//div[@id='profile']/div/text()"
+##       description: "//div[@id='content2']"
+##     apply_template_after_extracted:
+##       content_encoded: '<div><%= title %></div>'
+##
+require 'yapra/plugin/mechanize_base'
+
+module Yapra::Plugin::Filter
+  class EntryFullText < Yapra::Plugin::MechanizeBase
+    def run(data)
+      regexp = nil
+      if config['regexp']
+        regexp = Regexp.new(config['regexp'])
+      else
+        regexp = /^(https?|ftp)(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)$/
+      end
+      
+      wait = config['wait'] || 1
+      
+      data.map! do |item|
+        url = item
+        if item.respond_to?('link')
+          url = item.link
+        end
+
+        if regexp =~ url
+          page = agent.get(url)
+          sleep wait
+          logger.info "Process: #{url}"
+
+          unless(item.instance_of?(RSS::RDF::Item))
+            new_item = RSS::RDF::Item.new
+            new_item.title = item.title rescue item.to_s
+            new_item.date = item.date rescue Time.now
+            new_item.description = item.description rescue item.to_s
+            new_item.link = item.link rescue '#'
+            item = new_item
+          end
+
+          extract_attribute_from page.root, item
+
+        end
+        item
+      end
+      
+      data
+    end
+  end
+end

data/lib-plugins/yapra/plugin/publish/file_download.rb

@@ -0,0 +1,109 @@
+## Publish::FileDownload -- Yuanying
+##
+## download file with WWW::Mechanize.
+## 
+## example: 
+## 
+## - module: Publish::FileDownload
+##   config:
+##     regexp: http://www\.yahoo\.co\.jp/*
+##     dir: '/Users/yuanying/Downloads/'
+##     auto_suffix: true
+##     before_hook: "agent.get('http://www.yahoo.co.jp/'); sleep 1"
+##     url:
+##       attribute: link
+##       replace: index(\d*?)\.html
+##       to: file\1.zip
+##     filename:
+##       attribute: title
+##       replace: 'Yahoo'
+##       to: ''
+##     referrer:
+##       attribute: link
+##       replace: 'zip'
+##       to: 'html'
+##
+require 'yapra/plugin/mechanize_base'
+
+module Yapra::Plugin::Publish
+  class FileDownload < Yapra::Plugin::MechanizeBase
+    def run(data)
+      regexp = nil
+      if config['regexp']
+        regexp = Regexp.new(config['regexp'])
+      else
+        regexp = /^(https?|ftp)(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)$/
+      end
+      
+      wait = config['wait'] || 3
+
+      data.each do |item|
+        url = construct_data(config['url'], item, item.respond_to?('link') ? item.link : item)
+
+        if regexp =~ url
+          referrer = construct_data(config['referrer'], item)
+          download(item, url, referrer)
+          sleep wait
+        end
+      end
+      
+      data
+    end
+    
+    protected
+    def construct_data(config, item, original=nil)
+      if config && config['attribute']
+        if item.respond_to?(config['attribute'])
+          original = item.__send__(config['attribute'])
+        end
+      elsif config
+        original = item
+      end
+
+      if original && config && config['replace']
+        original = original.gsub(config['replace'], config['to'] || Time.now.to_s)
+      end
+
+      original
+    end
+    
+    def discover_extensions page
+      require 'mime/types'
+      types = MIME::Types[page.content_type]
+      if types.size > 0 && types[0].extensions.size > 0
+        return type[0].extensions[0]
+      else
+        logger.info 'suitable extention is not found.'
+        return nil
+      end
+    rescue LoadError => ex
+      logger.warn 'require mime-types is failed.'
+    end
+    
+    def download(item, url, referrer)
+      dir = config['dir']
+      filename = construct_data(config['filename'], item)
+      
+      if config['before_hook']
+        eval(config['before_hook'])
+      end
+      page = agent.get(url, referrer)
+      logger.info "Download: #{url}"
+      if config['after_hook']
+        eval(config['after_hook'])
+      end
+      
+      filename = page.filename unless filename
+      
+      if config['auto_suffix']
+        ext = discover_extensions(page)
+        filename = "#{filename}.#{ext}" if ext
+      end
+
+      path = File.join(dir, filename)
+      open(path, 'w') do |io|
+        io.write page.body
+      end
+    end
+  end
+end

data/lib-plugins/yapra/plugin/publish/gmail.rb

@@ -0,0 +1,27 @@
+# module: Publish::Gmail -- Yuanying
+#
+# publish entry to imap mail.
+#
+# example:
+#
+# - module: Publish::Gmail
+#   config:
+#     username: username
+#     password: password
+#     wait: 1
+#     mail:
+#       subject_prefix: '[Yapra]'
+#       from: 'test@example.com'
+#       to: 'test2@example.com'
+#
+require 'net/imap'
+require 'yapra/plugin/publish/imap'
+
+module Yapra::Plugin::Publish
+  class Gmail < Yapra::Plugin::Publish::Imap
+    protected
+    def create_imap server, port, usessl
+      Net::IMAP.new('imap.gmail.com', 993, true)
+    end
+  end
+end

data/lib-plugins/yapra/plugin/publish/imap.rb

@@ -0,0 +1,82 @@
+# module: Publish::Imap -- Yuanying
+#
+# publish entry to imap mail.
+#
+# example:
+#
+# - module: Publish::Imap
+#   config:
+#     username: username
+#     password: password
+#     imap_server: imap.gmail.com
+#     port: 993
+#     ssl: on
+#     wait: 1
+#     mail:
+#       subject_prefix: '[Yapra]'
+#       from: 'test@example.com'
+#       to: 'test2@example.com'
+#
+require 'net/imap'
+require 'yapra/plugin/base'
+
+module Yapra::Plugin::Publish
+  class Imap < Yapra::Plugin::Base
+    def run(data)
+      username  = config['username']
+      password  = config['password']
+      server    = config['imap_server'] || 'imap.gmail.com'
+      port      = config['port'] || 993
+      usessl    = ('off' != config['ssl'])
+      mailbox   = config['mailbox'] || 'inbox'
+      wait      = config['wait'] || 1
+      
+      unless config['mail']
+        config['mail'] = {}
+      end
+      subject_prefix  = config['mail']['subject_prefix'] || ''
+      from            = config['mail']['from'] || 'yapra@localhost'
+      to              = config['mail']['to']   || 'me@localhost'
+      
+      imap = create_imap server, port, usessl
+      logger.info(imap.greeting)
+      
+      imap.login(username, password)
+      imap.examine(mailbox)
+      data.each do |item|
+        date = item.date || item.dc_date || Time.now
+        content = item.content_encoded || item.description || 'from Yapra.'
+        content = [content].pack('m')
+        imap.append("inbox", <<EOF.gsub(/¥n/, "¥r¥n"), nil, date)
+From: #{from}
+To: #{to}
+Date: #{date.rfc2822}
+MIME-Version: 1.0
+Content-type: text/html; charset=UTF-8
+Content-transfer-encoding: base64
+X-Mailer: Yapra 0.1
+Subject: #{encode_field(subject_prefix + item.title)}
+
+#{content}
+EOF
+        sleep wait
+      end
+      imap.disconnect
+      
+      data
+    end
+    
+    protected
+    def create_imap server, port, usessl
+      logger.debug("server: #{server}:#{port}, usessl = #{usessl}")
+      Net::IMAP.new(server, port, usessl)
+    end
+    def encode_field field
+      #field.gsub(/[　-瑤]\S*\s*/) {|x|
+        field.scan(/.{1,10}/).map {|y|
+          "=?UTF-8?B?" + y.to_a.pack('m').chomp + "?="
+        }.join("\n ")
+      #}
+    end
+  end
+end

data/lib-plugins/yapra/plugin/test/append_entry.rb

@@ -0,0 +1,25 @@
+# module: Test::AppendEntry -- Yuanying
+#
+# append entry to data array.
+#
+# example:
+#
+# - module: Test::AppendEntry
+#   config:
+#     title: 'title'
+#     description: 'description.'
+#
+require 'yapra/plugin/base'
+
+module Yapra::Plugin::Test
+  class AppendEntry < Yapra::Plugin::Base
+    def run(data)
+      item = RSS::RDF::Item.new
+      plugin_config.each do |k, v|
+        set_attribute_to item, k, v
+      end
+      data << item
+      data
+    end
+  end
+end

data/plugins/Filter/deduped.rb

@@ -0,0 +1,57 @@
+## Filter::deduped - Plugin to get Deduped entries -- emergent
+## 
+## Plugin to get Deduped entries
+## Cache path can be set.
+##
+## - module: Filter::deduped
+##   config:
+##    path: /tmp/cache/hoge
+##
+require 'pathname'
+require 'digest/md5'
+
+def mkdir_p path
+  begin
+    Dir.mkdir(path)
+  rescue Errno::ENOENT
+    mkdir_p Pathname.new(path).parent
+    retry
+  rescue Errno::EACCES
+    raise
+  end
+  0
+end
+
+def deduped config, data
+  cacheroot = Pathname(__FILE__).parent.parent.parent.realpath + 'cache'
+  cachepath = Pathname.new(config['path']) || root
+  if cachepath.relative?
+    cachepath = cacheroot + cachepath
+  end
+  #puts 'cache path: ' + cachepath
+
+  if !File.exists?(cachepath)
+    begin
+      mkdir_p cachepath
+    rescue
+      STDERR.puts "could'nt make cache directory"
+      return data
+    end
+  end
+  
+  attribute = config['attribute']
+  
+  deduped_data = data.select {|d|
+    v = d.link rescue d.to_s
+    if attribute && d.respond_to?(attribute)
+      v = d.__send__(attribute).to_s
+    end
+    hashpath = cachepath.to_s + '/' + Digest::MD5.hexdigest(v)
+    if File.exists?(hashpath)
+      false
+    else 
+      File.open(hashpath, "wb").write(v) rescue false
+    end
+  }
+  return deduped_data
+end

data/plugins/Filter/sort.rb

@@ -0,0 +1,28 @@
+def create_compare_proc config
+  if(config==nil || config["method"] == nil)
+    return lambda do |a, b|
+      a <=> b
+    end
+  elsif config["method"].kind_of?(Hash)
+    return lambda do |a, b|
+      eval_pragger(config["method"],[a])[0] <=> eval_pragger(config["method"],[b])[0]
+    end
+  else
+    return lambda do |a, b|
+      method = config['method']
+      a = a.respond_to?(method) ? a.__send__(method) : nil
+      b = b.respond_to?(method) ? b.__send__(method) : nil
+      return a <=> b if a.respond_to?('<=>')
+      return b <=> a if b.respond_to?('<=>')
+      nil
+    end
+  end
+end
+
+def sort(config,data)
+  proc = create_compare_proc(config)
+  return data.sort do|a, b|
+    proc.call(a, b)
+  end
+end
+

data/plugins/Filter/subs.rb

@@ -0,0 +1,16 @@
+def subs(config,data)
+  reg = Regexp.new(config["regex"])
+  to = config["to"]
+  attribute = config['attribute']
+  data.map! do |i|
+    if attribute
+      if i.respond_to?(attribute) && i.respond_to?("#{attribute}=")
+        i.__send__("#{attribute}=", i.__send__("#{attribute}").gsub(reg, to))
+      end
+    else
+      i = i.gsub(reg,to)
+    end
+    i
+  end
+  return data
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/yapra.rb'}"
+puts "Loading yapra gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/script/txt2html

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+GEM_NAME = 'yapra' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'yapra'
+
+require 'rubygems'
+begin
+  require 'newgem'
+  require 'rubyforge'
+rescue LoadError
+  puts "\n\nGenerating the website requires the newgem RubyGem"
+  puts "Install: gem install newgem\n\n"
+  exit(1)
+end
+require 'redcloth'
+require 'syntax/convertors/html'
+require 'erb'
+require File.dirname(__FILE__) + "/../lib/#{GEM_NAME}/version.rb"
+
+version  = Yapra::VERSION::STRING
+download = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+def rubyforge_project_id
+  RubyForge.new.autoconfig["group_ids"][RUBYFORGE_PROJECT]
+end
+
+class Fixnum
+  def ordinal
+    # teens
+    return 'th' if (10..19).include?(self % 100)
+    # others
+    case self % 10
+    when 1: return 'st'
+    when 2: return 'nd'
+    when 3: return 'rd'
+    else    return 'th'
+    end
+  end
+end
+
+class Time
+  def pretty
+    return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}"
+  end
+end
+
+def convert_syntax(syntax, source)
+  return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^<pre>|</pre>$!,'')
+end
+
+if ARGV.length >= 1
+  src, template = ARGV
+  template ||= File.join(File.dirname(__FILE__), '/../website/template.html.erb')
+else
+  puts("Usage: #{File.split($0).last} source.txt [template.html.erb] > output.html")
+  exit!
+end
+
+template = ERB.new(File.open(template).read)
+
+title = nil
+body = nil
+File.open(src) do |fsrc|
+  title_text = fsrc.readline
+  body_text_template = fsrc.read
+  body_text = ERB.new(body_text_template).result(binding)
+  syntax_items = []
+  body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)</\1>!m){
+    ident = syntax_items.length
+    element, syntax, source = $1, $2, $3
+    syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
+    "syntax-temp-#{ident}"
+  }
+  title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+  body = RedCloth.new(body_text).to_html
+  body.gsub!(%r!(?:<pre><code>)?syntax-temp-(\d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
+end
+stat = File.stat(src)
+created = stat.ctime
+modified = stat.mtime
+
+$stdout << template.result(binding)