automatic 13.1.0 → 13.2.0

data/automatic.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "automatic"
-  s.version = "13.1.0"
+  s.version = "13.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["id774"]
-  s.date = "2013-01-08"
+  s.date = "2013-02-13"
   s.description = "Ruby General Automation Framework"
   s.email = "idnanashi@gmail.com"
   s.executables = ["automatic", "automatic-config"]
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
     "README.md",
     "Rakefile",
     "VERSION",
+    "assets/siteinfo/items_all.json",
     "automatic.gemspec",
     "bin/automatic",
     "bin/automatic-config",
@@ -42,15 +43,17 @@ Gem::Specification.new do |s|
     "lib/automatic/recipe.rb",
     "plugins/custom_feed/svn_log.rb",
     "plugins/filter/absolute_uri.rb",
+    "plugins/filter/full_feed.rb",
     "plugins/filter/ignore.rb",
     "plugins/filter/image.rb",
     "plugins/filter/image_source.rb",
-    "plugins/filter/reverse.rb",
+    "plugins/filter/sort.rb",
     "plugins/filter/tumblr_resize.rb",
     "plugins/notify/ikachan.rb",
     "plugins/publish/console.rb",
     "plugins/publish/google_calendar.rb",
     "plugins/publish/hatena_bookmark.rb",
+    "plugins/publish/instapaper.rb",
     "plugins/publish/mail.rb",
     "plugins/publish/smtp.rb",
     "plugins/store/database.rb",
@@ -58,6 +61,7 @@ Gem::Specification.new do |s|
     "plugins/store/permalink.rb",
     "plugins/store/target_link.rb",
     "plugins/subscription/feed.rb",
+    "plugins/subscription/google_reader_star.rb",
     "plugins/subscription/link.rb",
     "plugins/subscription/tumblr.rb",
     "plugins/subscription/twitter.rb",
@@ -69,35 +73,42 @@ Gem::Specification.new do |s|
     "spec/lib/automatic_spec.rb",
     "spec/plugins/custom_feed/svn_log_spec.rb",
     "spec/plugins/filter/absolute_uri_spec.rb",
+    "spec/plugins/filter/full_feed_spec.rb",
     "spec/plugins/filter/ignore_spec.rb",
     "spec/plugins/filter/image_source_spec.rb",
     "spec/plugins/filter/image_spec.rb",
-    "spec/plugins/filter/reverse_spec.rb",
+    "spec/plugins/filter/sort_spec.rb",
     "spec/plugins/filter/tumblr_resize_spec.rb",
     "spec/plugins/notify/ikachan_spec.rb",
     "spec/plugins/publish/console_spec.rb",
     "spec/plugins/publish/google_calendar_spec.rb",
     "spec/plugins/publish/hatena_bookmark_spec.rb",
+    "spec/plugins/publish/instapaper_spec.rb",
     "spec/plugins/publish/mail_spec.rb",
     "spec/plugins/publish/smtp_spec.rb",
     "spec/plugins/store/full_text_spec.rb",
     "spec/plugins/store/permalink_spec.rb",
     "spec/plugins/store/target_link_spec.rb",
     "spec/plugins/subscription/feed_spec.rb",
+    "spec/plugins/subscription/google_reader_star_spec.rb",
     "spec/plugins/subscription/link_spec.rb",
     "spec/plugins/subscription/tumblr_spec.rb",
     "spec/plugins/subscription/twitter_spec.rb",
     "spec/spec_helper.rb",
     "spec/user_dir/plugins/store/mock.rb",
     "test/fixtures/sampleOPML.xml",
+    "test/integration/test_absoluteurl.yml",
     "test/integration/test_activerecord.yml",
     "test/integration/test_fulltext.yml",
     "test/integration/test_googlealert.yml",
+    "test/integration/test_googlestar.yml",
     "test/integration/test_hatenabookmark.yml",
     "test/integration/test_ignore.yml",
     "test/integration/test_ignore2.yml",
     "test/integration/test_image2local.yml",
-    "test/integration/test_reverse.yml",
+    "test/integration/test_instapaper.yml",
+    "test/integration/test_link2local.yml",
+    "test/integration/test_sort.yml",
     "test/integration/test_svnlog.yml",
     "test/integration/test_tumblr2local.yml",
     "vendor/.gitkeep"

data/bin/automatic

@@ -3,7 +3,7 @@
 # Name::      Automatic::Ruby
 # Author::    774 <http://id774.net>
 # Created::   Feb 18, 2012
-# Updated::   Mar 16, 2012
+# Updated::   Jan 22, 2013
 # Copyright:: 774 Copyright (c) 2012
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -27,8 +27,8 @@ parser = OptionParser.new { |parser|
 }
 
 begin
+  Automatic::Log.puts("info", "Loading #{recipe_path}") unless recipe_path.empty?
   parser.parse!
-  puts "Loading #{recipe_path}" unless recipe_path.empty?
 rescue OptionParser::ParseError => err
   $stderr.puts err.message
   $stderr.puts parser.help

data/bin/automatic-config

@@ -4,8 +4,8 @@
 # Author::    kzgs
 #             774 <http://id774.net>
 # Created::   Mar 11, 2012
-# Updated::   Dec 20, 2012
-# Copyright:: kzgs Copyright (c) 2012
+# Updated::   Jan 24, 2013
+# Copyright:: kzgs Copyright (c) 2012-2013
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
 root_dir = File.expand_path("..", File.dirname(__FILE__))
@@ -36,6 +36,12 @@ subparsers = {
         puts "Creating #{dir}"
       end
     }
+    dir = (File.expand_path('~/.automatic/assets'))
+    unless File.exist?(dir)
+      FileUtils.mkdir_p(dir)
+      FileUtils.cp_r(root_dir + '/assets/siteinfo', dir + '/siteinfo')
+      puts "Creating #{dir}"
+    end
     dir = (File.expand_path('~/.automatic/config'))
     unless File.exist?(dir)
       FileUtils.mkdir_p(dir)

data/doc/ChangeLog

@@ -1,3 +1,22 @@
+=== 13.2.0 / 2013-02-13
+
+* All subscription and download plugins implemented retry function.
+
+* Plugin Filter::Ignore has become item-based filter.
+
+* Plugin Filter::Reverse renamed to Filter::Sort.
+
+* Added new plugins.
+
+  * Subscription::GoogleReaderStar
+  * Filter::FullFeed
+  * Publish::Instapaper
+
+* Improvement tests.
+
+* And other minor bug fix.
+
+
 === 13.1.0 / 2013-01-08
 
 * Fix interval bugs.

data/doc/PLUGINS

@@ -16,6 +16,8 @@ SubscriptionFeed
       feeds:
         - FEED
         - FEED ...
+      retry: RETRY_COUNT
+      interval: INTERVAL_FOR_RETRY (in seconds.)
 
 
 SubscriptionLink
@@ -33,54 +35,76 @@ SubscriptionLink
         - URL
         - URL ...
       interval: INTERVAL_FOR_SCRAPING (in seconds.)
+      retry: RETRY_COUNT
 
 
-SubscriptionTwitter
--------------------
+SubscriptionTumblr
+------------------
 [Path]
- /plugins/subscription/twitter.rb
+ /plugins/subscription/tumblr.rb
 
 [Abstract]
- Subscribe twitter and extract there statuses and the links.
+ Subscribe tumblr and extract the links.
+ If pages specified, including past pages.
 
 [Syntax]
-  - module: SubscriptionTwittter
+  - module: SubscriptionTumblr
     config:
       urls:
         - URL
         - URL ...
+      pages: PAGES
       interval: INTERVAL_FOR_SCRAPING (in seconds.)
+      retry: RETRY_COUNT
 
 
-SubscriptionTumblr
-------------------
+SubscriptionGoogleReaderStar
+----------------------------
 [Path]
- /plugins/subscription/tumblr.rb
+ /plugins/subscription/google_reader_star.rb
 
 [Abstract]
- Subscribe tumblr and extract the links.
- If pages specified, including past pages.
+ Subscribe to Google reader star feed.
 
 [Syntax]
-  - module: SubscriptionTumblr
+  - module: SubscriptionGoogleReaderStar
+    config:
+      feeds:
+        - FEED
+        - FEED ...
+
+
+SubscriptionTwitter
+-------------------
+[Path]
+ /plugins/subscription/twitter.rb
+
+[Abstract]
+ Subscribe twitter and extract there statuses and the links.
+
+[Syntax]
+  - module: SubscriptionTwittter
     config:
       urls:
         - URL
         - URL ...
-      pages: PAGES
       interval: INTERVAL_FOR_SCRAPING (in seconds.)
+      retry: RETRY_COUNT
 
 
-FilterReverse
--------------
+FilterSort
+----------
 [Path]
- /plugins/filter/reverse.rb
+ /plugins/filter/sort.rb
 
 [Abstract]
- Sort feed by date asc.
+ Sort feed by date asc or desc.
+ If desc is specified, it's sort feed by date desc.
 
 [Syntax]
-  - module: FilterReverse
+  - module: FilterSort
+    config:
+      sort: asc or desc
 
 
 FilterIgnore
@@ -94,7 +118,10 @@ FilterIgnore
 [Syntax]
   - module: FilterIgnore
     config:
-      exclude:
+      link:
+        - KEYWORD
+        - KEYWORD ...
+      description:
         - KEYWORD
         - KEYWORD ...
 
@@ -125,6 +152,20 @@ FilterImageSource
   - module: FilterImageSource
 
 
+FilterFullFeed
+--------------
+[Path]
+ /plugins/filter/full_feed.rb
+
+[Abstract]
+ Get full text of feed entry.
+
+[Syntax]
+  - module: FilterFeed
+    config:
+      siteinfo: your siteinfo file's path
+
+
 FilterAbsoluteURI
 -----------------
 [Path]
@@ -200,6 +241,7 @@ StoreTargetLink
   - Module: StoreTargetLink
     config:
       path: SAVE_TO_PATH
+      retry: RETRY_COUNT
       interval: INTERVAL_FOR_DOWNLOAD (in seconds.)
 
 
@@ -235,6 +277,25 @@ PublishHatenaBookmark
       interval: INTERVAL_FOR_BOOKMARK (in seconds.)
 
 
+PublishInstapaper
+-----------------
+[Path]
+ /plugins/publish/instapaper.rb
+
+[Abstract]
+ Register an appointment to Instapaper.
+
+[Description]
+ Submission to Instapaper API.
+
+[Syntax]
+  - module: PublishInstapaper
+    config:
+      email: INSTAPAPER_ACCOUNT
+      password: PASSWORD
+      interval: INTERVAL
+
+
 PublishGoogleCalendar
 ---------------------
 [Path]

data/doc/PLUGINS.ja

@@ -16,6 +16,8 @@ SubscriptionFeed
       feeds:
         - フィード名
         - フィード名 ...
+      retry: エラー時のリトライ回数 (回数, 省略時 0)
+      interval: エラー時のリトライ間隔 (秒数, 省略可)
 
 
 SubscriptionLink
@@ -33,54 +35,77 @@ SubscriptionLink
         - URL
         - URL ...
       interval: スクレイピングの間隔 (秒数, 省略可)
+      retry: エラー時のリトライ回数 (回数, 省略時 0)
 
 
-SubscriptionTwitter
--------------------
+SubscriptionTumblr
+------------------
 [パス]
- /plugins/subscription/link.rb
+ /plugins/subscription/tumblr.rb
 
 [概要]
- Twitter を購読し、発言とそのリンクを抽出する
+ Tumblr を購読し、リンクを抽出する
+ pages が指定された場合、過去のページも遡って購読する
 
 [レシピ記法]
-  - module: SubscriptionTwitter
+  - module: SubscriptionTumblr
     config:
       urls:
         - URL
         - URL ...
+      pages: PAGES
       interval: スクレイピングの間隔 (秒数, 省略可)
+      retry: エラー時のリトライ回数 (回数, 省略時 0)
 
 
-SubscriptionTumblr
-------------------
+SubscriptionGoogleReaderStar
+----------------------------
 [パス]
- /plugins/subscription/tumblr.rb
+ /plugins/subscription/google_reader_star.rb
 
 [概要]
- Tumblr を購読し、リンクを抽出する
- pages が指定された場合、過去のページも遡って購読する
+ Google リーダーのスターフィードを購読する
 
 [レシピ記法]
-  - module: SubscriptionTumblr
+  - module: SubscriptionGoogleReaderStar
+    config:
+      feeds:
+        - フィード名
+        - フィード名 ...
+
+
+SubscriptionTwitter
+-------------------
+[パス]
+ /plugins/subscription/link.rb
+
+[概要]
+ Twitter を購読し、発言とそのリンクを抽出する
+
+[レシピ記法]
+  - module: SubscriptionTwitter
     config:
       urls:
         - URL
         - URL ...
-      pages: PAGES
       interval: スクレイピングの間隔 (秒数, 省略可)
+      retry: エラー時のリトライ回数 (回数, 省略時 0)
 
 
-FilterReverse
--------------
+FilterSort
+----------
 [パス]
- /plugins/filter/reverse.rb
+ /plugins/filter/sort.rb
 
 [概要]
  日付の昇順にソートする
+ もし config で sort に desc が指定されると
+ 昇順ではなく降順にソートする
 
 [レシピ記法]
-  - module: FilterReverse
+  - module: FilterSort
+    config:
+      sort: asc or desc
 
 
 FilterIgnore
@@ -94,7 +119,10 @@ FilterIgnore
 [レシピ記法]
   - module: FilterIgnore
     config:
-      exclude:
+      link:
+        - 無視キーワード
+        - 無視キーワード ...
+      description:
         - 無視キーワード
         - 無視キーワード ...
 
@@ -125,6 +153,20 @@ FilterImageSource
   - module: FilterImageSource
 
 
+FilterFullFeed
+--------------
+[パス]
+ /plugins/filter/full_feed.rb
+
+[概要]
+ フィードの全文を取得する
+
+[レシピ記法]
+  - module: FilterFeed
+    config:
+      siteinfo: siteinfo ファイル名
+
+
 FilterAbsoluteURI
 -----------------
 [パス]
@@ -199,6 +241,7 @@ StoreTargetLink
   - module: StoreTargetLink
     config:
       path: 保存先のフォルダ
+      retry: エラー時のリトライ回数 (回数, 省略時 0)
       interval: 保存する間隔 (秒)
 
 
@@ -234,6 +277,25 @@ PublishHatenaBookmark
       interval: はてブする間隔 (秒)
 
 
+PublishInstapaper
+-----------------
+[パス]
+ /plugins/publish/instapaper.rb
+
+[概要]
+ Instapaper に登録する
+
+[説明]
+ Instapaper API へ送信する
+
+[レシピ記法]
+  - module: PublishInstapaper
+    config:
+      email: Instapaper メールアドレス
+      password: パスワード
+      interval: 複数の予定の投稿間隔 (秒)
+
+
 PublishGoogleCalendar
 ---------------------
 [パス]

data/doc/README

@@ -257,6 +257,11 @@ Directory and file structure
 |
 | If ~/.automatic/db not exist, save db file here.
 |
++-- assets
+|
+| There are some files that needed in a plug-ins.
+| If ~/.automatic/assets exist, takes precedence it.
+|
 +-+ script
 | |
 | +-- build
@@ -446,6 +451,8 @@ $ automatic-config
 
 $ automatic-config scaffold
  Make ~/.automatic directory in your home directory.
+ This operation makes 'plugins', 'db', 'config', 'assets' directories.
+ These takes precedence the origin directory.
 
 $ automatic-config unscaffold
  Delete ~/.automatic directory.
@@ -485,7 +492,7 @@ Update History
 TODO
 ====
 
- Refer to the issues of github
+ Refer to the issues of github.
  https://github.com/id774/automaticruby/issues
 
 
@@ -493,21 +500,9 @@ TODO
 Environment
 ===========
 
- ruby 1.8 or later
-
-
-The following packages depend on the gem.
-
-sqlite3
-activesupport
-hashie
-activerecord
-gcalapi
-xml-simple
-feedbag
-nokogiri
+ Ruby 1.8 - 2.0.
 
-For more information, see Gemfile.
+ Depend on the packages that described in Gemfile.
 
 
 =====

data/doc/README.ja

@@ -234,10 +234,14 @@ plugins:
 |            レシピを読み込む
 |
 +- db
-|
 |   ~/.automatic/db が無い場合、ここにデータベースが
 |   保存される
 |
++- assets
+|   json ファイルなどプラグインの中で適宜必要とされる
+|   ファイルが格納される
+|   ~/.automatic/assets がある場合はそちらが優先される
+|
 +- script
 |   |
 |   +- build
@@ -454,6 +458,9 @@ $ automatic-config
 
 $ automatic-config scaffold
  ホームディレクトリに ~/.automatic ディレクトリを生成する。
+ この操作により ~/.automatic の下に plugins, db, config, assets
+ディレクトリが生成される。
+ これらは本体のそれぞれのディレクトリよりも優先される。
 
 $ automatic-config unscaffold
  ホームディレクトリの ~/.automatic ディレクトリを削除する。
@@ -501,21 +508,9 @@ TODO
 動作環境
 ========
 
-Ruby 1.8 以降
-
-
-以下の gem パッケージに依存
-
-sqlite3
-activesupport
-hashie
-activerecord
-gcalapi
-xml-simple
-feedbag
-nokogiri
+Ruby 1.8 - 2.0
 
-詳細は Gemfile を参照
+Gemfile に記述された gem パッケージに依存
 
 
 ========

data/lib/automatic.rb

@@ -2,7 +2,7 @@
 # Name::      Automatic::Ruby
 # Author::    774 <http://id774.net>
 # Created::   Feb 18, 2012
-# Updated::   Jan  8, 2013
+# Updated::   Feb 13, 2013
 # Copyright:: 774 Copyright (c) 2012
 # License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
 
@@ -13,7 +13,7 @@ module Automatic
   require 'automatic/log'
   require 'automatic/feed_parser'
 
-  VERSION = "13.1.0"
+  VERSION = "13.2.0"
   USER_DIR = "/.automatic"
 
   class << self

data/plugins/filter/full_feed.rb

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+# Name::      Automatic::Plugin::Filter::FullFeed
+# Author::    progd <http://d.hatena.ne.jp/progd/20120429/automatic_ruby_filter_full_feed>
+#             774 <http://id774.net>
+# Created::   Apr 29, 2012
+# Updated::   Jan 24, 2013
+# Copyright:: progd
+#             774 Copyright (c) 2012-2013
+# License::   Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
+
+module Automatic::Plugin
+
+  class FilterFullFeed
+    require 'nokogiri'
+
+    SITEINFO_TYPES = %w[SBM INDIVIDUAL IND SUBGENERAL SUB GENERAL GEN]
+
+    def initialize(config, pipeline=[])
+      @config = config
+      @pipeline = pipeline
+      @siteinfo = get_siteinfo
+    end
+
+    def get_siteinfo
+      Automatic::Log.puts(:info, "Loading siteinfo from #{@config['siteinfo']}")
+      siteinfo = JSON.load(open(File.join(assets_dir, @config['siteinfo'])).read)
+      siteinfo.select! { |info| SITEINFO_TYPES.include? (info['data']['type']) }
+      siteinfo.sort! { |a, b|
+        atype, btype = a['data']['type'], b['data']['type']
+        SITEINFO_TYPES.index(atype) <=> SITEINFO_TYPES.index(btype)
+      }
+      return siteinfo
+    end
+
+    def assets_dir
+      dir = (File.expand_path('~/.automatic/assets/siteinfo'))
+      if File.directory?(dir)
+        dir
+      else
+        File.join(File.dirname(__FILE__), '..', '..', 'assets', 'siteinfo')
+      end
+    end
+
+    def fulltext(feed)
+      return feed unless feed.link
+      @siteinfo.each { |info|
+        begin
+          if feed.link.match(info['data']['url'])
+            Automatic::Log.puts(:info, "Siteinfo matched: #{info['data']['url']}")
+            html = Nokogiri::HTML.parse(open(feed.link))
+            body = html.xpath(info['data']['xpath'])
+            feed.description = body.to_html.encode('UTF-8', :undef => :replace)
+            return feed
+          end
+        rescue
+          return feed
+        end
+      }
+      Automatic::Log.puts(:info, "Fulltext SITEINFO not found: #{feed.link}")
+      return feed
+    end
+
+    def run
+      @return_feeds = []
+      @pipeline.each {|feeds|
+        unless feeds.nil?
+          feeds.items.each {|feed|
+            feed = fulltext(feed)
+          }
+        end
+        @return_feeds << feeds
+      }
+      @return_feeds
+    end
+  end
+end