automatic 12.6.0 → 12.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/VERSION +1 -1
  2. data/automatic.gemspec +10 -22
  3. data/bin/automatic-config +4 -4
  4. data/doc/ChangeLog +30 -1
  5. data/doc/PLUGINS +46 -178
  6. data/doc/PLUGINS.ja +46 -177
  7. data/doc/README +2 -10
  8. data/doc/README.ja +2 -9
  9. data/lib/automatic.rb +2 -3
  10. data/plugins/custom_feed/svn_log.rb +0 -1
  11. data/plugins/filter/absolute_uri.rb +39 -0
  12. data/plugins/filter/image.rb +46 -0
  13. data/plugins/filter/reverse.rb +0 -1
  14. data/plugins/store/database.rb +8 -23
  15. data/plugins/store/full_text.rb +4 -4
  16. data/plugins/store/permalink.rb +12 -9
  17. data/plugins/store/target_link.rb +3 -2
  18. data/plugins/subscription/link.rb +55 -0
  19. data/plugins/subscription/twitter.rb +57 -0
  20. data/script/build +1 -1
  21. data/spec/lib/automatic_spec.rb +3 -2
  22. data/spec/plugins/custom_feed/svn_log_spec.rb +1 -1
  23. data/spec/plugins/filter/absolute_uri_spec.rb +61 -0
  24. data/spec/plugins/filter/image_source_spec.rb +1 -1
  25. data/spec/plugins/filter/image_spec.rb +65 -0
  26. data/spec/plugins/publish/google_calendar_spec.rb +1 -1
  27. data/spec/plugins/store/full_text_spec.rb +0 -1
  28. data/spec/plugins/store/permalink_spec.rb +52 -3
  29. data/spec/plugins/store/target_link_spec.rb +1 -2
  30. data/spec/plugins/subscription/feed_spec.rb +0 -1
  31. data/spec/plugins/subscription/{uri_spec.rb → link_spec.rb} +9 -10
  32. data/spec/plugins/subscription/twitter_spec.rb +42 -0
  33. data/spec/spec_helper.rb +12 -26
  34. data/spec/user_dir/plugins/store/mock.rb +1 -1
  35. data/test/integration/test_get_image.yml +4 -6
  36. data/test/integration/test_image2local.yml +1 -1
  37. metadata +10 -22
  38. data/config/html2console.yml +0 -16
  39. data/plugins/extract/link.rb +0 -32
  40. data/plugins/filter/image_link.rb +0 -37
  41. data/plugins/publish/dump.rb +0 -24
  42. data/plugins/store/link.rb +0 -47
  43. data/plugins/store/target.rb +0 -41
  44. data/plugins/subscription/uri.rb +0 -31
  45. data/spec/fixtures/extractLink.html +0 -14
  46. data/spec/fixtures/filterImageLink.html +0 -34
  47. data/spec/fixtures/publishDump.html +0 -14
  48. data/spec/fixtures/storeLink.html +0 -34
  49. data/spec/fixtures/storeLink2.html +0 -36
  50. data/spec/fixtures/storeTarget.html +0 -11
  51. data/spec/fixtures/storeTarget2.html +0 -11
  52. data/spec/plugins/extract/link_spec.rb +0 -38
  53. data/spec/plugins/filter/image_link_spec.rb +0 -51
  54. data/spec/plugins/publish/dump_spec.rb +0 -32
  55. data/spec/plugins/store/link_spec.rb +0 -47
  56. data/spec/plugins/store/target_spec.rb +0 -41
data/config/html2console.yml DELETED
@@ -1,16 +0,0 @@
1
- global:
2
- timezone: Asia/Tokyo
3
- cache:
4
- base: /tmp
5
- log:
6
- level: info
7
-
8
- plugins:
9
- - module: SubscriptionURI
10
- config:
11
- urls:
12
- - http://id774.net/
13
-
14
- - module: ExtractLink
15
-
16
- - module: PublishDump
data/plugins/extract/link.rb DELETED
@@ -1,32 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Extract::Link
3
- # Author:: 774 <http://id774.net>
4
- # Created:: May 24, 2012
5
- # Updated:: Jun 13, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- module Automatic::Plugin
10
- class ExtractLink
11
- require 'nokogiri'
12
-
13
- def initialize(config, pipeline=[])
14
- @config = config
15
- @pipeline = pipeline
16
- end
17
-
18
- def run
19
- @return_html = []
20
- @pipeline.each {|html|
21
- img_url = ""
22
- unless html.nil?
23
- doc = Nokogiri::HTML(html)
24
- (doc/:a).each {|link|
25
- @return_html << link[:href]
26
- }
27
- end
28
- }
29
- @return_html
30
- end
31
- end
32
- end
data/plugins/filter/image_link.rb DELETED
@@ -1,37 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Filter::ImageLink
3
- # Author:: 774 <http://id774.net>
4
- # Created:: May 24, 2012
5
- # Updated:: Jun 13, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- module Automatic::Plugin
10
- class FilterImageLink
11
- def initialize(config, pipeline=[])
12
- @config = config
13
- @pipeline = pipeline
14
- end
15
-
16
- def run
17
- @return_html = []
18
- @pipeline.each {|html|
19
- unless html.nil?
20
- html.scan(/(.*?\.jp.*g$)/i) { |matched|
21
- @return_html << matched.join(" ")
22
- }
23
- html.scan(/(.*?\.png$)/i) { |matched|
24
- @return_html << matched.join(" ")
25
- }
26
- html.scan(/(.*?\.gif$)/i) { |matched|
27
- @return_html << matched.join(" ")
28
- }
29
- html.scan(/(.*?\.tiff$)/i) { |matched|
30
- @return_html << matched.join(" ")
31
- }
32
- end
33
- }
34
- @return_html
35
- end
36
- end
37
- end
data/plugins/publish/dump.rb DELETED
@@ -1,24 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Publish::Dump
3
- # Author:: 774 <http://id774.net>
4
- # Created:: Jun 13, 2012
5
- # Updated:: Jun 13, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- module Automatic::Plugin
10
- class PublishDump
11
- require 'pp'
12
-
13
- def initialize(config, pipeline=[])
14
- @config = config
15
- @pipeline = pipeline
16
- @output = STDOUT
17
- end
18
-
19
- def run
20
- pp(@pipeline, out=@output)
21
- @pipeline
22
- end
23
- end
24
- end
data/plugins/store/link.rb DELETED
@@ -1,47 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Store::Link
3
- # Author:: 774 <http://id774.net>
4
- # Created:: May 24, 2012
5
- # Updated:: Jun 14, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- require 'plugins/store/database'
10
-
11
- module Automatic::Plugin
12
- class Link < ActiveRecord::Base
13
- end
14
-
15
- class StoreLink
16
- include Automatic::Plugin::Database
17
-
18
- def initialize(config, pipeline=[])
19
- @config = config
20
- @pipeline = pipeline
21
- end
22
-
23
- def column_definition
24
- return {
25
- :url => :string,
26
- :created_at => :string
27
- }
28
- end
29
-
30
- def unique_key
31
- return :url
32
- end
33
-
34
- def model_class
35
- return Automatic::Plugin::Link
36
- end
37
-
38
- def run
39
- return for_each_new_link { |link|
40
- Link.create(
41
- :url => link,
42
- :created_at => Time.now.strftime("%Y/%m/%d %X"))
43
- Automatic::Log.puts("info", "Saving: #{link}")
44
- }
45
- end
46
- end
47
- end
data/plugins/store/target.rb DELETED
@@ -1,41 +0,0 @@
1
- #!/usr/bin/env ruby
2
- # -*- coding: utf-8 -*-
3
- # Name:: Automatic::Plugin::Store::Target
4
- # Author:: 774 <http://id774.net>
5
- # Created:: May 24, 2012
6
- # Updated:: May 28, 2012
7
- # Copyright:: 774 Copyright (c) 2012
8
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
9
-
10
- module Automatic::Plugin
11
- class StoreTarget
12
- def initialize(config, pipeline=[])
13
- @config = config
14
- @pipeline = pipeline
15
- end
16
-
17
- def wget(url)
18
- filename = url.split(/\//).last
19
- open(url) { |source|
20
- open(File.join(@config['path'], filename), "w+b") { |o|
21
- o.print source.read
22
- }
23
- }
24
- end
25
-
26
- def run
27
- @pipeline.each {|html|
28
- begin
29
- unless html.nil?
30
- Automatic::Log.puts("info", "Get: #{html}")
31
- wget(html)
32
- end
33
- rescue
34
- Automatic::Log.puts("error", "Error found during file download.")
35
- end
36
- sleep @config['interval'].to_i
37
- }
38
- @pipeline
39
- end
40
- end
41
- end
data/plugins/subscription/uri.rb DELETED
@@ -1,31 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Subscription::URI
3
- # Author:: 774 <http://id774.net>
4
- # Created:: May 24, 2012
5
- # Updated:: Jun 13, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- module Automatic::Plugin
10
- class SubscriptionURI
11
- require 'open-uri'
12
-
13
- def initialize(config, pipeline=[])
14
- @config = config
15
- @pipeline = pipeline
16
- end
17
-
18
- def run
19
- @config['urls'].each {|url|
20
- begin
21
- Automatic::Log.puts("info", "Parsing: #{url}")
22
- doc = open(url).read
23
- @pipeline << doc
24
- rescue
25
- Automatic::Log.puts("error", "Fault in parsing: #{url}")
26
- end
27
- }
28
- return @pipeline
29
- end
30
- end
31
- end
data/spec/fixtures/extractLink.html DELETED
@@ -1,14 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://oh-news.net/live/wp-content/uploads/2011/04/Eila_omote.jpg">
11
- <img src="http://24.media.tumblr.com/tumblr_m5gneyJmsH1qza5ppo1_500.jpg" alt="" /></a>
12
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
13
- </body>
14
- </html>
data/spec/fixtures/filterImageLink.html DELETED
@@ -1,34 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://link_1.jpg">
11
- <img src="http://imgsrc_1.jpg" alt="" /></a>
12
- <a href="http://link_2.jpg">
13
- <img src="http://imgsrc_2.jpg" alt="" /></a>
14
- <a href="http://link_3.JPG">
15
- <img src="http://imgsrc_3.JPG" alt="" /></a>
16
- <a href="http://link_4.png">
17
- <img src="http://imgsrc_4.png" alt="" /></a>
18
- <a href="http://link_5.jpeg">
19
- <img src="http://imgsrc_5.jpeg" alt="" /></a>
20
- <a href="http://link_6.PNG">
21
- <img src="http://imgsrc_6.PNG" alt="" /></a>
22
- <a href="http://link_7.jpega">
23
- <img src="http://imgsrc_7.jpega" alt="" /></a>
24
- <a href="http://link_8.gif">
25
- <img src="http://imgsrc_8.gif" alt="" /></a>
26
- <a href="http://link_9.GIF">
27
- <img src="http://imgsrc_9.GIF" alt="" /></a>
28
- <a href="http://link_10.tiff">
29
- <img src="http://imgsrc_10.tiff" alt="" /></a>
30
- <a href="http://link_11.TIFF">
31
- <img src="http://imgsrc_11.TIFF" alt="" /></a>
32
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
33
- </body>
34
- </html>
data/spec/fixtures/publishDump.html DELETED
@@ -1,14 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://oh-news.net/live/wp-content/uploads/2011/04/Eila_omote.jpg">
11
- <img src="http://24.media.tumblr.com/tumblr_m5gneyJmsH1qza5ppo1_500.jpg" alt="" /></a>
12
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
13
- </body>
14
- </html>
data/spec/fixtures/storeLink.html DELETED
@@ -1,34 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://link_1.jpg">
11
- <img src="http://imgsrc_1.jpg" alt="" /></a>
12
- <a href="http://link_2.jpg">
13
- <img src="http://imgsrc_2.jpg" alt="" /></a>
14
- <a href="http://link_3.JPG">
15
- <img src="http://imgsrc_3.JPG" alt="" /></a>
16
- <a href="http://link_4.png">
17
- <img src="http://imgsrc_4.png" alt="" /></a>
18
- <a href="http://link_5.jpeg">
19
- <img src="http://imgsrc_5.jpeg" alt="" /></a>
20
- <a href="http://link_6.PNG">
21
- <img src="http://imgsrc_6.PNG" alt="" /></a>
22
- <a href="http://link_7.jpega">
23
- <img src="http://imgsrc_7.jpega" alt="" /></a>
24
- <a href="http://link_8.gif">
25
- <img src="http://imgsrc_8.gif" alt="" /></a>
26
- <a href="http://link_9.GIF">
27
- <img src="http://imgsrc_9.GIF" alt="" /></a>
28
- <a href="http://link_10.tiff">
29
- <img src="http://imgsrc_10.tiff" alt="" /></a>
30
- <a href="http://link_11.TIFF">
31
- <img src="http://imgsrc_11.TIFF" alt="" /></a>
32
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
33
- </body>
34
- </html>
data/spec/fixtures/storeLink2.html DELETED
@@ -1,36 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://id774.net">id774.net</a>
9
- <a href="http://reblog.id774.net">reblog.id774.net</a>
10
- <a href="http://link_1.jpg">
11
- <img src="http://imgsrc_1.jpg" alt="" /></a>
12
- <a href="http://link_2.jpg">
13
- <img src="http://imgsrc_2.jpg" alt="" /></a>
14
- <a href="http://link_3.JPG">
15
- <img src="http://imgsrc_3.JPG" alt="" /></a>
16
- <a href="http://link_4.png">
17
- <img src="http://imgsrc_4.png" alt="" /></a>
18
- <a href="http://link_5.jpeg">
19
- <img src="http://imgsrc_5.jpeg" alt="" /></a>
20
- <a href="http://link_6.PNG">
21
- <img src="http://imgsrc_6.PNG" alt="" /></a>
22
- <a href="http://link_7.jpega">
23
- <img src="http://imgsrc_7.jpega" alt="" /></a>
24
- <a href="http://link_8.gif">
25
- <img src="http://imgsrc_8.gif" alt="" /></a>
26
- <a href="http://link_9.GIF">
27
- <img src="http://imgsrc_9.GIF" alt="" /></a>
28
- <a href="http://link_10.tiff">
29
- <img src="http://imgsrc_10.tiff" alt="" /></a>
30
- <a href="http://link_11.TIFF">
31
- <img src="http://imgsrc_11.TIFF" alt="" /></a>
32
- <a href="http://link_12.jpg">
33
- <img src="http://imgsrc_12.jpg" alt="" /></a>
34
- <a href="http://blog.id774.net/post/">blog.id774.net</a>
35
- </body>
36
- </html>
data/spec/fixtures/storeTarget.html DELETED
@@ -1,11 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="http://oh-news.net/live/wp-content/uploads/2011/04/Eila_omote.jpg">
9
- <img src="http://24.media.tumblr.com/tumblr_m5gneyJmsH1qza5ppo1_500.jpg" alt="" /></a>
10
- </body>
11
- </html>
data/spec/fixtures/storeTarget2.html DELETED
@@ -1,11 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="ja">
3
- <head>
4
- <title>Sample</title>
5
- </head>
6
- <body>
7
- <p>A simple <b>test</b> string.</p>
8
- <a href="aaa">
9
- <img src="bbb" alt="" /></a>
10
- </body>
11
- </html>
data/spec/plugins/extract/link_spec.rb DELETED
@@ -1,38 +0,0 @@
1
- # -*- coding: utf-8 -*-
2
- # Name:: Automatic::Plugin::Extract::Link
3
- # Author:: 774 <http://id774.net>
4
- # Created:: Jun 12, 2012
5
- # Updated:: Jun 14, 2012
6
- # Copyright:: 774 Copyright (c) 2012
7
- # License:: Licensed under the GNU GENERAL PUBLIC LICENSE, Version 3.0.
8
-
9
- require File.expand_path(File.dirname(__FILE__) + '../../../spec_helper')
10
-
11
- require 'extract/link'
12
-
13
- describe Automatic::Plugin::ExtractLink do
14
- context "with html contain link tag" do
15
- subject {
16
- Automatic::Plugin::ExtractLink.new({},
17
- AutomaticSpec.generate_pipeline {
18
- html "extractLink.html"
19
- }
20
- )}
21
-
22
- describe "#run" do
23
- its(:run) { should have(4).items }
24
- specify {
25
- subject.run
26
- expect = "http://id774.net"
27
- subject.instance_variable_get(:@return_html)[0].should == expect
28
- expect = "http://reblog.id774.net"
29
- subject.instance_variable_get(:@return_html)[1].should == expect
30
- expect = "http://oh-news.net/live/wp-content/uploads/2011/04/Eila_omote.jpg"
31
- subject.instance_variable_get(:@return_html)[2].should == expect
32
- expect = "http://blog.id774.net/post/"
33
- subject.instance_variable_get(:@return_html)[3].should == expect
34
- }
35
- end
36
- end
37
- end
38
-