bwkfanboy 1.4.1 → 2.0.0

data/bin/bwkfanboy_generate

@@ -1,25 +1,13 @@
 #!/usr/bin/env ruby
-# -*-ruby-*-
 
-require_relative '../lib/bwkfanboy/generate'
+require_relative '../lib/bwkfanboy/cliutils'
+require_relative '../lib/bwkfanboy/generator'
+include Bwkfanboy
 
-$conf = {
-  banner: "Usage: #{File.basename($0)} [options] < json",
-  check: false
-}
-
-# we are expection the input ONLY in UTF-8
-Encoding.default_external = 'UTF-8'
-
-o = Bwkfanboy::Utils.cl_parse(ARGV, $conf[:banner])
-o.on('--check', 'Validate the input (slow!)') { |i| $conf[:check] = true }
-Bwkfanboy::Utils.cl_parse(ARGV, $conf[:banner], o) # run cl parser
+### main
 
 begin
-  j = JSON.parse(STDIN.read)
-rescue
-  Bwkfanboy::Utils.errx(1, "stdin had invalid JSON");
+  puts Generator.atom(Generator.unpack $stdin)
+rescue GeneratorException
+  CliUtils.errx EX_DATAERR, $!.to_s
 end
-
-if $conf[:check] then Bwkfanboy::Generate.validate(j) end
-puts Bwkfanboy::Generate.atom(j)

data/bin/bwkfanboy_parse

@@ -1,27 +1,31 @@
 #!/usr/bin/env ruby
-# -*-ruby-*-
 
-require_relative '../lib/bwkfanboy/parser'
+require_relative '../lib/bwkfanboy/home'
+require_relative '../lib/bwkfanboy/plugin'
+include Bwkfanboy
 
-$conf = {
-  banner: "Usage: #{File.basename($0)} [options] /path/to/my/plugin.rb < html"
+### main
+
+home = Home.new {|loader, o|
+  loader.conf[:banner] = "#{File.basename $0} [options] plugin [opt1 opt2 ...] < html"
 }
 
-Bwkfanboy::Utils.cl_parse(ARGV, $conf[:banner], nil, true)
+CliUtils.errx EX_USAGE, home.conf[:banner] unless BH.all_set?(ARGV[0])
 
-if ARGV.size == 0 then
-  abort($conf[:banner])
-else
-  Bwkfanboy::Utils.plugin_load(ARGV[0], Bwkfanboy::Meta::PLUGIN_CLASS)
+# load plugin
+stream = [$stdin]
+begin
+  p = Plugin.new home.conf[:plugins_path], ARGV[0],  ARGV[1..-1] do |loader|
+    CliUtils.veputs 2, "'#{ARGV[0]}' plugin loaded from #{loader.origin}"
+  end
+  p.run_parser stream
+rescue PluginException
+  CliUtils.errx EX_DATAERR, $!.to_s
 end
 
-opt = Bwkfanboy::Utils.plugin_opts(ARGV)
-pn = Page.new(opt)
-pn.check()
-pn.parse(STDIN)
-
-if Bwkfanboy::Utils.cfg[:verbose] >= 2 then
-  pn.dump()
+# print output
+if home.conf[:verbose] >= 1
+  pp p
 else
-  puts pn.to_json()
+  p.pack $stdout
 end

data/bwkfanboy.gemspec

@@ -0,0 +1,40 @@
+# -*-ruby-*-
+require File.expand_path('../lib/bwkfanboy/meta', __FILE__)
+include Bwkfanboy
+
+Gem::Specification.new do |gem|
+  gem.authors       = [Meta::NAME]
+  gem.email         = [Meta::EMAIL]
+  gem.description   = 'A converter from a raw HTML to an Atom feed. You can use it to watch sites that do not provide its own feed'
+  gem.summary       = gem.description + '.'
+  gem.homepage      = Meta::HOMEPAGE
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^test/test_.+\.rb})
+  gem.name          = Meta::NAME
+  gem.version       = Meta::VERSION
+
+  gem.required_ruby_version = '>= 1.9.2'
+  gem.extra_rdoc_files      = gem.files.grep(%r{^doc/})
+  gem.rdoc_options          << '-m' << 'README.rdoc'
+
+  gem.post_install_message  = <<-MESSAGE
+This version was rewritten from scratch.
+Plugins from & for previous versions (0.x & 1.x) WILL NOT WORK.
+
+See also doc/NEWS.rdoc file.
+  MESSAGE
+
+  gem.add_dependency "open4", "~> 1.3.0"
+  gem.add_dependency "msgpack", "~> 0.4.6"
+  gem.add_dependency "rake", "~> 0.9.2.2"
+  gem.add_dependency "nokogiri", "~> 1.5.2"
+  gem.add_dependency "sinatra", "~> 1.3.2"
+  gem.add_dependency "haml", "~> 3.1.4"
+  gem.add_dependency "rdoc", "~> 3.12"
+  
+  gem.add_development_dependency "minitest", "~> 2.12.1"
+  gem.add_development_dependency "fakefs", "~> 0.4.0"
+  gem.add_development_dependency "rack-test", "~> 0.6.1"
+end

data/config.ru

@@ -0,0 +1,3 @@
+require './lib/bwkfanboy/server'
+run Bwkfanboy::MyApp
+

data/doc/NEWS.rdoc

@@ -1,92 +1,34 @@
-=== 1.4.1
+=== 2.0.0
 
-Wed Apr 20 21:12:16 EEST 2011
+Mon Apr 23 07:28:52 EEST 2012
 
-- Repackage under RVM due to a strange bug (in native FreeBSD ruby +
-  rubygems 1.7.2) with a date in the generated gem.
+Changes from 1.4.1:
 
-=== 1.4.0
+* CGI script & webrick-based server are gone. Use rackup to start a
+  Sinatra app instead. HTTP API is completely new, which means you'll
+  need to change URLs in your RSS reader.
 
-Wed Apr  6 23:45:39 EEST 2011
+* bwkfanboy_server is gone.
+  
+* Plugin interface API completely changed to work nicely with the
+  long-running Sinatra app.
 
-- A new plugin: articles (per-user) from inc.com.
+  This means you'll need to update all your personal plugins. I am very
+  sorry.
 
-- Removed quora plugin.
+* bwkfanboy_fetch CL utility is gone. If you still need it, use curl
+  instead.
 
-- Fixed a but with '-t' command line option.
+* bwkfanboy:
 
-=== 1.3.2
+  - '-t' CL option is '-s' now;
 
-Mon Mar 28 12:13:31 EEST 2011
+  - output in '-i' is different;
 
-- Fixed a small bug with an Atom feed for plugins with Meta::URI options.
+  - '-p' is '--plugins-path'.
 
-- Updated quora plugin (a rather useless operation, because from [insert
-  date] Quora provides its own answers-feed). This plugin will be
-  removed.
+* bwkfanboy_parse emits MessagePack instead of JSON. (Who cares?)
 
-=== 1.3.1
+* Log files in /tmp are gone. If you need logs from the Sinatra app,
+  comment in etc/sinatra.rb a monkey patch for rack.
 
-Thu Mar 17 17:19:52 EET 2011
-
-- Updated quora plugin.
-
-- Updated 1 test for jsonschema 2.0.1.
-
-- Updated the list of dependencies.
-
-=== 1.3.0
-
-Thu Feb 10 11:17:57 EET 2011
-
-- A new plugin: articles form econlib.org.
-
-- (internal) Injected a small part of falsework's naive template.
-
-=== 1.2.9
-
-- Reverted to rubygems 1.3.7 due to mysterious 1.5.0 behaviour.
-
-=== 1.2.8
-
-- Updated quora plugin for nodejs 0.3.7.
-
-=== 1.2.7
-
-- Fixed a bug with JavaScript parsing in quora plugin.
-
-=== 1.2.5
-
-- Added '-t' CL option for bwkfanboy util.
-
-- Updated quora plugin.
-
-=== 1.1.4
-
-- INCOMPATIBILITY: from now on, all plugins must do NOT read the stdin
-  but bwkfanboy's provided stream. See doc/plugins.rdoc.
-
-- Moved the code from bwkfanboy_* to libraries.
-
-- From now on, bwkfanboy util by default do not run utils in a pipe but
-  uses libraries directly. One can restore the old functionality with
-  '-O' CL swith.
-
-=== 0.1.4
-
-- The minimum required Ruby version is 1.9.2.
-- bwkfanboy_server now inserts a Last-Modified header.
-- Fixed (?) a bug in bwkfanboy_generate with external encoding.
-- Updated tests for Ruby 1.9.2.
-
-=== 0.1.3
-
-- Updated Quora plugin.
-- Added '-V' command line option to all utils.
-- Updated tests.
-
-=== 0.1.1
-
-- Plugins can have user-supplied options in realtime.
-- Added quora.rb plugin.
-- Fixed some typos.

data/doc/plugin.rdoc

@@ -1,88 +1,81 @@
 = How to Write a \Plugin
 
-First of all, look at examples provided with bwkfanboy. They were
-intended to be 100% working because I was writing them for myself.
+First of all, look at examples included with bwkfanboy.
 
-Basically, all you need is to write a class named _Page_ that inherits
-class Bwkfanboy::Parse, override in the child \#myparse method and write
-a simple module named _Meta_ inside your _Page_ class.
+Basically, all you need is to write 1 method (function) called +parse+
+and set several instance variables.
 
 == Skeleton
 
-Here is a skeleton of a plugin:
-
-  require 'nokogiri'
- 
-  class Page < Bwkfanboy::Parse
-    module Meta
-      URI = 'http://example.org/news'
-      ENC = 'UTF-8'
-      VERSION = 1
-      COPYRIGHT = '(c) 2010 John Doe'
-      TITLE = "News from example.org"
-      CONTENT_TYPE = 'html'
-    end
- 
-    def myparse(stream)
-      # read 'stream' IO object and parse it
-      doc = Nokogiri::HTML(stream, nil, Meta::ENC)
-      doc.xpath("XPATH QUERY").each {|i|
-        t = clean(i.xpath("XPATH QUERY").text())
-        l = clean(i.xpath("XPATH QUERY").text())
-        u = date(i.xpath("XPATH QUERY").text())
-        a = clean(i.xpath("XPATH QUERY").text())
-        c = clean(i.xpath("XPATH QUERY").text())
- 
-        self << { title: t, link: l, updated: u, author: a, content: c }
+Here is a simple plugin skeleton:
+
+  @uri << 'http://example.org/news'
+  @enc = 'UTF-8'
+  @version = 1
+  @copyright = '(c) 2012 John Doe'
+  @title = "News from example.org"
+  @content_type = 'html'
+
+  def parse streams
+    streams.each do |io|
+      doc = Nokogiri::HTML io, nil, @enc
+      doc.xpath("XPATH QUERY").each {|idx|
+        t = idx.xpath("XPATH QUERY").text
+        l = idx.xpath("XPATH QUERY").text
+        u = BH.date idx.xpath("XPATH QUERY").text
+        a = idx.xpath("XPATH QUERY").text
+        c = idx.xpath("XPATH QUERY").text
+
+        self << { 'title' => t, 'link' => l, 'updated' => u,
+          'author' => a, 'content' => c }
       }
     end
   end
 
+
 You can get the skeleton in the current directory by typing:
 
-  % bwkfanboy -t myplugin.rb
+  % bwkfanboy -s myplugin
 
-As you see, we are using Nokogiri for HTML parsing. You are not
-required to use it too--take the parser whatever you like. Nokogiri
-is nice, because it's able to read a broken HTML and search thought
-it via XPath. If you would like to use, for example, REXML, beware
-that it loves only strict XML--you may need to clean the HTML with
-an external utility such as Tide.
+As you see, we are using Nokogiri for HTML parsing. You are not required
+to use it too--choose the parser whatever you like. Nokogiri is nice,
+because it's able to read a broken HTML and search thought it via
+XPath. If you would like to use, for example, REXML, beware that it
+loves only strict XML--you may need to clean the HTML with an external
+utility such as Tide.
 
-Bwkfanboy loads a plugin from 1 file as a valid Ruby code. It means
-that the plugin can contain *any* Ruby code, but doesn't mean that
-it should.
+Bwkfanboy loads a plugin from 1 file as a valid Ruby code (via
+<tt>instance_eval</tt>). This means that the plugin can contain *any*
+Ruby code, but doesn't mean that it should.
 
 === \Meta
 
-Module _Meta_ can only have constants--and *all* constants listed in the
-skeleton are mandatory.
+Several instance variable are required to be set.
 
-* <tt>URI</tt>--can be a <tt>http(s)://</tt> or <tt>ftp://</tt> URL
-  or just a path to a file on your local machine, as
-  <tt>/home/bob/huzza.html</tt>. This is the source that
-  bwkfanboy will be transforming to the Atom feed.
+* <tt>@uri</tt>--is a an <b>array of strings</b> <tt>http(s)://</tt> or
+  <tt>ftp://</tt> URL or just a path to a file on your local machine, as
+  <tt>/home/bob/huzza.html</tt>. This is sources that bwkfanboy will use
+  to create 1 Atom feed.
 
-* <tt>ENC</tt>--an encoding for URI.
+* <tt>@enc</tt>--an encoding for *all* URIs in @uri.
 
-* <tt>VERSION</tt>--a version of a plugin.
+* <tt>@version</tt>--a version of a plugin. Any unsigned integer.
 
-* <tt>COPYRIGHT</tt>--some boring string.
+* <tt>@copyright</tt>--some boring string.
 
-* <tt>TITLE</tt>--a short description of the future feed. It'll be
-  used later in the resulting XML.
+* <tt>@title</tt>--a short description of the future feed.
 
-* <tt>CONTENT_TYPE</tt>--one of +xhtml+, +html+ or +text+ values. This is
-  very important constant because it says in what format entries
-  will be placed in the feed. Usually it's safe to use +html+.
+* <tt>@content_type</tt>--one of +xhtml+, +html+ or +text+ values. This
+  is very important veriable because it says what format entries would
+  have in the feed. Usually it's safe to use +html+.
 
-=== myparse
+=== parse
 
-In \#myparse method read 'stream' IO object. The contents of it is the
++parse+ method uses an array of IO objects. The contents of them is the
 raw HTML you want to parse. The general idea:
 
 * Atom feed must contain at least 1 entry, so look in the HTML for some
-  crap which you must break into 5 peaces:
+  crap that you will break into 5 peaces:
 
   - a title of the entry
   - a link for it
@@ -93,7 +86,8 @@ raw HTML you want to parse. The general idea:
 * After you scan and grab 1 entry, create a hash and add it to
   _self_ as it was in the skeleton:
 
-    self << { title: t, link: l, updated: u, author: a, content: c }
+    self << { 'title' => t, 'link' => l, 'updated' => u,
+              'author' => a, 'content' => c }
 
   Here variables _t_, _l_, _u_, _a_ and _c_ contains the actual
   value of 5 peaces for the entry. Names of the keys in the hash are
@@ -102,10 +96,9 @@ raw HTML you want to parse. The general idea:
 * Probably there would be more crap in the HTML that you can use to
   construct another entry. Keep parsing and adding entries.
 
-* While you're scanning, use the 2 helper methods for cleaning each
-  peace: \#clean, which removed duplicate spaces and \#date, which parses
-  a string and return a date in ISO8601 format. You may override \#date
-  method if you like.
+* While you're scanning, use the <tt>BH.date</tt> helper method, which
+  parses a string and return a date in ISO8601 format.
+
 
 === Options
 
@@ -116,40 +109,31 @@ it is silly to write a new plugin every time for a new
 participant. Instead, you can write 1 plugin which have an _option_ to
 take a parameter (a user name, in this case).
 
-Options (if any) are always accessible via \#opt method which is just an
-attr_reader of an array.
+Options (if any) are always accessible via <tt>@opt</tt> valiable which is an
+array of strings.
 
-One can play the really interesting trick with Meta::URI constant. It is
+One can play the really interesting trick with <tt>@uri</tt>. It is
 possible to make it dynamic, for example:
 
-  URI = 'http://www.quora.com/#{opt[0]}/answers'
+  @opt.each {|i| @uri << 'http://www.inc.com/author/' + i }
 
-Then, if a user will provide 1 option (say 'Mark-Suster')--it will
+Then, if a user will provide 1 option (say 'jason-fried')--it will
 appear in the final URI as follows:
 
-  http://www.quora.com/Mark-Suster/answers
+  http://www.inc.com/author/jason-fried
 
-Such dynamic is possible only for Meta::URI constant and in such case,
-_option_ becomes mandatory for the end-user.
 
 == How to test all this
 
 To test how nice your plugin works, save the html page to the file
 and type:
 
-  % bwkfanboy_parse -vv /path/to/the/plugin.rb < saved_page.html
-
-to see the result as in plain text, or
-
-  % bwkfanboy_parse -v /path/to/the/plugin.rb < saved_page.html
-
-as a pretty JSON.
+  % bwkfanboy_parse -v plugin < saved_page.html
 
 For option-enabled plugins, supply additional parameters for them after
 the plugin path:
 
-  % bwkfanboy_parse -vv /path/to/the/plugin.rb \
-    option_1 "option 2" < saved_page.html
+  % bwkfanboy_parse -v plugin opt1 opt2 < saved_page.html
 
 <tt>bwkfanboy_parse</tt> return 0 if no errors occurred or >= 1 if you
 have errors in your plugin code. N.B.: the output from