bwkfanboy 1.3.2 → 1.4.0

data/test/semis/links.txt

@@ -1,4 +1,4 @@
 # output_file URL
-quora.html http://www.quora.com/Brandon-Smietana/answers
+inc.html http://www.inc.com/author/jason-fried
 bwk.html http://www.dailyprincetonian.com/advanced_search/?author=Brian+Kernighan
 econlib.html http://www.econlib.org/cgi-bin/searcharticles.pl?sortby=DD&query=ha*

data/test/test_server.rb

@@ -25,19 +25,20 @@ class TestServer < MiniTest::Unit::TestCase
     assert_raises(OpenURI::HTTPError) { open("http://#{ADDR}:#{@port}") }
     assert_raises(OpenURI::HTTPError) { open("http://#{ADDR}:#{@port}/?p=INVALID") }
     # 'o' is missing
-    assert_raises(OpenURI::HTTPError) { open("http://#{ADDR}:#{@port}/?p=quora") }
+    assert_raises(OpenURI::HTTPError) { open("http://#{ADDR}:#{@port}/?p=inc") }
   end
 
-  def test_right_plugin
-    r = ''
-    open("http://#{ADDR}:#{@port}/?p=bwk") { |f| r = f.read }
-    # wget -q -O - 127.0.0.1:9042/\?p=bwk | md5
-    assert_equal('64186fac2c52e5a969ad5675b9cc95ed', Digest::MD5.hexdigest(r))
-
-    r = ''
-    open("http://#{ADDR}:#{@port}/?p=quora&o=foo") { |f| r = f.read }
-    # bin/bwkfanboy_server -Dd
-    # wget -q -O - '127.0.0.1:9042/\?p=quora&o=foo' | md5
-    assert_equal('0f3f6607768392d69d15621eee815ab3', Digest::MD5.hexdigest(r))
+  def test_right_plugins
+    plugins = {
+      'bwk' => '64186fac2c52e5a969ad5675b9cc95ed',
+      'econlib' => '11f6114a9ab54d6ec67a26cbd76f5260',
+      'inc' => '13dae248c81dd6407ff327dd5575f8b5',
+    }
+    plugins.each {|k,v|
+      r = ''
+      open("http://#{ADDR}:#{@port}/?p=#{k}&o=foo") { |f| r = f.read }
+      # wget -q -O - '127.0.0.1:9042/?p=inc&o=foo' | md5
+      assert_equal(v, Digest::MD5.hexdigest(r))
+    }
   end
 end

metadata

@@ -1,108 +1,82 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: bwkfanboy
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 3
-  - 2
-  version: 1.3.2
+version: !ruby/object:Gem::Version
+  version: 1.4.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Alexander Gromnitsky
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-03-28 00:00:00 +03:00
-default_executable: bwkfanboy
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-04-06 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: open4
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &338037960 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 0
-        - 1
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.0.1
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: activesupport
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *338037960
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &338035630 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 3
-        - 0
-        - 5
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 3.0.5
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *338035630
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &338034330 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 4
-        - 4
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.4.4
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: jsonschema
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *338034330
+- !ruby/object:Gem::Dependency
+  name: jsonschema
+  requirement: &338032480 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 0
-        - 1
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 2.0.1
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: git
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: *338032480
+- !ruby/object:Gem::Dependency
+  name: git
+  requirement: &338031220 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 2
-        - 5
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.2.5
   type: :development
-  version_requirements: *id005
-description: A converter from HTML to Atom feed that you can use to watch sites that do not provide its own feed.
+  prerelease: false
+  version_requirements: *338031220
+description: bwkfanboy is a converter from a raw HTML to an Atom feed. You can use
+  it to watch sites that do not provide its own feed.
 email: alexander.gromnitsky@gmail.com
-executables: 
+executables:
 - bwkfanboy_generate
 - bwkfanboy_parse
 - bwkfanboy
 - bwkfanboy_server
 - bwkfanboy_fetch
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - doc/NEWS.rdoc
 - doc/plugin.rdoc
 - doc/bwkfanboy_fetch.rdoc
@@ -110,7 +84,7 @@ extra_rdoc_files:
 - doc/bwkfanboy_parse.rdoc
 - doc/bwkfanboy_server.rdoc
 - doc/README.rdoc
-files: 
+files:
 - README.rdoc
 - Rakefile
 - bin/bwkfanboy
@@ -137,8 +111,7 @@ files:
 - lib/bwkfanboy/plugins/bwk.rb
 - lib/bwkfanboy/plugins/econlib.rb
 - lib/bwkfanboy/plugins/freebsd-ports-update.rb
-- lib/bwkfanboy/plugins/quora.js
-- lib/bwkfanboy/plugins/quora.rb
+- lib/bwkfanboy/plugins/inc.rb
 - lib/bwkfanboy/schema.js
 - lib/bwkfanboy/utils.rb
 - test/helper.rb
@@ -149,52 +122,44 @@ files:
 - test/semis/bwk.html
 - test/semis/bwk.json
 - test/semis/econlib.html
+- test/semis/inc.html
 - test/semis/links.txt
-- test/semis/quora.html
 - test/test_fetch.rb
 - test/test_generate.rb
 - test/test_parse.rb
 - test/test_server.rb
 - test/xml-clean.sh
 - web/bwkfanboy.cgi
-has_rdoc: true
 homepage: http://github.com/gromnitsky/bwkfanboy
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - -m
 - doc/README.rdoc
 - -x
 - plugins
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 1
-      - 9
-      - 2
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.9.2
-required_rubygems_version: !ruby/object:Gem::Requirement 
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.7.2
 signing_key: 
 specification_version: 3
-summary: A converter from HTML to Atom feed that you can use to watch sites that do not provide its own feed.
-test_files: 
+summary: bwkfanboy is a converter from a raw HTML to an Atom feed. You can use it
+  to watch sites that do not provide its own feed.
+test_files:
 - test/test_fetch.rb
 - test/test_parse.rb
 - test/test_server.rb

data/lib/bwkfanboy/plugins/quora.js

@@ -1,111 +0,0 @@
-#!/usr/bin/env node
-
-/*
-  A companion to 'quora.rb' plugin.
-*/
-
-var u = require('util')
-
-function get_timestamps(data) {
-	var e = {}
-	var link = null
-	for (i in data) {
-		var f = data[i][0]
-	
-		if (f == 'FeedStoryItem') {
-			link = data[i][3]['q_path']
-//			u.puts(link)
-		}
-
-		if (f == 'DateTimeComponent') {
-			e[link] = data[i][3]['epoch_us']
-		}
-	}
-	return e
-}
-
-function collect_stdin(callback) {
-	var input = '';
-    process.stdin.resume();
-	process.stdin.setEncoding('ascii');
-    process.stdin.on('data', function (chunk) {
-        input += chunk
-    });
-    process.stdin.on('end', function () {
-		callback(input);
-    });
-};
-
-function prepare4eval(body) {
-	var head = "function W2() {}\n" +
-"W2.addComponentMetadata = function(foo) {}\n" +
-"W2.registerComponents = function(foo) {}\n" +
-"W2._ConnectionWarningCls = function(args) {}\n" +
-"W2._LoadingCls = function(args) {}\n" +
-"W2._InteractionModeCls = function(args) {}\n" +
-"W2.onLoad = function(args) {}\n" +
-"\n" +
-"document = 'foo'\n" +
-"$ = function(foo) { return $ }\n" +
-"$.ready = function(foo) {}\n" +
-"\n" +
-"arr = function(args) {\n" +
-"	a = []\n" +
-"	a.push(args.callee.name)\n" +
-"	for(var i = 0; i < args.length; i++) { a.push(args[i]) }\n" +
-"	return a\n" +
-"}\n" +
-"\n" +
-"function LoginButton(args) { return arr(arguments) }\n" +
-"function ContextNavigator(args) { return arr(arguments) }\n" +
-"function TypeaheadContextText(args) { return arr(arguments) }\n" +
-"function TypeaheadResults(args) { return arr(arguments) }\n" +
-"function QuestionAddLink(args) { return arr(arguments) }\n" +
-"function TitleNotificationsCount(args) { return arr(arguments) }\n" +
-"function TextareaAutoSize(args) { return arr(arguments) }\n" +
-"function PMsgContainer(args) { return arr(arguments) }\n" +
-"function UserAdminMenuLink(args) { return arr(arguments) }\n" +
-"function PagedList(args) { return arr(arguments) }\n" +
-"function FeedStoryItem(args) { return arr(arguments) }\n" +
-"function QuestionLink(args) { return arr(arguments) }\n" +
-"function QuestionBestSourceIcon(args) { return arr(arguments) }\n" +
-"function AnswerVotingButtons(args) { return arr(arguments) }\n" +
-"function TruncatePhraseList(args) { return arr(arguments) }\n" +
-"function CommentLink(args) { return arr(arguments) }\n" +
-"function DateTimeComponent(args) { return arr(arguments) }\n" +
-"function AnswerComments(args) { return arr(arguments) }\n" +
-"function Comment(args) { return arr(arguments) }\n" +
-"function FeedAnswerItem(args) { return arr(arguments) }\n" +
-"function HoverMenu(args) { return arr(arguments) }\n" +
-"function ExpandableQText(args) { return arr(arguments) }\n" +
-"function TruncatedQText(args) { return arr(arguments) }\n" +
-"function UseMobileSite(args) { return arr(arguments) }\n" +
-"function LoginSignal(args) { return arr(arguments) }\n" +
-"function LiveLogin(args) { return arr(arguments) }\n" +
-"function PresencePageMonitor(args) { return arr(arguments) }\n" +
-"function UserSig(args) { return arr(arguments) }\n" +
-"function HeaderLogo(args) { return arr(arguments) }\n" +
-"function NavElement(args) { return arr(arguments) }\n" +
-"function UserFollowLink(args) { return arr(arguments) }\n" +
-"function FlashClient(args) { return arr(arguments) }\n" +
-"function AddQuestionLink(args) { return arr(arguments) }\n" +
-"function QTextImageEnlarger(args) { return arr(arguments) }\n" +
-"function NavigatorResults(args) { return arr(arguments) }\n" +
-"function RateAnswerApproveButton(args) { return arr(arguments) }\n" +
-"function AnswerVoterNamesExpandable(args) { return arr(arguments) }\n" +
-"function PrefetchManager(args) { return arr(arguments) }\n" +
-"function ResponseRecorder(args) { return arr(arguments) }\n" +
-	'';
-	var tail = "\n_components;\n";
-
-	return head + body + tail;
-}
-
-collect_stdin(function(t) {
-//	u.puts(t)
-
-    var script = process.binding('evals').Script
-	var code = script.runInThisContext(prepare4eval(t))
-// 	u.puts(u.inspect(code, false, null))
- 	u.puts(JSON.stringify(get_timestamps(code), null, '  '))
-})

data/lib/bwkfanboy/plugins/quora.rb

@@ -1,81 +0,0 @@
-# Quora uses JavaScript to dynamically insert timestamps for the
-# questions/comments. To combat this, we cut the JS from the page,
-# evaluate it in nodejs and construct a hash with 'article-name =>
-# timestamp' pairs.
-#
-# Requires:
-#
-# - 'quora.js' script in directory with the plugin;
-# - 1 option: a Quora user's name, for example 'Brandon-Smietana'
-
-require 'nokogiri'
-
-class Page < Bwkfanboy::Parse
-  module Meta
-    URI = 'http://www.quora.com/#{opt[0]}/answers'
-    URI_DEBUG = '/home/alex/lib/software/alex/bwkfanboy/test/semis/quora.html'
-    ENC = 'UTF-8'
-    VERSION = 13
-    COPYRIGHT = "See bwkfanboy's LICENSE file"
-    TITLE = "Last n answers (per-user) from Quora; requires nodejs 0.3.7+"
-    CONTENT_TYPE = 'html'
-  end
-  
-  def myparse(stream)
-    profile = opt[0] # for example, 'Brandon-Smietana'
-    
-    # read stdin
-    doc = Nokogiri::HTML(stream, nil, Meta::ENC)
-
-    # extract & evaluate JavaScript into tstp
-    tstp = nil
-    doc.xpath("//script").each {|i|
-      js = i.text
-      if js.include?('"epoch_us"')
-        if Bwkfanboy::Utils.cfg[:verbose] >= 3
-          File.open("#{File.basename(__FILE__)}-epoch.js.raw", "w+") {|i| i.puts js }
-        end
-        # open a pipe, write js to it & read a JSON result
-        r = ''
-        begin
-          pipe = IO.popen("#{File.dirname(__FILE__)}/quora.js", 'w+')
-          pipe.puts js
-          pipe.close_write
-          while line = pipe.gets
-            r << line
-          end
-          pipe.close
-        rescue
-          fail "evaluation in nodejs failed: #{$!}"
-        end
-#        r = Bwkfanboy::Utils.cmd_run("echo '#{js}' | #{File.dirname(__FILE__)}/quora.js")
-#        fail "evaluation in nodejs failed: #{r[1]}" if r[0] != 0
-        tstp = JSON.parse r
-        break
-      end
-    }
-
-    # xpath movements
-    url = 'http://www.quora.com'
-    a = clean(doc.xpath("//h1").text())
-      
-    doc.xpath("//div[@class='feed_item_question']").each {|i|
-      t = clean(i.xpath("h2").text())
-
-      l = clean(i.xpath("h2//a")[0].attributes['href'].value())
-      next unless tstp.key?(l)  # ignore answers without timestamps
-      u = date(Time.at(tstp[l]/1000/1000).to_s)
-#      u = DateTime.new.iso8601
-      l = url + l + '/answer/' + profile
-      
-      c = i.xpath("../div[@class='hidden expanded_q_text']/div").inner_html(encoding: Meta::ENC)
-      if c == ''
-        c = i.xpath("../../div/div/div/div[3]")
-        c.xpath("div").each {|j| j.remove() }
-        c = c.inner_html(encoding: Meta::ENC)
-      end
-      
-      self << { title: t, link: l, updated: u, author: a, content: c }
-    }
-  end
-end