RubyGems - pstuteville-scrubyt - Versions diffs - 0.4.31 - Mend

pstuteville-scrubyt 0.4.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (520) hide show

data/test/blackbox_test.rb ADDED Viewed

@@ -0,0 +1,60 @@
+$lib_path = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+$:.unshift $lib_path
+require 'scrubyt'
+require 'test/unit'
+def perform_test(test_path, detailed = false)
+  out = $stdout
+  $stdout = StringIO.new unless detailed
+  cwd = Dir.getwd
+  Dir.chdir(File.dirname(test_path))
+  out.puts "Test: #{test_path}" if detailed
+  out.puts "========== Print Output ==========" if detailed
+  begin
+    expected_xml = File.read(File.basename(test_path)[0..-4] + ".expected.xml")
+    scrubyt_result_native = Scrubyt::Extractor.load(File.basename(test_path))
+    exported_code = scrubyt_result_native.export({:template => 'lambda'})
+    scrubyt_result_exported = Scrubyt::Extractor.define(&eval(exported_code))
+  ensure
+    if detailed
+      out.puts "========== Native Extractor =========="
+      out.puts IO.read(File.basename(test_path))
+      out.puts "========== Exported Extractor =========="
+      out.puts exported_code
+      out.puts "========== Expected =========="
+      out.puts expected_xml
+      out.puts "========== Result (native) =========="
+      out.puts scrubyt_result_native.to_xml
+      out.puts "========== Result (exported) =========="
+      out.puts scrubyt_result_exported.to_xml
+    end
+  end
+  assert_equal expected_xml, scrubyt_result_native.to_xml
+  assert_equal expected_xml, scrubyt_result_exported.to_xml
+ensure
+  Dir.chdir(cwd)
+  $stdout = out
+end
+if $0 == __FILE__ && ARGV[0]
+  include Test::Unit::Assertions
+  perform_test(ARGV[0], true)
+  exit
+end
+class BlackboxTest < Test::Unit::TestCase
+  tests = Dir.glob(File.join(File.dirname(__FILE__), 'blackbox_tests', '**', '*.rb'))
+  tests = tests.sort
+  tests.each do |test_path|
+    define_method("test_#{test_path.gsub('/', '_')}") do
+      perform_test(test_path)
+    end
+  end
+end

data/test/blackbox_tests/basic/multi_root.expected.xml ADDED Viewed

@@ -0,0 +1,8 @@
+<root>
+  <entry>1</entry>
+  <entry>2</entry>
+  <entry>3</entry>
+  <data>4</data>
+  <data>5</data>
+  <data>6</data>
+</root>

data/test/blackbox_tests/basic/multi_root.rb ADDED Viewed

@@ -0,0 +1,6 @@
+lambda do
+  fetch(File.join(File.dirname(__FILE__), "three_divs.html"))
+  entry '1'
+  data '4'
+end

data/test/blackbox_tests/basic/simple.expected.xml ADDED Viewed

@@ -0,0 +1,5 @@
+<root>
+  <entry>1</entry>
+  <entry>2</entry>
+  <entry>3</entry>
+</root>

data/test/blackbox_tests/basic/simple.rb ADDED Viewed

@@ -0,0 +1,5 @@
+lambda do
+  fetch(File.join(File.dirname(__FILE__), "three_divs.html"))
+  entry '1'
+end

data/test/blackbox_tests/basic/three_divs.html ADDED Viewed

@@ -0,0 +1,12 @@
+<html>
+  <body>
+    <div>1</div>
+    <div>2</div>
+    <div>3</div>
+    <p>
+      <div>4</div>
+      <div>5</div>
+      <div>6</div>
+    </p>
+  </body>
+</html>

data/test/blackbox_tests/detail_page/detail_page_1.html ADDED Viewed

@@ -0,0 +1,7 @@
+<html>
+  <body>
+    <div>Detail 1</div>
+    <div>Detail 2</div>
+    <div>Detail 3</div>
+  </body>
+</html>

data/test/blackbox_tests/detail_page/detail_page_2.html ADDED Viewed

@@ -0,0 +1,7 @@
+<html>
+  <body>
+    <div>Detail 4</div>
+    <div>Detail 5</div>
+    <div>Detail 6</div>
+  </body>
+</html>

data/test/blackbox_tests/detail_page/main_page_1.html ADDED Viewed

@@ -0,0 +1,5 @@
+<html>
+  <body>
+    <a href="detail_page_1.html">Main 1</a>
+  </body>
+</html>

data/test/blackbox_tests/detail_page/main_page_2.html ADDED Viewed

@@ -0,0 +1,6 @@
+<html>
+  <body>
+    <a href="detail_page_1.html">Main 1</a>
+    <a href="detail_page_2.html">Main 2</a>
+  </body>
+</html>

data/test/blackbox_tests/detail_page/one_detail_page.expected.xml ADDED Viewed

@@ -0,0 +1,7 @@
+<root>
+  <main>
+    <detail>Detail 1</detail>
+    <detail>Detail 2</detail>
+    <detail>Detail 3</detail>
+  </main>
+</root>

data/test/blackbox_tests/detail_page/one_detail_page.rb ADDED Viewed

@@ -0,0 +1,9 @@
+lambda do
+  fetch(File.join(File.dirname(__FILE__), "main_page_1.html"))
+  main 'Main 1' do
+    xyz_detail do
+      detail 'Detail 1'
+    end
+  end
+end

data/test/blackbox_tests/detail_page/two_detail_pages.expected.xml ADDED Viewed

@@ -0,0 +1,12 @@
+<root>
+  <main>
+    <detail>Detail 1</detail>
+    <detail>Detail 2</detail>
+    <detail>Detail 3</detail>
+  </main>
+  <main>
+    <detail>Detail 4</detail>
+    <detail>Detail 5</detail>
+    <detail>Detail 6</detail>
+  </main>
+</root>

data/test/blackbox_tests/detail_page/two_detail_pages.rb ADDED Viewed

@@ -0,0 +1,9 @@
+lambda do
+  fetch(File.join(File.dirname(__FILE__), "main_page_2.html"))
+  main 'Main 1' do
+    xyz_detail do
+      detail 'Detail 1'
+    end
+  end
+end

data/test/blackbox_tests/next_page/next_page_link.expected.xml ADDED Viewed

@@ -0,0 +1,11 @@
+<root>
+  <entry>1</entry>
+  <entry>2</entry>
+  <entry>3</entry>
+  <entry>4</entry>
+  <entry>5</entry>
+  <entry>6</entry>
+  <entry>7</entry>
+  <entry>8</entry>
+  <entry>9</entry>
+</root>

data/test/blackbox_tests/next_page/next_page_link.rb ADDED Viewed

@@ -0,0 +1,7 @@
+lambda do
+  fetch(File.join(File.dirname(__FILE__), "page_1.html"))
+  entry '1'
+  next_page 'Next'
+end

data/test/blackbox_tests/next_page/page_1.html ADDED Viewed

@@ -0,0 +1,11 @@
+<html>
+  <body>
+    <div>1</div>
+    <div>2</div>
+    <div>3</div>
+    <a href="page_2.html">Next</a>
+    <a href="page_1.html">Page 1</a>
+    <a href="page_2.html">Page 2</a>
+  </body>
+</html>

data/test/blackbox_tests/next_page/page_2.html ADDED Viewed

@@ -0,0 +1,11 @@
+<html>
+  <body>
+    <div>4</div>
+    <div>5</div>
+    <div>6</div>
+    <a href="page_3.html">Next</a>
+    <a href="page_2.html">Page 2</a>
+    <a href="page_3.html">Page 3</a>
+  </body>
+</html>

data/test/blackbox_tests/next_page/page_3.html ADDED Viewed

@@ -0,0 +1,7 @@
+<html>
+  <body>
+    <div>7</div>
+    <div>8</div>
+    <div>9</div>
+  </body>
+</html>

data/test/blackbox_tests/next_page/page_list_links.expected.xml ADDED Viewed

@@ -0,0 +1,11 @@
+<root>
+  <entry>1</entry>
+  <entry>2</entry>
+  <entry>3</entry>
+  <entry>4</entry>
+  <entry>5</entry>
+  <entry>6</entry>
+  <entry>7</entry>
+  <entry>8</entry>
+  <entry>9</entry>
+</root>

data/test/blackbox_tests/next_page/page_list_links.rb ADDED Viewed

@@ -0,0 +1,7 @@
+lambda do
+  fetch(File.join(File.dirname(__FILE__), "page_1.html"))
+  entry '1'
+  page_list 'Page 2'
+end

data/test/blackbox_tests/next_page/page_list_links.tofix ADDED Viewed

@@ -0,0 +1,7 @@
+lambda do
+  fetch(File.join(File.dirname(__FILE__), "page_1.html"))
+  entry '1'
+  page_list 'Page 2'
+end

data/todo/backlog.txt ADDED Viewed

@@ -0,0 +1,73 @@
+scRUBYt! Backlog
+[NEW] Rails runner - possibility to upload (or type into a form) an extractor, launch it and get back the result
+[NEW] when crawling to more pages, save the XMLs one by one rather than once at the end
+[NEW] select indices should evaluate as much results as needed (if possible - if there is :last for example, this is not possiblle)
+[IDEA] One thing I’d like to suggest is the use of a proxy to record interactions with a webpage and automatically allow the user to select items to parse or record a user’s interactions, generate the right script to get the user to that page (including POSTs / GETS) and let the user then select items to parse out of the target page.
+Discuss this with the others
+[NEW] exporting of crawling-to-next-page pattern (xpath, '.*' => '.*/page{num}')
+TODO: CHEATSHEET!
+TODO: all_img is substitute for
+      all_img '//img' etc.
+      i.e. if a pattern name is also a html tag, behave like this
+TODO: lotsa unit tests
+TODO: to_csv, to_txt, to_rss, to_ruport, to_class, to_db, to_whatever (Tim Fletcher)
+TODO: simple xslt transformation
+TODO: ensure_value_is_greater_than 10
+TODO: Performance optimization (check that nothing is loaded twice, probably just reference the
+      input document instead of using it as a hash key twice and hash value once etc.)
+      Discuss this with the others
+TODO: Possibility of explicitly define example type - for example it may be possible that
+      I want to match an XPath on the page and thus the example may look like '/html/body/table[1]'
+      yet it is not an XPath but a String example; in these cases, possibility to say:
+      xpath_stuff '/html/body/table[1]', :type => :string
+TODO: Compound pattern, e.g.:
+      <item>
+        <price>$1.99</price>
+        <shipping>$8.01</shipping>
+        <total>$10.00</total>      --> This is the compund pattern
+      </item>
+      This should look something like:
+      item do
+        price '$1.99'
+        shipping '$8.01'
+        total 'price', 'shipping', :type => compound, :function => :add_prices
+      end
+      def add_prices(price, shipping)
+        price(1..price.size).to_i + price(1..shipping.size).to_i
+      end
+TODO: setup Joomla (move it to extractors.scrubyt.org)
+TODO: refactor the whole constraint system
+TODO: unit tests!!!
+TODO: Widgets module - e.g. Widgets::Dropdown
+usage (victor from mexico):
+Widgets::Dropdown('houses').each_value do |house|
+  Widgets::Dropdown('districts').each_value do  |district|
+    scrape ( house, district )
+TODO: RSS feed on the recent changes
+TODO: to_db(pattern_name)
+e.g.
+root
+--- book
+    --- title
+    --- author
+to_db(book) will create a table book with columns title and author
+TODO: self correcting wrappers; see Assaf's ideas (after learning, save the CSS selectors, IDs, classes etc) and
+re-learn from those)

data/todo/scenario_ideas.txt ADDED Viewed

@@ -0,0 +1,19 @@
+check out OPENKAPOW!!!
+- ibm spidering article rewrite
+- table to csv
+- cia factbook
+- book price comparison
+- cnn news headlines
+- google news
+- Scenario "antisocial": watch del.icio.us, dzone and digg articles on one page!
+    * (possibly requires more wrappers in one extractor - or more extractors?) + some nice
+    * XSLT with CSS
+    * something like:
+        .) scenario.watch(:delicious, 'Peter's super article')
+        .) scenario.watch(:digg, 'xxx')
+- dreamhost spam subjects
+- James Brit 'coffee shop' example