sphinxcrawl 0.0.1

data/.gitignore

@@ -0,0 +1,7 @@
+.DS_Store
+results.html
+pkg
+html
+.bundle
+tags
+tmp

data/Gemfile

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sphinxcrawl.gemspec
+gemspec
+
+gem 'guard'
+gem 'guard-rspec'
+gem 'guard-ctags-bundler'
+gem 'guard-cucumber'
+
+group :darwin do
+  gem 'rb-fsevent'
+end
+
+group :linux do
+  gem 'rb-inotify'
+end

data/Gemfile.lock

@@ -0,0 +1,89 @@
+PATH
+  remote: .
+  specs:
+    sphinxcrawl (0.0.1)
+      methadone (~> 1.2.2)
+      nokogiri (~> 1.5.5)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.2)
+    aruba (0.5.0)
+      childprocess (= 0.2.3)
+      cucumber (>= 1.1.1)
+      ffi (>= 1.0.11)
+      rspec-expectations (>= 2.7.0)
+    builder (3.1.4)
+    childprocess (0.2.3)
+      ffi (~> 1.0.6)
+    coderay (1.0.8)
+    crack (0.3.1)
+    cucumber (1.2.1)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.3)
+      gherkin (~> 2.11.0)
+      json (>= 1.4.6)
+    diff-lcs (1.1.3)
+    ffi (1.0.11)
+    gherkin (2.11.5)
+      json (>= 1.4.6)
+    guard (1.5.3)
+      listen (>= 0.4.2)
+      lumberjack (>= 1.0.2)
+      pry (>= 0.9.10)
+      thor (>= 0.14.6)
+    guard-ctags-bundler (0.1.6)
+      guard (>= 1.1)
+    guard-cucumber (1.2.2)
+      cucumber (>= 1.2.0)
+      guard (>= 1.1.0)
+    guard-rspec (2.1.1)
+      guard (>= 1.1)
+      rspec (~> 2.11)
+    json (1.7.5)
+    listen (0.5.3)
+    lumberjack (1.0.2)
+    methadone (1.2.2)
+      bundler
+    method_source (0.8.1)
+    nokogiri (1.5.5)
+    pry (0.9.10)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.3.1)
+    rake (0.9.2.2)
+    rb-fsevent (0.9.2)
+    rb-inotify (0.8.8)
+      ffi (>= 0.5.0)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.3)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.3)
+    slop (3.3.3)
+    thor (0.16.0)
+    webmock (1.8.10)
+      addressable (>= 2.2.7)
+      crack (>= 0.1.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aruba
+  guard
+  guard-ctags-bundler
+  guard-cucumber
+  guard-rspec
+  rake
+  rb-fsevent
+  rb-inotify
+  rdoc
+  sphinxcrawl!
+  webmock

data/Guardfile

@@ -0,0 +1,16 @@
+guard 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { 'spec' }
+end
+
+guard 'ctags-bundler', :src_path => ['lib', 'spec'] do
+  watch(/^(lib)\/.*\.rb$/)
+  watch('Gemfile.lock')
+end
+
+guard 'cucumber' do
+  watch(%r{^features/.+\.feature$})
+  watch(%r{^features/support/.+$}) { 'features' }
+  watch(%r{^features/step_definitions/(.+)_steps\.rb$}) { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'features' }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Jon Doveston
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,69 @@
+# Sphinxcrawl
+
+Simple command line tool gem to crawl a website and generate an xml stream for a
+sphinx search index.
+
+## Installation
+
+Install it with:
+
+    $ gem install sphinxcrawl
+
+## Usage
+
+After installation the sphinxcrawl command should be available.
+
+Create a sphinx.conf file, for example
+
+
+    source page
+    {
+      type = xmlpipe2
+      xmlpipe_command = sphinxcrawl http://www.example.com -d 2 2>/dev/null
+    }
+
+    index page
+    {
+      source = page
+      path = sphinx/page
+      morphology = stem_en
+      charset_type = utf-8
+      html_strip = 1
+    }
+
+Install sphinx, with debianoid linuxes this is
+
+    sudo apt-get install sphinxsearch
+
+You should now have the indexer and search commands
+
+    indexer -c sphinx.conf page
+
+and you can search in the index with
+
+    search -c sphinx.conf name
+
+## Requirements
+
+This gem is not Google! It uses nokogiri to parse html so will fail on badly
+formed html. Also more importantly it only indexes part of the page marked with
+a specific html attribute. This means you can only index sites that you control.
+For example
+
+    ...
+    <div data-field="description">
+    <p>This is a description</p>
+    </div>
+    ...
+
+will index the text inside the div tag (stripping other tags) and put the text
+in a sphinx field called description. All fields are aggregated from all the
+pages from a site crawl.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/README.rdoc

@@ -0,0 +1,13 @@
+= sphinxcrawl - html crawler and sphinx xmlstream generator
+
+Author:: Jon Doveston (jon@doveston.me.uk)
+Copyright:: Copyright (c) 2012 Jon Doveston
+
+
+A command line gem that can crawl html (files or http) and generate an xml
+stream for sphinx search.
+
+== Links
+
+* {Source on Github}[https://github.com/hatoishi/sphinxcrawl]
+

data/Rakefile

@@ -0,0 +1,66 @@
+def dump_load_path
+  puts $LOAD_PATH.join("\n")
+  found = nil
+  $LOAD_PATH.each do |path|
+    if File.exists?(File.join(path,"rspec"))
+      puts "Found rspec in #{path}"
+      if File.exists?(File.join(path,"rspec","core"))
+        puts "Found core"
+        if File.exists?(File.join(path,"rspec","core","rake_task"))
+          puts "Found rake_task"
+          found = path
+        else
+          puts "!! no rake_task"
+        end
+      else
+        puts "!!! no core"
+      end
+    end
+  end
+  if found.nil?
+    puts "Didn't find rspec/core/rake_task anywhere"
+  else
+    puts "Found in #{path}"
+  end
+end
+require 'bundler'
+require 'rake/clean'
+
+begin
+require 'rspec/core/rake_task'
+rescue LoadError
+dump_load_path
+raise
+end
+
+require 'cucumber'
+require 'cucumber/rake/task'
+gem 'rdoc' # we need the installed RDoc gem, not the system one
+require 'rdoc/task'
+
+include Rake::DSL
+
+Bundler::GemHelper.install_tasks
+
+
+RSpec::Core::RakeTask.new do |t|
+  # Put spec opts in a file named .rspec in root
+end
+
+
+CUKE_RESULTS = 'results.html'
+CLEAN << CUKE_RESULTS
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty --no-source -x"
+  t.fork = false
+end
+
+Rake::RDocTask.new do |rd|
+  
+  rd.main = "README.rdoc"
+  
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+end
+
+task :default => [:spec,:features]
+

data/bin/sphinxcrawl

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'methadone'
+require 'sphinxcrawl'
+
+class App
+  include Methadone::Main
+  include Methadone::CLILogging
+
+  main do |index|
+    index ||= 'index.html'
+    if index =~ /http:/
+      crawler = Sphinxcrawl::WebCrawler.new(index, options[:depth].to_i)
+    else
+      exit_now!(1, "file #{index} not found") unless File.exists?(index)
+      index = index+'/index.html' if File.directory?(index)
+      exit_now!(1, "file #{index} not found") unless File.exists?(index)
+      crawler = Sphinxcrawl::FileCrawler.new(index, options[:depth].to_i)
+    end
+    pages = crawler.pages
+    stream = Sphinxcrawl::Stream.new(pages)
+    puts stream.to_xml
+  end
+
+  description 'Generate sphinx xml stream from html pages'
+  arg :index, :optional
+
+  options['depth'] = '0'
+  on('-d depth', '--depth', 'Crawl depth', /^\d+$/)
+
+  version Sphinxcrawl::VERSION
+  use_log_level_option
+  go!
+end

data/example.conf

@@ -0,0 +1,14 @@
+source page
+{
+  type = xmlpipe2
+  xmlpipe_command = bundle exec ./bin/sphinxcrawl spec/fixtures/index_file.html -d 2 2>/dev/null
+}
+
+index page
+{
+  source = page
+  path = tmp/page
+  morphology = stem_en
+  charset_type = utf-8
+  html_strip = 1
+}

data/features/sphinxcrawl.feature

@@ -0,0 +1,70 @@
+Feature: Generation of sphinx xml
+  Scenario: help page
+    When I get help for "sphinxcrawl"
+    Then the exit status should be 0
+    And the banner should be present
+    And the banner should document that this app takes options
+    And the following options should be documented:
+      |--version|
+
+  Scenario: xml for a nonexistent file
+    When I run `sphinxcrawl spec/fixtures/blah.html`
+    Then the exit status should be 1
+
+  Scenario: xml for a nonexistent index
+    When I run `sphinxcrawl spec`
+    Then the exit status should be 1
+
+  Scenario: xml for a single file
+    Given a file named "test.html" with:
+    """
+    <html>
+    <head>
+    </head>
+    <body>
+    <h1 data-field="name">My name</h1>
+    <p data-field="description">My description</p>
+    <p data-field="body"><span>My</span><span>body</span></p>
+    </body>
+    </html>
+    """
+    When I run `sphinxcrawl test.html`
+    Then the exit status should be 0
+    And the output should contain "name"
+    And the output should contain "description"
+    And the output should contain "body"
+
+  Scenario: index the xml
+    Given a file named "test.html" with:
+    """
+    <html>
+    <head>
+    </head>
+    <body>
+    <h1 data-field="name">My name</h1>
+    <p data-field="description">My description</p>
+    <p data-field="body"><span>My</span><span>body</span></p>
+    </body>
+    </html>
+    """
+    And a file named "test.conf" with:
+    """
+    indexer
+    {
+    }
+    source page
+    {
+    type = xmlpipe2
+    xmlpipe_command = ../../bin/sphinxcrawl test.html 2>/dev/null
+    }
+    index page
+    {
+    source = page
+    path = page
+    morphology = stem_en
+    charset_type = utf-8
+    html_strip = 1
+    }
+    """
+    When I run `indexer -c test.conf page`
+    Then the exit status should be 0

data/features/step_definitions/sphinxcrawl_steps.rb

@@ -0,0 +1 @@
+# Put your step definitions here

data/features/support/env.rb

@@ -0,0 +1,16 @@
+require 'aruba/cucumber'
+require 'methadone/cucumber'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+  # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+end

data/lib/sphinxcrawl.rb

@@ -0,0 +1,9 @@
+require 'sphinxcrawl/version'
+require 'sphinxcrawl/page'
+require 'sphinxcrawl/stream'
+require 'sphinxcrawl/crawler'
+require 'sphinxcrawl/file_crawler'
+require 'sphinxcrawl/web_crawler'
+
+module Sphinxcrawl
+end

data/lib/sphinxcrawl/crawler.rb

@@ -0,0 +1,36 @@
+require 'set'
+
+module Sphinxcrawl
+  class Crawler
+    attr_reader :depth
+
+    def initialize(depth=0)
+      @depth = depth
+    end
+
+    def pages
+      return @pages if @pages
+      return [] unless index
+      @pages = Set.new([index])
+      return @pages if @depth == 0
+
+      current_pages = Set.new([index])
+      depth.times do
+        links = current_pages.map(&:links).flatten.compact.uniq
+        current_pages = Set.new(links.map{ |url| get_page(url) }.compact) - @pages
+        @pages += current_pages
+      end
+      @pages
+    end
+
+    private
+
+    def index
+      nil
+    end
+
+    def get_page(url)
+      nil
+    end
+  end
+end

data/lib/sphinxcrawl/file_crawler.rb

@@ -0,0 +1,20 @@
+module Sphinxcrawl
+  class FileCrawler < Crawler
+    def initialize(index_file_name, depth=0)
+      @directory = File.dirname(index_file_name)
+      @basename = File.basename(index_file_name)
+      super(depth)
+    end
+
+    private
+
+    def index
+      @index ||= get_page(@basename)
+    end
+
+    def get_page(url)
+      html = File.read(@directory + '/' + url) rescue nil
+      Page.new(url, html) if html
+    end
+  end
+end

data/lib/sphinxcrawl/page.rb

@@ -0,0 +1,58 @@
+require 'nokogiri'
+
+module Sphinxcrawl
+  class Page
+    attr_reader :html, :url
+
+    def initialize(url, html)
+      @url = url
+      @html = html
+      @field_data = {}
+    end
+
+    def empty?
+      !html || html.empty?
+    end
+
+    def field_names
+      fields.map do |n|
+        n.attribute('data-field').value
+      end.uniq
+    end
+
+    def field(name)
+      @field_data[name] ||= get_field(name)
+    end
+
+    def links
+      @links ||= document.xpath('//a/@href').map(&:value).map do |url|
+        # only local links (no http://)
+        uri = URI.parse(url)
+        uri.host.nil? && uri.scheme.nil? ? url : nil
+      end.compact
+    end
+
+    def eql?(compare)
+      url == compare.url
+    end
+    alias :== :eql?
+
+    def hash
+      url.hash
+    end
+
+    private
+
+    def get_field(name)
+      document.xpath("//*[@data-field='#{name}']//text()").map(&:content).join(' ')
+    end
+
+    def fields
+      @fields ||= document.xpath('//*[@data-field]')
+    end
+
+    def document
+      @document ||= Nokogiri::HTML(html)
+    end
+  end
+end

data/lib/sphinxcrawl/stream.rb

@@ -0,0 +1,85 @@
+require 'rexml/document'
+
+module Sphinxcrawl
+  class Stream
+    include REXML
+
+    attr_reader :pages
+
+    def initialize(pages)
+      @pages = Array(pages)
+      @count = 0
+    end
+
+    def empty?
+      pages.length == 0
+    end
+
+    def number_of_pages
+      pages.length
+    end
+
+    def field_names
+      @field_names ||= pages.map(&:field_names).flatten.uniq
+    end
+
+    def to_xml
+      add_fields
+      add_documents
+      ''.tap { |xml| document.write(xml) }
+    end
+
+    private
+
+    def next_id
+      @count += 1
+    end
+
+    def add_fields
+      field_names.each do |name|
+        add_field(name)
+      end
+    end
+
+    def add_documents
+      pages.each do |page|
+        add_document(page)
+      end
+    end
+
+    def add_field(name)
+      schema.add_element('sphinx:field').tap do |field|
+        field.add_attribute('name', name)
+      end
+    end
+
+    def add_document(page)
+      if page.field_names.length > 0
+        root.add_element('sphinx:document').tap do |doc|
+          doc.add_attribute('id', next_id)
+          doc.add_element('url').text = CData.new(page.url)
+          field_names.each do |name|
+            doc.add_element(name).text = CData.new(page.field(name))
+          end
+        end
+      end
+    end
+
+    def document
+      @document ||= Document.new.tap do |doc|
+        doc << XMLDecl.new('1.0', 'UTF-8')
+        @docset = doc.add_element('sphinx:docset')
+      end
+    end
+
+    def root
+      document.root
+    end
+
+    def schema
+      @schema ||= root.add_element('sphinx:schema').tap do |sch|
+        sch.add_element('sphinx:attr').add_attributes('name' => 'url', 'type' => 'string')
+      end
+    end
+  end
+end

data/lib/sphinxcrawl/version.rb

@@ -0,0 +1,3 @@
+module Sphinxcrawl
+  VERSION = '0.0.1'
+end

data/lib/sphinxcrawl/web_crawler.rb

@@ -0,0 +1,43 @@
+require 'uri'
+require 'net/http'
+
+module Sphinxcrawl
+  class WebCrawler < Crawler
+    def initialize(url, depth=0)
+      @uri = URI.parse(url)
+      super(depth)
+    end
+
+    private
+
+    def host
+      @uri.host
+    end
+
+    def port
+      @uri.port
+    end
+
+    def index_path
+      @uri.path
+    end
+
+    def http
+      Net::HTTP.new(host, port)
+    end
+
+    def index
+      @index ||= get_page(index_path)
+    end
+
+    def get_page(path)
+      path = path && !path.empty? ? path : '/'
+      location = path
+      begin
+        response = http.request(Net::HTTP::Get.new(location))
+      end while location = response.header['location']
+      html = response.body
+      Page.new(path, html) if html
+    end
+  end
+end

data/spec/fixtures/child_file.html

@@ -0,0 +1,11 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <h1 data-field="name">My name is child</h1>
+    <p data-field="description">My description is child</p>
+    <p data-field="body"><span>My</span><span>body</span>is child</p>
+    <a href="index_file.html">index</a>
+    <a href="tree_file.html">tree</a>
+  </body>
+</html>

data/spec/fixtures/child_web.html

@@ -0,0 +1,11 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <h1 data-field="name">My name is child</h1>
+    <p data-field="description">My description is child</p>
+    <p data-field="body"><span>My</span><span>body</span>is child</p>
+    <a href="/index_web.html">index</a>
+    <a href="/tree_web.html">tree</a>
+  </body>
+</html>

data/spec/fixtures/index_file.html

@@ -0,0 +1,12 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <h1 data-field="name">My name</h1>
+    <p data-field="description">My description</p>
+    <p data-field="body"><span>My</span><span>body</span></p>
+    <a href="tree_file.html">tree</a>
+    <a href="index_file.html">index</a>
+    <a href="http://www.google.com">google</a>
+  </body>
+</html>

data/spec/fixtures/index_web.html

@@ -0,0 +1,11 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <h1 data-field="name">My name</h1>
+    <p data-field="description">My description</p>
+    <p data-field="body"><span>My</span><span>body</span></p>
+    <a href="/tree_web.html">tree</a>
+    <a href="/index_web.html">index</a>
+  </body>
+</html>

data/spec/fixtures/tree_file.html

@@ -0,0 +1,12 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <h1 data-field="name">My name is tree</h1>
+    <p data-field="description">My description is tree</p>
+    <p data-field="body"><span>My</span><span>body</span>is tree</p>
+    <a href="index_file.html">index</a>
+    <a href="child_file.html">child</a>
+    <a href="tree_file.html">tree</a>
+  </body>
+</html>

data/spec/fixtures/tree_web.html

@@ -0,0 +1,12 @@
+<html>
+  <head>
+  </head>
+  <body>
+    <h1 data-field="name">My name is tree</h1>
+    <p data-field="description">My description is tree</p>
+    <p data-field="body"><span>My</span><span>body</span>is tree</p>
+    <a href="/index_web.html">index</a>
+    <a href="/child_web.html">child</a>
+    <a href="/tree_web.html">tree</a>
+  </body>
+</html>

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+require 'sphinxcrawl'
+require 'webmock/rspec'

data/spec/sphinxcrawl/file_crawler_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe Sphinxcrawl::FileCrawler do
+  let(:dir) { 'spec/fixtures' }
+  let(:index) { 'index_file.html' }
+  let(:html) { dir + '/' + index }
+  subject { Sphinxcrawl::FileCrawler.new(html) }
+
+  its(:depth) { should be == 0 }
+  specify { subject.pages.length.should be == 1 }
+
+  context "with a depth of 1" do
+    subject { Sphinxcrawl::FileCrawler.new(html, 1) }
+    its(:depth) { should be == 1 }
+    specify { subject.pages.length.should be == 2 }
+  end
+
+  context "with a depth of 2" do
+    subject { Sphinxcrawl::FileCrawler.new(html, 2) }
+    its(:depth) { should be == 2 }
+    specify { subject.pages.length.should be == 3 }
+  end
+end

data/spec/sphinxcrawl/page_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe Sphinxcrawl::Page do
+  let(:html) { File.read('spec/fixtures/index_file.html') }
+  let(:same) { Sphinxcrawl::Page.new(subject.url, 'blah') }
+  subject { Sphinxcrawl::Page.new('index_file.html', html) }
+
+  its(:html) { should be == html }
+  it { should_not be_empty }
+
+  its(:field_names) { should be == %w[name description body] }
+  specify { subject.field('name').should be == 'My name' }
+  specify { subject.field('description').should be == 'My description' }
+  specify { subject.field('body').should be == 'My body' }
+
+  its(:links) { should be == ['tree_file.html', 'index_file.html'] }
+
+  specify { subject.should be == same }
+  specify { subject.should eq(same) }
+  specify { subject.should eql(same) }
+  specify { subject.should_not equal same }
+  specify { [subject, same].uniq.length.should be == 1 }
+end

data/spec/sphinxcrawl/stream_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Sphinxcrawl::Stream do
+  let(:html) { File.read('spec/fixtures/index_file.html') }
+  let(:page) { Sphinxcrawl::Page.new('index_file.html', html) }
+  subject { Sphinxcrawl::Stream.new(page) }
+  let(:xml) { subject.to_xml }
+
+  it { should_not be_empty }
+  its(:number_of_pages) { should be == 1 }
+
+  its(:field_names) { should be == %w[name description body] }
+
+  its(:to_xml) { should include('name') }
+  its(:to_xml) { should include('description') }
+  its(:to_xml) { should include('body') }
+  its(:to_xml) { should include('My name') }
+  its(:to_xml) { should include('My description') }
+  its(:to_xml) { should include('My body') }
+end

data/spec/sphinxcrawl/web_crawler_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Sphinxcrawl::WebCrawler do
+  let(:dir) { 'spec/fixtures' }
+  let(:domain) { 'http://www.example.com' }
+  let(:index) { 'index_web.html' }
+  let(:tree) { 'tree_web.html' }
+  let(:child) { 'child_web.html' }
+
+  before do
+    stub_request(:get, domain+'/'+index).to_return(:body => File.new(dir+'/'+index), :status => 200)
+    stub_request(:get, domain+'/'+tree).to_return(:body => File.new(dir+'/'+tree), :status => 200)
+    stub_request(:get, domain+'/'+child).to_return(:body => File.new(dir+'/'+child), :status => 200)
+  end
+
+  subject { Sphinxcrawl::WebCrawler.new(domain+'/'+index) }
+
+  its(:depth) { should be == 0 }
+  specify { subject.pages.length.should be == 1 }
+
+  context "with a depth of 1" do
+    subject { Sphinxcrawl::WebCrawler.new(domain+'/'+index, 1) }
+    its(:depth) { should be == 1 }
+    specify { subject.pages.length.should be == 2 }
+  end
+
+  context "with a depth of 2" do
+    subject { Sphinxcrawl::WebCrawler.new(domain+'/'+index, 2) }
+    its(:depth) { should be == 2 }
+    specify { subject.pages.length.should be == 3 }
+  end
+end

data/sphinxcrawl.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sphinxcrawl/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'sphinxcrawl'
+  gem.version       = Sphinxcrawl::VERSION
+  gem.authors       = ['Jon Doveston']
+  gem.email         = ['jon@doveston.me.uk']
+  gem.description   = %q{Simple command to crawl a site and process html into a sphinx xmlstream}
+  gem.summary       = %q{Simple command to crawl a site and process html into s sphinx xmlstream}
+  gem.homepage      = 'https://github.com/hatoishi/sphinxcrawl'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+  gem.add_dependency('methadone', '~> 1.2.2')
+  gem.add_dependency('nokogiri', '~> 1.5.5')
+
+  gem.add_development_dependency('rdoc')
+  gem.add_development_dependency('aruba')
+  gem.add_development_dependency('rake')
+  gem.add_development_dependency('webmock')
+end

metadata

@@ -0,0 +1,194 @@
+--- !ruby/object:Gem::Specification
+name: sphinxcrawl
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jon Doveston
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: methadone
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple command to crawl a site and process html into a sphinx xmlstream
+email:
+- jon@doveston.me.uk
+executables:
+- sphinxcrawl
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- Guardfile
+- LICENSE.txt
+- README.markdown
+- README.rdoc
+- Rakefile
+- bin/sphinxcrawl
+- example.conf
+- features/sphinxcrawl.feature
+- features/step_definitions/sphinxcrawl_steps.rb
+- features/support/env.rb
+- lib/sphinxcrawl.rb
+- lib/sphinxcrawl/crawler.rb
+- lib/sphinxcrawl/file_crawler.rb
+- lib/sphinxcrawl/page.rb
+- lib/sphinxcrawl/stream.rb
+- lib/sphinxcrawl/version.rb
+- lib/sphinxcrawl/web_crawler.rb
+- spec/fixtures/child_file.html
+- spec/fixtures/child_web.html
+- spec/fixtures/index_file.html
+- spec/fixtures/index_web.html
+- spec/fixtures/tree_file.html
+- spec/fixtures/tree_web.html
+- spec/spec_helper.rb
+- spec/sphinxcrawl/file_crawler_spec.rb
+- spec/sphinxcrawl/page_spec.rb
+- spec/sphinxcrawl/stream_spec.rb
+- spec/sphinxcrawl/web_crawler_spec.rb
+- sphinxcrawl.gemspec
+homepage: https://github.com/hatoishi/sphinxcrawl
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3810366116509016959
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3810366116509016959
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Simple command to crawl a site and process html into s sphinx xmlstream
+test_files:
+- features/sphinxcrawl.feature
+- features/step_definitions/sphinxcrawl_steps.rb
+- features/support/env.rb
+- spec/fixtures/child_file.html
+- spec/fixtures/child_web.html
+- spec/fixtures/index_file.html
+- spec/fixtures/index_web.html
+- spec/fixtures/tree_file.html
+- spec/fixtures/tree_web.html
+- spec/spec_helper.rb
+- spec/sphinxcrawl/file_crawler_spec.rb
+- spec/sphinxcrawl/page_spec.rb
+- spec/sphinxcrawl/stream_spec.rb
+- spec/sphinxcrawl/web_crawler_spec.rb