olek-libcraigscrape 1.0.1

data/COPYING.LESSER

@@ -0,0 +1,165 @@
+       GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/README

@@ -0,0 +1,89 @@
+= libcraigscrape - A craigslist URL-scraping support Library
+
+An easy library to do the heavy lifting between you and Craigslist's posting database. Given a URL, libcraigscrape will
+follow links, scrape fields, and make ruby-sense out of the raw html from craigslist's servers.
+
+For more information, head to the {craiglist monitoring}[http://www.derosetechnologies.com/community/libcraigscrape] help section of our website.
+
+== craigwatch
+libcraigscrape was primarily developed to support the included craigwatch[link:files/bin/craigwatch.html] script. See the included craigwatch script for
+examples of libcraigscape in action, and (hopefully) to serve an immediate craigscraping need.
+
+== Installation
+
+Install via RubyGems:
+
+  sudo gem install libcraigscrape
+
+== Usage
+
+=== Scrape Craigslist Listings since Sep 10
+
+On the 'miami.craigslist.org' site, using the query "search/sss?query=apple"
+
+  require 'rubygems'
+  require 'libcraigscrape'
+  require 'date'
+  require 'pp'
+
+  miami_cl = CraigScrape.new 'us/fl/miami'
+  miami_cl.posts_since(Time.parse('Sep 10'), 'search/sss?query=apple').each do |post|
+    pp post
+  end
+
+=== Scrape Last 225 Craigslist Listings
+
+On the 'miami.craigslist.org'  under the 'apa' category
+
+  require 'rubygems'
+  require 'libcraigscrape'
+  require 'pp'
+
+  i=1
+  CraigScrape.new('us/fl/miami').each_post('apa') do |post|
+    break if i > 225
+     i+=1
+     pp post
+  end
+
+=== Multiple site with multiple section/search enumeration of posts
+
+In Florida, with the exception of 'miami.craigslist.org' & 'keys.craigslist.org' sites, output each post in
+the 'crg' category and for the search 'artist needed'
+
+  require 'rubygems'
+  require 'libcraigscrape'
+  require 'pp'
+
+  non_sfl_sites = CraigScrape.new('us/fl', '- us/fl/miami', '- us/fl/keys')
+  non_sfl_sites.each_post('crg', 'search/sss?query=artist+needed') do |post|
+     pp post
+  end
+
+=== Scrape Single Craigslist Posting
+
+This grabs the full details under the specific post http://miami.craigslist.org/mdc/sys/1140808860.html
+
+  require 'rubygems'
+  require 'libcraigscrape'
+
+  post = CraigScrape::Posting.new 'http://miami.craigslist.org/mdc/sys/1140808860.html'
+  puts "(%s) %s:\n %s" % [ post.post_time.strftime('%b %d'), post.title, post.contents_as_plain ]
+
+=== Scrape Single Craigslist Listing
+
+This grabs the post summaries of the single listings at http://miami.craigslist.org/search/sss?query=laptop
+
+  require 'rubygems'
+  require 'libcraigscrape'
+
+  listing = CraigScrape::Listings.new 'http://miami.craigslist.org/search/sss?query=laptop'
+  puts 'Found %d posts for the search "laptop" on this page' % listing.posts.length
+
+== Author
+- Chris DeRose (cderose@derosetechnologies.com)
+- DeRose Technologies, Inc. http://www.derosetechnologies.com
+
+== License
+
+See COPYING[link:files/COPYING.html]

data/Rakefile

@@ -0,0 +1,125 @@
+require 'rake'
+require 'rake/clean'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/testtask'
+require 'fileutils'
+require 'tempfile'
+
+include FileUtils
+
+RbConfig = Config unless defined? RbConfig
+
+NAME = "olek-libcraigscrape"
+VERS = ENV['VERSION'] || "1.0.1"
+PKG = "#{NAME}-#{VERS}"
+
+RDOC_OPTS = ['--quiet', '--title', 'The libcraigscrape Reference', '--main', 'README', '--inline-source']
+RDOC_FILES = ['README', "CHANGELOG", "COPYING","COPYING.LESSER", 'bin/craigwatch']
+PKG_FILES = (%w(Rakefile) + RDOC_FILES + Dir.glob("{bin,test,lib}/**/*")).uniq.sort_by{|a,b| (a == 'lib/libcraigscrape.rb') ? -1 : 0 }
+
+SPEC =
+  Gem::Specification.new do |s|
+    s.name = NAME
+    s.version = VERS
+    s.platform = Gem::Platform::RUBY
+    s.has_rdoc = true
+    s.bindir = 'bin'
+    s.executables = 'craigwatch'
+    s.rdoc_options += RDOC_OPTS
+    s.extra_rdoc_files = RDOC_FILES
+    s.summary = "quick, easy, craigslist parsing library that takes the monotony out of working with craigslist posts and listings"
+    s.description = s.summary
+    s.author = "Chris DeRose, DeRose Technologies, Inc."
+    s.email = 'cderose@derosetechnologies.com'
+    s.homepage = 'http://www.derosetechnologies.com/community/libcraigscrape'
+    s.rubyforge_project = 'libcraigwatch'
+    s.files = PKG_FILES
+    s.require_paths = ["lib"]
+    s.test_files = FileList['test/test_*.rb']
+    s.add_dependency 'nokogiri',     '>= 1.4.4'
+    s.add_dependency 'htmlentities', '>= 4.0.0'
+    s.add_dependency 'activesupport','>= 2.3.0', '< 3'
+    s.add_dependency 'activerecord', '>= 2.3.0', '< 3'
+    s.add_dependency 'actionmailer', '>= 2.3.0', '< 3'
+    s.add_dependency 'kwalify', '>= 0.7.2'
+    s.add_dependency 'sqlite3'
+  end
+
+desc "Run all the tests"
+Rake::TestTask.new do |t|
+    t.libs << "test"
+    t.test_files = FileList['test/test_*.rb']
+    t.verbose = true
+end
+
+Rake::RDocTask.new do |rdoc|
+    rdoc.rdoc_dir = 'doc/rdoc'
+    rdoc.options += RDOC_OPTS
+    rdoc.main = "README"
+    # NOTE: If you don't put libcraigscrape.rb at the beginning, the rdoc ends up looking a little screwy
+    rdoc.rdoc_files.add RDOC_FILES+Dir.glob('lib/*.rb').sort_by{|a,b| (a == 'lib/libcraigscrape.rb') ? -1 : 0 }
+end
+
+Rake::GemPackageTask.new(SPEC) do |p|
+  p.need_tar = false
+  p.need_tar_gz = false
+  p.need_tar_bz2 = false
+  p.need_zip = false
+  p.gem_spec = SPEC
+end
+
+task "lib" do
+  directory "lib"
+end
+
+task :install do
+  sh %{rake package}
+  sh %{sudo gem install pkg/#{NAME}-#{VERS}}
+end
+
+task :uninstall => [:clean] do
+  sh %{sudo gem uninstall #{NAME}}
+end
+
+require 'roodi'
+require 'roodi_task'
+
+namespace :code_tests do
+  desc "Analyze for code complexity"
+  task :flog do
+    require 'flog'
+
+    flog = Flog.new
+    flog.flog_files ['lib']
+    threshold = 105
+
+    bad_methods = flog.totals.select do |name, score|
+       score > threshold
+    end
+
+    bad_methods.sort { |a,b| a[1] <=> b[1] }.each do |name, score|
+      puts "%8.1f: %s" % [score, name]
+    end
+
+    puts "WARNING : #{bad_methods.size} methods have a flog complexity > #{threshold}" unless bad_methods.empty?
+  end
+
+  desc "Analyze for code duplication"
+    require 'flay'
+    task :flay do
+    threshold = 25
+    flay = Flay.new({:fuzzy => false, :verbose => false, :mass => threshold})
+    flay.process(*Flay.expand_dirs_to_files(['lib']))
+
+    flay.report
+
+    raise "#{flay.masses.size} chunks of code have a duplicate mass > #{threshold}" unless flay.masses.empty?
+  end
+
+  RoodiTask.new 'roodi', ['lib/*.rb'], 'roodi.yml'
+end
+
+desc "Run all code tests"
+task :code_tests => %w(code_tests:flog code_tests:flay code_tests:roodi)
+

data/bin/craig_report_schema.yml

@@ -0,0 +1,68 @@
+type: map
+class: CraigReportDefinition
+mapping:
+   "debug_database":    { type: bool, required: no }
+   "debug_mailer":      { type: bool, required: no }
+   "debug_craigscrape": { type: bool, required: no }
+
+   "report_name":   { type: str, required: no  }
+   "email_to":      { type: str, required: yes }
+   "email_from":    { type: str, required: no  }
+   "smtp_settings":
+      type: map
+      required: no
+      mapping:
+         "address":        { type: str, required: yes }
+         "port":           { type: int, required: no, default: 25 }
+         "user_name":      { type: str, required: no }
+         "domain":         { type: str, required: no }
+         "password":       { type: str, required: no }
+         "authentication": { type: str, required: no }
+         "enable_starttls_auto": { type: bool, required: no }
+         "tls":            { type: bool, required: no }
+   "tracking_database":
+      type: map
+      mapping:
+         "adapter":  { type: str, required: yes }
+         "dbfile":   { type: str, required: no }
+         "host":    { type: str, required: no }
+         "username": { type: str, required: no }
+         "password": { type: str, required: no }
+         "socket":   { type: str, required: no }
+         "database": { type: str, required: no }
+   "searches":
+      type:  seq
+      sequence:
+         - type: map
+           class: CraigReportDefinition::SearchDefinition
+           mapping:
+              "name":                        {type: str,  required: yes, unique: yes}
+              "has_image":                   {type: bool, required: no}
+              "newest_first":                {type: bool, required: no, default: no}
+              "price_required":              {type: bool, required: no, default: no}
+              "price_greater_than":          {type: int, required: no}
+              "price_less_than":             {type: int, required: no}
+              "full_post_has":               {type: seq, required: no, sequence: [ {type: str, unique: yes} ]}
+              "full_post_has_no":            {type: seq, required: no, sequence: [ {type: str, unique: yes} ]}
+              "summary_post_has":            {type: seq, required: no, sequence: [ {type: str, unique: yes} ]}
+              "summary_post_has_no":         {type: seq, required: no, sequence: [ {type: str, unique: yes} ]}
+              "summary_or_full_post_has":    {type: seq, required: no, sequence: [ {type: str, unique: yes} ]}
+              "summary_or_full_post_has_no": {type: seq, required: no, sequence: [ {type: str, unique: yes} ]}
+              "location_has":                {type: seq, required: no, sequence: [ {type: str, unique: yes} ]}
+              "location_has_no":             {type: seq, required: no, sequence: [ {type: str, unique: yes} ]}
+              "sites":
+                 type: seq
+                 required: yes
+                 sequence:
+                   - type: str
+                     unique: yes
+              "listings":
+                 type: seq
+                 required: yes
+                 sequence:
+                   - type: str
+                     unique: yes
+              "starting":
+                 type: str
+                 required: no
+                 pattern: /^[\d]{1,2}\/[\d]{1,2}\/(?:[\d]{2}|[\d]{4})$/