bliss 0.0.7 → 0.0.9

data/CHANGELOG.rdoc

@@ -0,0 +1,12 @@
+== 0.0.9
+
+* Features
+
+  * added support for Gzip (content type is autodetected).
+  * on_max_unhandled_bytes callback block. Which receives the amount of bytes and a block to execute when that limit is reached.
+  * on_tag_close maybe used without a specific tag name, and block now handles "current depth" array.
+  * Featuring changelog.
+
+* Bugfixes
+
+  * When XML is too small, it ended through http.callback block, and did not closed the file. That lead to errors and is fixed in this version.

data/Gemfile

@@ -10,8 +10,8 @@ gem "em-http-request", ">= 1.0.2"
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
-  gem "shoulda", ">= 0"
-  gem "bundler", "~> 1.0.0"
+  gem "rspec", "~> 2.8.0"
+  gem "bundler", "~> 1.1.3"
   gem "jeweler", "~> 1.6.4"
   gem "simplecov"
   #gem "rcov", ">= 0"

data/Gemfile.lock

@@ -3,6 +3,7 @@ GEM
   specs:
     addressable (2.2.7)
     cookiejar (0.3.0)
+    diff-lcs (1.1.3)
     em-http-request (1.0.2)
       addressable (>= 2.2.3)
       cookiejar
@@ -21,11 +22,14 @@ GEM
     multi_json (1.1.0)
     nokogiri (1.5.2)
     rake (0.9.2.2)
-    shoulda (3.0.1)
-      shoulda-context (~> 1.0.0)
-      shoulda-matchers (~> 1.0.0)
-    shoulda-context (1.0.0)
-    shoulda-matchers (1.0.0)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
     simplecov (0.6.1)
       multi_json (~> 1.0)
       simplecov-html (~> 0.5.3)
@@ -35,10 +39,10 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.0.0)
+  bundler (~> 1.1.3)
   em-http-request (>= 1.0.2)
   eventmachine (>= 1.0.0.beta.4)
   jeweler (~> 1.6.4)
   nokogiri (>= 1.5.2)
-  shoulda
+  rspec (~> 2.8.0)
   simplecov

data/Rakefile

@@ -26,6 +26,12 @@ Jeweler::Tasks.new do |gem|
 end
 Jeweler::RubygemsDotOrgTasks.new
 
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
@@ -41,7 +47,7 @@ end
 #  test.rcov_opts << '--exclude "gems/*"'
 #end
 
-task :default => :test
+task :default => :spec
 
 require 'rake/rdoctask'
 Rake::RDocTask.new do |rdoc|

data/VERSION

@@ -1 +1 @@
-0.0.7
+0.0.9

data/bliss.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "bliss"
-  s.version = "0.0.7"
+  s.version = "0.0.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Fernando Alonso"]
-  s.date = "2012-03-21"
+  s.date = "2012-06-04"
   s.description = "streamed xml parsing tool"
   s.email = "krakatoa1987@gmail.com"
   s.extra_rdoc_files = [
@@ -18,6 +18,7 @@ Gem::Specification.new do |s|
   ]
   s.files = [
     ".document",
+    "CHANGELOG.rdoc",
     "Gemfile",
     "Gemfile.lock",
     "LICENSE.txt",
@@ -25,13 +26,20 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "bliss.gemspec",
+    "complete_test.rb",
+    "gzip_support.rb",
     "hash.rb",
     "http-machine.rb",
     "lib/bliss.rb",
+    "lib/bliss/constraint.rb",
+    "lib/bliss/encoding_error.rb",
+    "lib/bliss/format.rb",
     "lib/bliss/parser.rb",
     "lib/bliss/parser_machine.rb",
-    "lib/bliss/sax_parser.rb",
     "lib/hash_extension.rb",
+    "spec.yml",
+    "spec/format_spec.rb",
+    "spec/spec_helper.rb",
     "test.rb",
     "test/helper.rb",
     "test/test_bliss.rb"
@@ -39,7 +47,7 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/krakatoa/bliss"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
+  s.rubygems_version = "1.8.10"
   s.summary = "streamed xml parsing tool"
 
   if s.respond_to? :specification_version then
@@ -49,16 +57,16 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.5.2"])
       s.add_runtime_dependency(%q<eventmachine>, [">= 1.0.0.beta.4"])
       s.add_runtime_dependency(%q<em-http-request>, [">= 1.0.2"])
-      s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.1.3"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_development_dependency(%q<simplecov>, [">= 0"])
     else
       s.add_dependency(%q<nokogiri>, [">= 1.5.2"])
       s.add_dependency(%q<eventmachine>, [">= 1.0.0.beta.4"])
       s.add_dependency(%q<em-http-request>, [">= 1.0.2"])
-      s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+      s.add_dependency(%q<bundler>, ["~> 1.1.3"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_dependency(%q<simplecov>, [">= 0"])
     end
@@ -66,8 +74,8 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<nokogiri>, [">= 1.5.2"])
     s.add_dependency(%q<eventmachine>, [">= 1.0.0.beta.4"])
     s.add_dependency(%q<em-http-request>, [">= 1.0.2"])
-    s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<rspec>, ["~> 2.8.0"])
+    s.add_dependency(%q<bundler>, ["~> 1.1.3"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
     s.add_dependency(%q<simplecov>, [">= 0"])
   end

data/complete_test.rb

@@ -0,0 +1,74 @@
+require 'rubygems'
+require 'bliss'
+
+#path = 'http://www.universobit.com.ar/AvisosUniversobit/trovit/AvisosUniversobit_1.xml'
+#path = 'http://www.aestrenar.com.ar/backend/rssAestrenar.xml'
+#path = 'http://procarnet.es/feed/sumavisos/sumavisos.xml'
+#path = 'http://taakidom.pl/import/trovit/trovit.xml'
+#path = 'http://www.deautos.com/sumavisos/feed.xml'
+#path = 'http://www.autocosmos.com.ar/webservices/exchange/sumavisos.ar.xml.gz'
+
+#path = 'http://www.workgate.co.jp/feeds/sumavisos/sumavisos.xml'
+#path = 'http://www.bydgoszczak.pl/export/trovit_praca'
+#path = 'http://www.espacioinmobiliario.mx/feeds/feed.xml' # da timeout
+
+#path = 'http://www.indexempleos.com/peru/jobs.xml'
+
+#path = 'http://www.bydgoszczak.pl/export/trovit_praca'
+
+path = 'http://www.tokkoro.com/cron/adsdeck_feed.xml'
+
+# encoding
+#path = 'http://www.topdiffusion.com/flux/topdiffusion_adsdeck.xml'
+#path = 'http://www.workgate.co.jp/feeds/sumavisos/sumavisos.xml' # el problema es que viene con job, en lugar de ad
+#path = 'http://www.ultramotors.com.br/trovit/'
+
+#path = 'http://localhost:8080/maixon.xml'
+
+#path = 'http://www.kasaki.com.br/somanuncios.xml' # uppercase
+#path = 'http://canadajobsandcareers.ca/feed_adsdeck.php'
+
+#path = 'http://www.goemploi.com/france/jobs.xml'
+#path = 'http://www.monsieurjob.com/feeds/adsdeck.php'
+#path = 'http://www.imovelajato.com.br/feeds/olx/olx.xml'
+
+p = Bliss::ParserMachine.new(path, 'output.xml')
+p.wait_tag_close('AD')
+#p.max_unhandled_bytes = 20000
+
+count = 0
+p.on_root { |root|
+  #puts root
+}
+#p.on_tag_open { |depth|
+#  if depth.last =~ /[A-Z]/ then
+#    puts 'uppercase detected!'
+#    p.close
+#  end
+#}
+p.on_tag_open('AD') { |depth|
+  #puts depth.inspect
+}
+p.on_tag_close('AD') { |hash|
+  count += 1
+
+  dict = {"make"=>"name"}
+  only_in_dict = false
+  hash = hash.inject({}) { |h,v| key = dict.invert[v[0]]; key ||= v[0] unless only_in_dict; h[key] = v[1] if key; h }
+  
+  #puts hash.inspect
+  #puts hash['type']
+  #puts hash.keys.inspect
+  #if count == 100
+  #  p.close
+  #end
+}
+
+begin
+  p.parse
+rescue Bliss::EncodingError
+  puts "resqued!"
+end
+
+puts "Root: #{p.root}"
+puts "Count : #{count}"

data/gzip_support.rb

@@ -0,0 +1,41 @@
+require 'rubygems'
+require 'eventmachine'
+require 'em-http-request'
+
+@bytes = 0
+@io_read, @io_write = IO.pipe
+
+require 'zlib'
+
+f = File.new('test.xml', 'w')
+
+EM.run do
+  http = EM::HttpRequest.new(url, :inactivity_timeout => 1).get # :head => {'accept-encoding' => "gzip, deflate"}
+  gzipped = false
+  http.headers do
+    puts http.response_header.inspect
+    if (/^attachment.+filename.+\.gz/i === http.response_header['CONTENT_DISPOSITION']) or http.response_header.compressed? or ["application/octet-stream", "application/x-gzip"].include? http.response_header['CONTENT_TYPE']
+      gzipped = true
+    end
+  end
+  http.stream { |chunk|
+    if @bytes > 15000
+      #f.close
+      EM.stop
+    else
+      if gzipped
+        @zstream ||= Zlib::Inflate.new(Zlib::MAX_WBITS+16)
+        chunk = @zstream.inflate(chunk)
+      end
+      puts chunk
+      @io_write << chunk
+    #f << chunk
+      @bytes += chunk.length
+    end
+  }
+end
+if @zstream
+  @zstream.close
+end
+
+puts @bytes

data/http-machine.rb

@@ -6,9 +6,7 @@ require 'em-http-request'
 @io_read, @io_write = IO.pipe
 
 EM.run do
-  #url = 'http://www.universobit.com.ar/AvisosUniversobit/trovit/AvisosUniversobit_1.xml'
-  #url = 'http://www.aestrenar.com.ar/backend/rssAestrenar.xml'
-  url = 'http://procarnet.es/feed/sumavisos/sumavisos.xml'
+  url = ''
 
   http = EM::HttpRequest.new(url).get
   http.stream { |chunk|
@@ -17,6 +15,7 @@ EM.run do
       EM.stop
     else
       @io_write << chunk
+      puts chunk
       @bytes += chunk.length
     end
   }

data/lib/bliss.rb

@@ -1,9 +1,13 @@
 require 'nokogiri'
 require 'eventmachine'
 require 'em-http-request'
+require 'zlib'
 
 require 'hash_extension'
 
-require 'bliss/sax_parser'
+require 'bliss/constraint'
+require 'bliss/format'
+
+require 'bliss/encoding_error'
 require 'bliss/parser_machine'
 require 'bliss/parser'

data/lib/bliss/constraint.rb

@@ -0,0 +1,78 @@
+module Bliss
+  class Constraint
+    #attr_reader :field, :type, :state
+    attr_reader :depth, :setting, :state
+
+    #TYPES = [:exist, :not_blank, :possible_values]
+
+    def initialize(depth, setting, params={})
+      @depth = depth
+      @setting = setting
+
+      @state = :not_checked
+    end
+
+    #def initialize(field, type, possible_values=nil)
+    #  if field.is_a? Array
+    #    @field = field
+    #  else
+    #    @field = [field]
+    #  end
+    #  @type = type
+    #  @possible_values = possible_values
+    #
+    #  @state = :not_checked
+    #end
+
+    def run!(hash)
+      @state = :not_checked
+      #@field.each do |field|
+        #if @state == :passed
+        #  break
+        #end
+        case @setting
+          when :tag_name_required
+            if !hash.keys.include?(depth.last)
+              @state = :not_passed
+            else
+              @state = :passed
+            end
+          #when :not_blank
+          #  if hash.has_key?(field) and !hash[field].to_s.empty?
+          #    @state = :passed
+          #  else
+          #    @state = :not_passed
+          #  end
+          #when :possible_values
+          #  if hash.has_key?(field) and @possible_values.include?(hash[field])
+          #    @state = :passed
+          #  else
+          #    @state = :not_passed
+          #  end
+        end
+      #end
+      @state
+    end
+
+    def detail
+      if @state == :not_passed
+        detail = case @type
+          when :tag_name_required
+            [@field.join(" or "), "missing"]
+          #when :not_blank
+          #  [@field.join(" or "), "blank"]
+          #when :possible_values
+          #  [@field.join(" or "), "invalid"]
+        end
+      end
+    end
+
+    #def self.build_constraint(depth, setting, params={})#, field, type, possible_values=nil)
+    #  constraints = []
+      #constraints.push Bliss::Constraint.new(field, :exist) if types.include?(:exist)
+      #constraints.push Bliss::Constraint.new(field, :not_blank) if types.include?(:not_blank)
+      #constraints.push BlissConstraint.new(field, :possible_values, possible_values) if types.include?(:possible_values)
+    #  constraints
+    #end
+  end
+end

data/lib/bliss/encoding_error.rb

@@ -0,0 +1,4 @@
+module Bliss
+  class EncodingError < ::StandardError
+  end
+end

data/lib/bliss/format.rb

@@ -0,0 +1,97 @@
+require 'yaml'
+
+module Bliss
+  class Format
+    @@keywords = %w{ tag_name_required content_required tag_name_type content_type tag_name_format content_format tag_name_values content_values  }
+
+    def initialize
+      yml = YAML.load_file('/home/fernando/desarrollo/workspace/experimentos/bliss/spec.yml')
+      self.specifications = yml
+    end
+
+    # TODO for debugging only!
+    def keywords
+      @@keywords
+    end
+
+    def specifications=(specs={})
+      @specs = specs.dup
+    end
+    alias :specs= :specifications=
+
+    def constraints
+      return [] if not (@specs.is_a? Hash and @specs.size > 0)
+
+      constraints = []
+
+      @specs.recurse(true) do |depth, value|
+        if !@@keywords.include?(depth.last)
+          settings = @specs.value_at_chain(depth).select{|key| @@keywords.include?(key) }
+        end
+        if settings
+          settings.merge!({"tag_name_required" => true}) if not settings.has_key?("tag_name_required")
+
+          puts settings.inspect
+
+          # tag_name_required constraint:
+
+          settings.each_pair { |setting, value|
+            case setting
+              when "tag_name_required"
+                if value == true
+                  constraints.push(Bliss::Constraint.new(depth, :tag_name_required))
+                end
+            end
+          }
+
+          #required_fields.each do |field|
+          #  constraints.concat(Sumavisos::Parsers::Constraint.build_constraint(field, [:exist, :not_blank]).dup)
+          #end
+
+          ###
+
+          #puts "#{depth.join('/')}: #{settings.inspect}"
+        end
+      end
+
+      puts constraints.inspect
+      
+      return constraints
+    end
+    
+    # during parsing
+    # Sumavisos::Parsers::Validator.check_constraints(ad, constraints.select{|c| [:not_checked, :passed].include?(c.state)})
+
+    # @constraints.select{|c| c.state == :not_passed }.collect(&:detail)
+    
+    def ad_constraints(root, vertical)
+      #required_fields = Sumavisos::Parsers::Validator::FIELDS['all']['required'].dup
+      #required_fields.concat(Sumavisos::Parsers::Validator::FIELDS[vertical]['required'])
+      
+      #constraints = []
+      #required_fields.each do |field|
+      #  constraints.concat(Sumavisos::Parsers::Constraint.build_constraint(field, [:exist, :not_blank]).dup)
+      #end
+
+      if vertical == 'property'
+        constraints.concat(Sumavisos::Parsers::Constraint.build_constraint(['type'], [:possible_values], Sumavisos::Parsers::Validator::VALID_PROPERTY_TYPES).dup)
+      end
+
+      constraints
+    end
+
+    def check_constraints(ads, constraints)
+      errors = []
+
+      ads = [ads] if not ads.is_a? Array
+
+      ads.each do |ad|
+        constraints.each do |constraint|
+          constraint.run!(ad)
+        end
+      end
+
+      return errors
+    end
+  end
+end