xamplr 1.2.0

data/examples/random-people/rawtc.rb

@@ -0,0 +1,91 @@
+
+require 'fileutils'
+require 'fastercsv'
+require 'tokyocabinet'
+
+module RawTokyoCabinet
+  include TokyoCabinet
+
+  def RawTokyoCabinet.note_errors(msg="TokyoCabinet Error:: %s\n")
+    result = yield
+
+    rmsg = nil
+    unless result then
+      rmsg = sprintf(msg, @tc_db.errmsg(@tc_db.ecode))
+      STDERR.printf(rmsg)
+      caller(0).each do |trace|
+        STDERR.puts(trace)
+      end
+    end
+    return rmsg
+  end
+
+  start_at = Time.now
+
+  arr_of_arrs = FasterCSV.read("random-names.csv")
+
+  parsed_at = Time.now
+
+  @tc_db = TDB.new
+  RawTokyoCabinet.note_errors("TC:: tuning error: %s\n") do
+    @tc_db.tune(-1, -1, -1, TDB::TDEFLATE)
+  end
+
+  FileUtils.mkdir_p('repo/raw') unless File.exist?('repo/raw')
+
+  10.times do | iter |
+    RawTokyoCabinet.note_errors("TC:: open error: %s\n") do
+      @tc_db.open('repo/raw/repo.tct', TDB::OWRITER | TDB::OCREAT | TDB::OLCKNB ) #TDB::OTSYNC slows it down by almost 50 times
+    end
+
+    # Don't care if there are errors (in fact, if the index exists a failure is the expected thing)
+    %w{ surname city state email }.each do | index_name |
+      @tc_db.setindex(index_name, TDB::ITLEXICAL | TDB::ITKEEP)
+    end
+
+    $tc_time = 0
+    record_count = @tc_db.rnum()
+
+    inner_start = Time.now
+
+    arr_of_arrs.each_with_index do | row, i |
+
+      #    indexes = {
+      #            'surname' => row[1],
+      #            'city' => row[3],
+      #            'state' => row[4],
+      #            'email' => row[6]
+      #    }
+
+      data = {
+              'given_name' => row[0],
+              'surname' => row[1],
+              'street_address' => row[2],
+              'city' => row[3],
+              'state' => row[4],
+              'postal_code' => row[5],
+              'email' => row[6],
+              'phone' => row[7]
+      }
+
+      RawTokyoCabinet.note_errors("TC:: write error: %s\n") do
+        start = Time.now
+        @tc_db.put("person-#{ record_count + i }", data)
+        $tc_time += Time.now - start
+      end
+
+    end
+
+    final_count = @tc_db.rnum
+
+    RawTokyoCabinet.note_errors("TC:: close error: %s\n") do
+      @tc_db.close
+    end
+
+    processed_at = Time.now
+
+    p "parsed in #{ parsed_at - start_at }, processed in: #{ processed_at - parsed_at }"
+    p "starting at #{ record_count } records, now with: #{ final_count }"
+  end
+
+end

data/examples/random-people/settings.rb

@@ -0,0 +1,3 @@
+Xampl.set_default_persister_kind(:tokyo_cabinet)
+Xampl.set_default_persister_format(:xml_format)
+#Xampl.set_default_persister_format(:ruby_format)

data/examples/random-people/what-to-query-on.rb

@@ -0,0 +1,80 @@
+$LOAD_PATH.unshift("xampl_generated_code")
+
+require 'set'
+require 'fastercsv'
+require 'RandomPeople'
+require 'people'
+require 'settings'
+
+module RandomPeople
+  arr_of_arrs = FasterCSV.read("random-names.csv")
+
+  surnames = Set.new
+  cities = Set.new
+  states = Set.new
+  email_domains = Set.new
+
+  arr_of_arrs.each do | row |
+    surname = row[1]
+    city = row[3]
+    state = row[4]
+
+    surnames << surname
+    cities << city
+    states << state
+
+    email = row[6]
+    domain = email.split("@")
+    if 2 == domain.size then
+      email_domains << domain[1]
+    end
+
+  end
+
+  puts "surnames: #{ surnames.size }"
+  puts "cities: #{ cities.size }"
+  puts "states: #{ states.size }"
+  puts "email_domains: #{ email_domains.size }"
+
+  puts "STATES:"
+  puts states.to_a.sort.inspect
+
+  puts "EMAIL DOMAINS:"
+  puts email_domains.to_a.sort.inspect
+
+  #GivenName,Surname,StreetAddress,City,State,ZipCode,EmailAddress,TelephoneNumber
+  #<people pid='' xmlns="http://xampl.com/people">
+  #    <person pid=''
+  #            given-name=''
+  #            surname=''
+  #            street-address=''
+  #            city=''
+  #            state=''
+  #            postal-code=''
+  #            email=''
+  #            phone=''/>
+  #</people>
+
+#  Xampl.transaction("random-people") do
+#
+#    people = People.new('people')
+#
+#    arr_of_arrs.each_with_index do | row, i |
+#      person = people.new_person("person-#{ i }")
+#      person.given_name = row[0]
+#      person.surname = row[1]
+#      person.street_address = row[2]
+#      person.city = row[3]
+#      person.state = row[4]
+#      person.postal_code = row[5]
+#      person.email = row[6]
+#      person.phone = row[7]
+#    end
+#  end
+#
+#  processed_at = Time.now
+#
+#  p counts
+#  p "parsed in #{ parsed_at - start_at }, counted in: #{ counted_at - parsed_at }, processed in: #{ processed_at - counted_at }"
+
+end

data/examples/random-people/xampl-gen.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby -w -I..
+
+if $0 == __FILE__ then
+
+  class File
+    def File.sjoin(*args)
+      File.join(args.select{ | o | o })
+    end
+  end
+
+  require 'xamplr-generator'
+
+  include XamplGenerator
+  include Xampl
+
+  Xampl.transaction("setup", :in_memory) do
+    directory = File.sjoin(".", "xampl_generated_code")
+
+    options = Xampl.make(Options) do |options|
+      options.new_index_attribute("pid").persisted = true
+      options.new_index_attribute("id")
+
+      options.resolve("http://xampl.com/people", "RandomPeople", "p")
+    end
+
+    filenames = Dir.glob("./xml/**/*.xml")
+
+    generator = Generator.new
+    generator.go(:options => options,
+                 :filenames => filenames,
+                 :directory => directory)
+
+    #puts generator.print_elements("./generated-elements.xml")
+    exit!
+  end
+end

data/examples/random-people/xml/people.xml

@@ -0,0 +1,11 @@
+<people pid='' xmlns="http://xampl.com/people">
+    <person pid=''
+            given-name=''
+            surname=''
+            street-address=''
+            city=''
+            state=''
+            postal-code=''
+            email=''
+            phone=''/>
+</people>

data/examples/read-testing/Makefile

@@ -0,0 +1,10 @@
+
+nope:
+
+read:
+	time ruby read.rb
+
+load:
+	rm -rf repo
+	time ruby load.rb
+

data/examples/read-testing/load.rb

@@ -0,0 +1,65 @@
+$LOAD_PATH.unshift("xampl_generated_code")
+
+require 'rubygems'
+require 'Perf'
+require 'settings'
+
+module Perf
+
+  count = 0
+  root = nil
+  $total_lines = 0
+
+  srand( 12345 )
+
+  def Perf.make_text(count)
+    lines = 1 + rand(10)
+    $total_lines += lines
+
+    s = ""
+    lines.times do | i |
+      s << "this is a line for count: #{ count }, #{ i } of #{ lines } with scary stuff: <>&'\" ..."
+    end
+    return s
+  end
+
+
+  #<root pid=''
+  #      xmlns="http://xampl.com/performance">
+  #    <text pid=''
+  #          size=''
+  #          stuff=''>
+  #        actual text
+  #    </text>
+  #</root>
+
+  start_at = Time.now
+
+  1.times do | outer |
+    Xampl.transaction("random-people") do
+      unless root then
+        root = Root.new('root')
+      end
+
+      10000.times do | iter |
+
+        count += 1
+
+        text = root.new_text("text-#{ count }")
+        text.stuff = "---<>&\"'---"
+        text.content = make_text(count)
+
+        #        puts "#{File.basename(__FILE__)}:#{__LINE__} #{ text.pp_xml }"
+
+      end
+    end
+
+    puts "transaction #{ outer } ending... #{ Time.now - start_at } seconds"
+#    puts "#{File.basename(__FILE__)}:#{__LINE__} #{ root.pp_xml }"
+  end
+
+  done_at = Time.now
+
+  p "loaded #{ count }, total lines: #{ $total_lines }, in #{ done_at - start_at }"
+
+end

data/examples/read-testing/read.rb

@@ -0,0 +1,51 @@
+require 'rubygems'
+require 'xampl_generated_code/Perf'
+require 'settings'
+
+module Perf
+
+  count = 0
+  root = nil
+  $total_lines = 0
+
+  #<root pid=''
+  #      xmlns="http://xampl.com/performance">
+  #    <text pid=''
+  #          size=''
+  #          stuff=''>
+  #        actual text
+  #    </text>
+  #</root>
+
+  start_at = Time.now
+
+  Xampl.transaction("random-people") do
+    root = Root['root']
+  end
+  root_at = Time.now
+  puts "#{File.basename(__FILE__)}:#{__LINE__} Root read in #{ root_at - start_at}"
+
+  root.text.each do | text |
+    count += 1
+  end
+
+  #puts root.pp_xml
+
+  looped_at = Time.now
+  puts "#{File.basename(__FILE__)}:#{__LINE__} looped over #{ count } text in #{ looped_at - root_at } NO LOADING"
+
+  count = 0
+  total_lines = 0
+
+  root.text.each do | text |
+    content = text.content
+#    puts "#{File.basename(__FILE__)}:#{__LINE__} content: [[#{ content }]]"
+    total_lines += content.length() if content
+    count += 1
+  end
+
+  done_at = Time.now
+
+  puts "loaded #{ count }, total lines: #{ total_lines }, in #{ done_at - looped_at } seconds, total time: #{ done_at - start_at } seconds"
+
+end

data/examples/read-testing/rrr.rb

@@ -0,0 +1,87 @@
+require 'rubygems'
+require 'libxml'
+
+xml = %Q{
+<root pid='aaaaa'
+      xmlns:x="http://xampl.com/extra"
+      xmlns="http://xampl.com/performance">
+    <text pid='bbbb'
+          x:size='ccccc'
+          stuff='ddddd'>
+        actual text &quot; well?
+        actual text &quot; well?
+        actual text &quot; well?
+        actual text &quot; well?
+        xxxxx <![CDATA[ <>"'& ]]> xxxxx
+    </text>
+<nothing/>
+    <x:blob v='blob value'/>
+</root>
+}
+
+reader = LibXML::XML::Reader.string(xml,
+                                    :options => LibXML::XML::Parser::Options::NOENT |
+                                            LibXML::XML::Parser::Options::NONET |
+                                            LibXML::XML::Parser::Options::NOCDATA |
+                                            LibXML::XML::Parser::Options::DTDATTR |
+                                            LibXML::XML::Parser::Options::NSCLEAN |
+                                            LibXML::XML::Parser::Options::COMPACT)
+
+while reader.read do
+  #  puts reader.node_type
+  #  puts "line: #{ reader.line_number }, column: #{ reader.column_number }"
+
+  case reader.node_type
+#    when LibXML::XML::Reader::TYPE_ATTRIBUTE
+    #      puts "attribute"
+    when LibXML::XML::Reader::TYPE_DOCUMENT
+      puts "DOCUMENT"
+    when LibXML::XML::Reader::TYPE_ELEMENT
+      attribute_count = reader.attribute_count
+      puts "element #{ reader.local_name }, ns: #{ reader.namespace_uri }, #attributes: #{ attribute_count }, depth: #{ reader.depth }"
+      if reader.has_attributes? then
+        reader.move_to_first_attribute
+        attribute_count.times do | i |
+          next if reader.namespace_declaration?
+          puts "      attr[#{ i }]:: lname: #{ reader.local_name }, ns: #{ reader.namespace_uri } ---> #{ reader.value }"
+          reader.move_to_next_attribute
+        end
+      end
+
+    when LibXML::XML::Reader::TYPE_END_ELEMENT
+      puts "END ELEMENT"
+    when LibXML::XML::Reader::TYPE_TEXT
+      puts "TEXT [[#{ reader.read_string }]]"
+    when LibXML::XML::Reader::TYPE_CDATA
+      puts "CDATA [[#{ reader.read_string }]]"
+    when LibXML::XML::Reader::TYPE_SIGNIFICANT_WHITESPACE
+      puts "SIGNIFICANT white space [[#{ reader.read_string }]]"
+    when LibXML::XML::Reader::TYPE_ENTITY_REFERENCE
+      puts "entity ref"
+    when LibXML::XML::Reader::TYPE_WHITESPACE
+      puts "whitespace"
+    when LibXML::XML::Reader::TYPE_PROCESSING_INSTRUCTION
+      puts "processing instruction"
+    when LibXML::XML::Reader::TYPE_COMMENT
+      puts "comment"
+    when LibXML::XML::Reader::TYPE_DOCUMENT_TYPE
+      puts "doc type"
+
+    when LibXML::XML::Reader::TYPE_XML_DECLARATION
+      puts "xml decl"
+    when LibXML::XML::Reader::TYPE_NONE
+      puts "NONE!!"
+    when LibXML::XML::Reader::TYPE_NOTATION
+      puts "notifiation"
+    when LibXML::XML::Reader::TYPE_DOCUMENT_FRAGMENT
+      puts "doc fragment"
+    when LibXML::XML::Reader::TYPE_ENTITY
+      puts "entity"
+    when LibXML::XML::Reader::TYPE_END_ENTITY
+      puts "
+      end entity"
+    else
+      puts "UNKNOWN: #{reader.node_type}"
+  end
+
+end