shoesgem 0.1424.0 → 0.1426.0

data/shoes/ruby/lib/ftsearch/document_map_reader.rb

@@ -1,106 +1,106 @@
-# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
-#
-
-module FTSearch
-
-class DocumentMapReader
-  DEFAULT_OPTIONS = {
-    :path => nil,
-    :io   => nil,
-  }
-  def initialize(options = {})
-    options = DEFAULT_OPTIONS.merge(options)
-    unless options[:path] || options[:io]
-      raise ArgumentError, "Need either the path to the suffix array file or an IO."
-    end
-    if options[:path]
-      io = File.open(options[:path], "rb")
-    else
-      io = options[:io]
-    end
-    @uri_tbl, @field_arr = Marshal.load(io)
-  end
-
-  def document_id(suffix_idx, offset)
-    idx = binary_search(@field_arr, offset, 0, @field_arr.size)
-    @field_arr[idx][1]
-  end
-
-  def field_id(suffix_idx, offset)
-    idx = binary_search(@field_arr, offset, 0, @field_arr.size)
-    @field_arr[idx][2]
-  end
-
-  def field_size(suffix_idx, offset)
-    idx = binary_search(@field_arr, offset, 0, @field_arr.size)
-    @field_arr[idx][3]
-  end
-
-  def field_info(suffix_idx, offset)
-    idx = binary_search(@field_arr, offset, 0, @field_arr.size)
-    @field_arr[idx]
-  end
-
-  def document_uri(suffix_idx, offset)
-    doc_id = document_id(suffix_idx, offset)
-    @uri_tbl[doc_id]
-  end
-
-  def document_id_to_uri(doc_id)
-    @uri_tbl[doc_id]
-  end
-
-  def offsets_to_field_infos(offsets)
-    offsets.map{|off| @field_arr[binary_search(@field_arr, off, 0, @field_arr.size)]}
-  end
-
-  def rank_offsets(offsets, weights)
-    h = Hash.new{|h,k| h[k] = 0.0}
-    offsets_to_field_infos(offsets).each do |offset, doc_id, field_id, field_size|
-      h[doc_id] += weights[field_id] / field_size
-    end
-    sort_score_hash(h)
-  end
-
-  def rank_offsets_probabilistic(offsets, weights, iterations)
-    h     = Hash.new{|h,k| h[k] = 0.0}
-    infos = offsets_to_field_infos(offsets)
-    max   = infos.size
-    while iterations > 0
-      offset, doc_id, field_id, field_size = infos[rand(max)]
-      h[doc_id] += weights[field_id] / field_size
-      iterations -= 1
-    end
-    sort_score_hash(h)
-  end
-
-  def dump_data
-    [@uri_tbl, @field_arr]
-  end
-
-  def num_documents
-    @uri_tbl.size
-  end
-
-  private
-  def sort_score_hash(h)
-    h.sort_by{|_,score| -score}
-  end
-
-  def binary_search(arr, offset, from, to)
-    middle = 0
-    while to - from > 1
-      middle = (from + to) / 2
-      pivot,  = arr[middle]
-      if offset < pivot
-        to = middle
-      else
-        from = middle
-      end
-    end
-
-    from
-  end
-
-end
-end  # FTSearch
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+module FTSearch
+
+class DocumentMapReader
+  DEFAULT_OPTIONS = {
+    :path => nil,
+    :io   => nil,
+  }
+  def initialize(options = {})
+    options = DEFAULT_OPTIONS.merge(options)
+    unless options[:path] || options[:io]
+      raise ArgumentError, "Need either the path to the suffix array file or an IO."
+    end
+    if options[:path]
+      io = File.open(options[:path], "rb")
+    else
+      io = options[:io]
+    end
+    @uri_tbl, @field_arr = Marshal.load(io)
+  end
+
+  def document_id(suffix_idx, offset)
+    idx = binary_search(@field_arr, offset, 0, @field_arr.size)
+    @field_arr[idx][1]
+  end
+
+  def field_id(suffix_idx, offset)
+    idx = binary_search(@field_arr, offset, 0, @field_arr.size)
+    @field_arr[idx][2]
+  end
+
+  def field_size(suffix_idx, offset)
+    idx = binary_search(@field_arr, offset, 0, @field_arr.size)
+    @field_arr[idx][3]
+  end
+
+  def field_info(suffix_idx, offset)
+    idx = binary_search(@field_arr, offset, 0, @field_arr.size)
+    @field_arr[idx]
+  end
+
+  def document_uri(suffix_idx, offset)
+    doc_id = document_id(suffix_idx, offset)
+    @uri_tbl[doc_id]
+  end
+
+  def document_id_to_uri(doc_id)
+    @uri_tbl[doc_id]
+  end
+
+  def offsets_to_field_infos(offsets)
+    offsets.map{|off| @field_arr[binary_search(@field_arr, off, 0, @field_arr.size)]}
+  end
+
+  def rank_offsets(offsets, weights)
+    h = Hash.new{|h,k| h[k] = 0.0}
+    offsets_to_field_infos(offsets).each do |offset, doc_id, field_id, field_size|
+      h[doc_id] += weights[field_id] / field_size
+    end
+    sort_score_hash(h)
+  end
+
+  def rank_offsets_probabilistic(offsets, weights, iterations)
+    h     = Hash.new{|h,k| h[k] = 0.0}
+    infos = offsets_to_field_infos(offsets)
+    max   = infos.size
+    while iterations > 0
+      offset, doc_id, field_id, field_size = infos[rand(max)]
+      h[doc_id] += weights[field_id] / field_size
+      iterations -= 1
+    end
+    sort_score_hash(h)
+  end
+
+  def dump_data
+    [@uri_tbl, @field_arr]
+  end
+
+  def num_documents
+    @uri_tbl.size
+  end
+
+  private
+  def sort_score_hash(h)
+    h.sort_by{|_,score| -score}
+  end
+
+  def binary_search(arr, offset, from, to)
+    middle = 0
+    while to - from > 1
+      middle = (from + to) / 2
+      pivot,  = arr[middle]
+      if offset < pivot
+        to = middle
+      else
+        from = middle
+      end
+    end
+
+    from
+  end
+
+end
+end  # FTSearch

data/shoes/ruby/lib/ftsearch/document_map_writer.rb

@@ -1,46 +1,46 @@
-# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
-#
-
-require 'ftsearch/util'
-
-module FTSearch
-class DocumentMapWriter
-  include InMemoryWriter
-
-  DEFAULT_OPTIONS = {
-    :path => "docmap-#{Process.pid}-#{rand(100000)}",
-  }
-  def initialize(options = {})
-    options    = DEFAULT_OPTIONS.merge(options)
-    @path      = options[:path]
-    @field_arr = []
-    @uri_tbl   = []
-    @data      = [@uri_tbl, @field_arr]
-    initialize_in_memory_buffer
-  end
-
-  def merge(doc_map_reader)
-    # TODO: general merge?
-    @uri_tbl, @field_arr = doc_map_reader.dump_data
-    @data = [@uri_tbl, @field_arr]
-  end
-
-  def add_document(doc_id, uri)
-    @uri_tbl[doc_id] = uri
-  end
-
-  def add_field(offset, doc_id, field_id, size)
-    @field_arr << [offset, doc_id, field_id, size]
-  end
-
-  def finish!
-    if @path
-      File.open(@path, "wb") do |f|
-        Marshal.dump(@data, f)
-      end
-    else
-      Marshal.dump(@data, @memory_io)
-    end
-  end
-end
-end  # FTSearch
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+require 'ftsearch/util'
+
+module FTSearch
+class DocumentMapWriter
+  include InMemoryWriter
+
+  DEFAULT_OPTIONS = {
+    :path => "docmap-#{Process.pid}-#{rand(100000)}",
+  }
+  def initialize(options = {})
+    options    = DEFAULT_OPTIONS.merge(options)
+    @path      = options[:path]
+    @field_arr = []
+    @uri_tbl   = []
+    @data      = [@uri_tbl, @field_arr]
+    initialize_in_memory_buffer
+  end
+
+  def merge(doc_map_reader)
+    # TODO: general merge?
+    @uri_tbl, @field_arr = doc_map_reader.dump_data
+    @data = [@uri_tbl, @field_arr]
+  end
+
+  def add_document(doc_id, uri)
+    @uri_tbl[doc_id] = uri
+  end
+
+  def add_field(offset, doc_id, field_id, size)
+    @field_arr << [offset, doc_id, field_id, size]
+  end
+
+  def finish!
+    if @path
+      File.open(@path, "wb") do |f|
+        Marshal.dump(@data, f)
+      end
+    else
+      Marshal.dump(@data, @memory_io)
+    end
+  end
+end
+end  # FTSearch

data/shoes/ruby/lib/ftsearch/field_infos.rb

@@ -1,46 +1,46 @@
-# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
-#
-
-require 'ftsearch/analysis/whitespace_analyzer'
-
-module FTSearch
-class FieldInfos
-  DEFAULT_OPTIONS = {
-    :default_analyzer => FTSearch::Analysis::WhiteSpaceAnalyzer.new,
-    :stored => true,
-  }
-  def initialize(options = {})
-    options = DEFAULT_OPTIONS.merge(options)
-    @fields = {}
-    @default_options = options
-  end
-
-  def add_field(options = {})
-    options = @default_options.merge(options)
-    raise "Need a name" unless options[:name]
-    store_field_info(options)
-  end
-
-  def [](field_name)
-    if field_info = @fields[field_name]
-      field_info
-    else
-      store_field_info(:name => field_name)
-    end
-  end
-
-  private
-  def store_field_info(options)
-    options = @default_options.merge(options)
-    unless options[:analyzer]
-      if klass = options[:analyzer_class]
-        options[:analyzer] = klass.new
-      else
-        options[:analyzer] = @default_options[:default_analyzer]
-      end
-    end
-    @fields[options[:name]] = options
-  end
-end
-end  # FTSearch
-
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+require 'ftsearch/analysis/whitespace_analyzer'
+
+module FTSearch
+class FieldInfos
+  DEFAULT_OPTIONS = {
+    :default_analyzer => FTSearch::Analysis::WhiteSpaceAnalyzer.new,
+    :stored => true,
+  }
+  def initialize(options = {})
+    options = DEFAULT_OPTIONS.merge(options)
+    @fields = {}
+    @default_options = options
+  end
+
+  def add_field(options = {})
+    options = @default_options.merge(options)
+    raise "Need a name" unless options[:name]
+    store_field_info(options)
+  end
+
+  def [](field_name)
+    if field_info = @fields[field_name]
+      field_info
+    else
+      store_field_info(:name => field_name)
+    end
+  end
+
+  private
+  def store_field_info(options)
+    options = @default_options.merge(options)
+    unless options[:analyzer]
+      if klass = options[:analyzer_class]
+        options[:analyzer] = klass.new
+      else
+        options[:analyzer] = @default_options[:default_analyzer]
+      end
+    end
+    @fields[options[:name]] = options
+  end
+end
+end  # FTSearch
+

data/shoes/ruby/lib/ftsearch/fragment_writer.rb

@@ -1,114 +1,114 @@
-# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
-#
-
-require 'fileutils'
-require 'ftsearch/suffix_array_writer'
-require 'ftsearch/document_map_writer'
-require 'ftsearch/fulltext_writer'
-require 'ftsearch/field_infos'
-
-require 'ftsearch/fulltext_reader'
-require 'ftsearch/document_map_reader'
-require 'ftsearch/suffix_array_reader'
-
-module FTSearch
-
-class FragmentWriter
-  DEFAULT_OPTIONS = {
-    :path                      => "ftsearch-#{Process.pid}-#{rand(100000)}",
-    :default_analyzer_class    => FTSearch::Analysis::WhiteSpaceAnalyzer,
-    :field_infos_class         => FieldInfos,
-    :fulltext_writer_class     => FulltextWriter,
-    :suffix_array_writer_class => SuffixArrayWriter,
-    :doc_map_writer_class      => DocumentMapWriter,
-    :field_infos               => nil,
-    :fulltext_writer           => nil,
-    :suffix_array_writer       => nil,
-    :doc_map_writer_nil        => nil,
-  }
-
-  attr_reader :fulltext_writer, :suffix_array_writer, :doc_map_writer
-  def initialize(options = {})
-    options = DEFAULT_OPTIONS.merge(options)
-    create  = lambda do |name, *args|
-      options[name] || options[(name.to_s + "_class").to_sym].new(*args)
-    end
-    build_path = lambda do |suffix|
-      if @path
-        File.join(@tmpdir, suffix)
-      else
-        nil
-      end
-    end
-    @path = options[:path]
-    if @path
-      @path   = File.expand_path(@path)
-      @tmpdir = @path + "#{Process.pid}-#{rand(100000)}"
-      FileUtils.mkdir_p(@tmpdir)
-    end
-
-    @fulltext_writer     = create.call(:fulltext_writer, :path     => build_path["fulltext"])
-    @suffix_array_writer = create.call(:suffix_array_writer, :path => build_path["suffixes"])
-    @doc_map_writer      = create.call(:doc_map_writer, :path      => build_path["docmap"])
-
-    default_analyzer = (klass = options[:default_analyzer_class]) ? klass.new : nil
-    @field_infos     = create.call(:field_infos, :default_analyzer => default_analyzer)
-    @num_documents   = 0
-    @field_map       = Hash.new{|h,k| h[k.to_sym] = h.size}
-    @field_map[:uri] # init
-  end
-
-  def add_document(doc_hash)
-    uri = doc_hash[:uri] || @num_documents.to_s
-    @fulltext_writer.add_document(@num_documents, doc_hash.merge(:uri => uri), 
-                                  @field_map, @field_infos, @suffix_array_writer, @doc_map_writer)
-    @num_documents += 1
-  end
-
-  def merge(fragment_directory)
-    raise "Cannot import old data unless the destination Fragment is empty." unless @num_documents == 0
-    # TODO: use a FragmentReader to access old data
-    fulltext_reader     = FulltextReader.new(:path => "#{fragment_directory}/fulltext")
-    suffix_array_reader = SuffixArrayReader.new(fulltext_reader, nil, 
-                                                :path => "#{fragment_directory}/suffixes")
-    doc_map_reader      = DocumentMapReader.new(:path => "#{fragment_directory}/docmap")
-    @fulltext_writer.merge(fulltext_reader)
-    @suffix_array_writer.merge(suffix_array_reader)
-    @doc_map_writer.merge(doc_map_reader)
-    #FIXME: .num_documents will be wrong if some URIs were repeated
-    @num_documents = doc_map_reader.num_documents
-    File.open(File.join(fragment_directory, "fieldmap"), "rb") do |f|
-      i = 0
-      f.each_line{|l| @field_map[l.chomp.to_sym] = i; i+= 1}
-    end
-  end
-
-  def fields
-    @field_map.sort_by{|field, fid| fid}.map{|field, fid| field}
-  end
-
-  def documents
-    @num_documents
-  end
-
-  def field_id(field)
-    @field_map.has_key?(field) && @field_map[field]
-  end
-
-  def finish!
-    @fulltext_writer.finish!
-    fulltext = @fulltext_writer.data
-    @suffix_array_writer.finish!(fulltext)
-    @doc_map_writer.finish!
-
-    if @path
-      File.open(File.join(@tmpdir, "fieldmap"), "wb") do |f|
-        @field_map.sort_by{|field_name, field_id| field_id}.each do |field_name, field_id| 
-          f.puts field_name
-        end
-        File.rename(@tmpdir, @path)
-      end
-    end
-  end
-end
-end  # FTSearch
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+require 'fileutils'
+require 'ftsearch/suffix_array_writer'
+require 'ftsearch/document_map_writer'
+require 'ftsearch/fulltext_writer'
+require 'ftsearch/field_infos'
+
+require 'ftsearch/fulltext_reader'
+require 'ftsearch/document_map_reader'
+require 'ftsearch/suffix_array_reader'
+
+module FTSearch
+
+class FragmentWriter
+  DEFAULT_OPTIONS = {
+    :path                      => "ftsearch-#{Process.pid}-#{rand(100000)}",
+    :default_analyzer_class    => FTSearch::Analysis::WhiteSpaceAnalyzer,
+    :field_infos_class         => FieldInfos,
+    :fulltext_writer_class     => FulltextWriter,
+    :suffix_array_writer_class => SuffixArrayWriter,
+    :doc_map_writer_class      => DocumentMapWriter,
+    :field_infos               => nil,
+    :fulltext_writer           => nil,
+    :suffix_array_writer       => nil,
+    :doc_map_writer_nil        => nil,
+  }
+
+  attr_reader :fulltext_writer, :suffix_array_writer, :doc_map_writer
+  def initialize(options = {})
+    options = DEFAULT_OPTIONS.merge(options)
+    create  = lambda do |name, *args|
+      options[name] || options[(name.to_s + "_class").to_sym].new(*args)
+    end
+    build_path = lambda do |suffix|
+      if @path
+        File.join(@tmpdir, suffix)
+      else
+        nil
+      end
+    end
+    @path = options[:path]
+    if @path
+      @path   = File.expand_path(@path)
+      @tmpdir = @path + "#{Process.pid}-#{rand(100000)}"
+      FileUtils.mkdir_p(@tmpdir)
+    end
+
+    @fulltext_writer     = create.call(:fulltext_writer, :path     => build_path["fulltext"])
+    @suffix_array_writer = create.call(:suffix_array_writer, :path => build_path["suffixes"])
+    @doc_map_writer      = create.call(:doc_map_writer, :path      => build_path["docmap"])
+
+    default_analyzer = (klass = options[:default_analyzer_class]) ? klass.new : nil
+    @field_infos     = create.call(:field_infos, :default_analyzer => default_analyzer)
+    @num_documents   = 0
+    @field_map       = Hash.new{|h,k| h[k.to_sym] = h.size}
+    @field_map[:uri] # init
+  end
+
+  def add_document(doc_hash)
+    uri = doc_hash[:uri] || @num_documents.to_s
+    @fulltext_writer.add_document(@num_documents, doc_hash.merge(:uri => uri), 
+                                  @field_map, @field_infos, @suffix_array_writer, @doc_map_writer)
+    @num_documents += 1
+  end
+
+  def merge(fragment_directory)
+    raise "Cannot import old data unless the destination Fragment is empty." unless @num_documents == 0
+    # TODO: use a FragmentReader to access old data
+    fulltext_reader     = FulltextReader.new(:path => "#{fragment_directory}/fulltext")
+    suffix_array_reader = SuffixArrayReader.new(fulltext_reader, nil, 
+                                                :path => "#{fragment_directory}/suffixes")
+    doc_map_reader      = DocumentMapReader.new(:path => "#{fragment_directory}/docmap")
+    @fulltext_writer.merge(fulltext_reader)
+    @suffix_array_writer.merge(suffix_array_reader)
+    @doc_map_writer.merge(doc_map_reader)
+    #FIXME: .num_documents will be wrong if some URIs were repeated
+    @num_documents = doc_map_reader.num_documents
+    File.open(File.join(fragment_directory, "fieldmap"), "rb") do |f|
+      i = 0
+      f.each_line{|l| @field_map[l.chomp.to_sym] = i; i+= 1}
+    end
+  end
+
+  def fields
+    @field_map.sort_by{|field, fid| fid}.map{|field, fid| field}
+  end
+
+  def documents
+    @num_documents
+  end
+
+  def field_id(field)
+    @field_map.has_key?(field) && @field_map[field]
+  end
+
+  def finish!
+    @fulltext_writer.finish!
+    fulltext = @fulltext_writer.data
+    @suffix_array_writer.finish!(fulltext)
+    @doc_map_writer.finish!
+
+    if @path
+      File.open(File.join(@tmpdir, "fieldmap"), "wb") do |f|
+        @field_map.sort_by{|field_name, field_id| field_id}.each do |field_name, field_id| 
+          f.puts field_name
+        end
+        File.rename(@tmpdir, @path)
+      end
+    end
+  end
+end
+end  # FTSearch