shoesgem 0.1480.0 → 0.1514.0

data/shoes/ruby/lib/ftsearch/field_infos.rb

@@ -1,46 +1,46 @@
-# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
-#
-
-require 'ftsearch/analysis/whitespace_analyzer'
-
-module FTSearch
-class FieldInfos
-  DEFAULT_OPTIONS = {
-    :default_analyzer => FTSearch::Analysis::WhiteSpaceAnalyzer.new,
-    :stored => true,
-  }
-  def initialize(options = {})
-    options = DEFAULT_OPTIONS.merge(options)
-    @fields = {}
-    @default_options = options
-  end
-
-  def add_field(options = {})
-    options = @default_options.merge(options)
-    raise "Need a name" unless options[:name]
-    store_field_info(options)
-  end
-
-  def [](field_name)
-    if field_info = @fields[field_name]
-      field_info
-    else
-      store_field_info(:name => field_name)
-    end
-  end
-
-  private
-  def store_field_info(options)
-    options = @default_options.merge(options)
-    unless options[:analyzer]
-      if klass = options[:analyzer_class]
-        options[:analyzer] = klass.new
-      else
-        options[:analyzer] = @default_options[:default_analyzer]
-      end
-    end
-    @fields[options[:name]] = options
-  end
-end
-end  # FTSearch
-
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+require 'ftsearch/analysis/whitespace_analyzer'
+
+module FTSearch
+class FieldInfos
+  DEFAULT_OPTIONS = {
+    :default_analyzer => FTSearch::Analysis::WhiteSpaceAnalyzer.new,
+    :stored => true,
+  }
+  def initialize(options = {})
+    options = DEFAULT_OPTIONS.merge(options)
+    @fields = {}
+    @default_options = options
+  end
+
+  def add_field(options = {})
+    options = @default_options.merge(options)
+    raise "Need a name" unless options[:name]
+    store_field_info(options)
+  end
+
+  def [](field_name)
+    if field_info = @fields[field_name]
+      field_info
+    else
+      store_field_info(:name => field_name)
+    end
+  end
+
+  private
+  def store_field_info(options)
+    options = @default_options.merge(options)
+    unless options[:analyzer]
+      if klass = options[:analyzer_class]
+        options[:analyzer] = klass.new
+      else
+        options[:analyzer] = @default_options[:default_analyzer]
+      end
+    end
+    @fields[options[:name]] = options
+  end
+end
+end  # FTSearch
+

data/shoes/ruby/lib/ftsearch/fragment_writer.rb

@@ -1,114 +1,114 @@
-# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
-#
-
-require 'fileutils'
-require 'ftsearch/suffix_array_writer'
-require 'ftsearch/document_map_writer'
-require 'ftsearch/fulltext_writer'
-require 'ftsearch/field_infos'
-
-require 'ftsearch/fulltext_reader'
-require 'ftsearch/document_map_reader'
-require 'ftsearch/suffix_array_reader'
-
-module FTSearch
-
-class FragmentWriter
-  DEFAULT_OPTIONS = {
-    :path                      => "ftsearch-#{Process.pid}-#{rand(100000)}",
-    :default_analyzer_class    => FTSearch::Analysis::WhiteSpaceAnalyzer,
-    :field_infos_class         => FieldInfos,
-    :fulltext_writer_class     => FulltextWriter,
-    :suffix_array_writer_class => SuffixArrayWriter,
-    :doc_map_writer_class      => DocumentMapWriter,
-    :field_infos               => nil,
-    :fulltext_writer           => nil,
-    :suffix_array_writer       => nil,
-    :doc_map_writer_nil        => nil,
-  }
-
-  attr_reader :fulltext_writer, :suffix_array_writer, :doc_map_writer
-  def initialize(options = {})
-    options = DEFAULT_OPTIONS.merge(options)
-    create  = lambda do |name, *args|
-      options[name] || options[(name.to_s + "_class").to_sym].new(*args)
-    end
-    build_path = lambda do |suffix|
-      if @path
-        File.join(@tmpdir, suffix)
-      else
-        nil
-      end
-    end
-    @path = options[:path]
-    if @path
-      @path   = File.expand_path(@path)
-      @tmpdir = @path + "#{Process.pid}-#{rand(100000)}"
-      FileUtils.mkdir_p(@tmpdir)
-    end
-
-    @fulltext_writer     = create.call(:fulltext_writer, :path     => build_path["fulltext"])
-    @suffix_array_writer = create.call(:suffix_array_writer, :path => build_path["suffixes"])
-    @doc_map_writer      = create.call(:doc_map_writer, :path      => build_path["docmap"])
-
-    default_analyzer = (klass = options[:default_analyzer_class]) ? klass.new : nil
-    @field_infos     = create.call(:field_infos, :default_analyzer => default_analyzer)
-    @num_documents   = 0
-    @field_map       = Hash.new{|h,k| h[k.to_sym] = h.size}
-    @field_map[:uri] # init
-  end
-
-  def add_document(doc_hash)
-    uri = doc_hash[:uri] || @num_documents.to_s
-    @fulltext_writer.add_document(@num_documents, doc_hash.merge(:uri => uri), 
-                                  @field_map, @field_infos, @suffix_array_writer, @doc_map_writer)
-    @num_documents += 1
-  end
-
-  def merge(fragment_directory)
-    raise "Cannot import old data unless the destination Fragment is empty." unless @num_documents == 0
-    # TODO: use a FragmentReader to access old data
-    fulltext_reader     = FulltextReader.new(:path => "#{fragment_directory}/fulltext")
-    suffix_array_reader = SuffixArrayReader.new(fulltext_reader, nil, 
-                                                :path => "#{fragment_directory}/suffixes")
-    doc_map_reader      = DocumentMapReader.new(:path => "#{fragment_directory}/docmap")
-    @fulltext_writer.merge(fulltext_reader)
-    @suffix_array_writer.merge(suffix_array_reader)
-    @doc_map_writer.merge(doc_map_reader)
-    #FIXME: .num_documents will be wrong if some URIs were repeated
-    @num_documents = doc_map_reader.num_documents
-    File.open(File.join(fragment_directory, "fieldmap"), "rb") do |f|
-      i = 0
-      f.each_line{|l| @field_map[l.chomp.to_sym] = i; i+= 1}
-    end
-  end
-
-  def fields
-    @field_map.sort_by{|field, fid| fid}.map{|field, fid| field}
-  end
-
-  def documents
-    @num_documents
-  end
-
-  def field_id(field)
-    @field_map.has_key?(field) && @field_map[field]
-  end
-
-  def finish!
-    @fulltext_writer.finish!
-    fulltext = @fulltext_writer.data
-    @suffix_array_writer.finish!(fulltext)
-    @doc_map_writer.finish!
-
-    if @path
-      File.open(File.join(@tmpdir, "fieldmap"), "wb") do |f|
-        @field_map.sort_by{|field_name, field_id| field_id}.each do |field_name, field_id| 
-          f.puts field_name
-        end
-        File.rename(@tmpdir, @path)
-      end
-    end
-  end
-end
-end  # FTSearch
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+require 'fileutils'
+require 'ftsearch/suffix_array_writer'
+require 'ftsearch/document_map_writer'
+require 'ftsearch/fulltext_writer'
+require 'ftsearch/field_infos'
+
+require 'ftsearch/fulltext_reader'
+require 'ftsearch/document_map_reader'
+require 'ftsearch/suffix_array_reader'
+
+module FTSearch
+
+class FragmentWriter
+  DEFAULT_OPTIONS = {
+    :path                      => "ftsearch-#{Process.pid}-#{rand(100000)}",
+    :default_analyzer_class    => FTSearch::Analysis::WhiteSpaceAnalyzer,
+    :field_infos_class         => FieldInfos,
+    :fulltext_writer_class     => FulltextWriter,
+    :suffix_array_writer_class => SuffixArrayWriter,
+    :doc_map_writer_class      => DocumentMapWriter,
+    :field_infos               => nil,
+    :fulltext_writer           => nil,
+    :suffix_array_writer       => nil,
+    :doc_map_writer_nil        => nil,
+  }
+
+  attr_reader :fulltext_writer, :suffix_array_writer, :doc_map_writer
+  def initialize(options = {})
+    options = DEFAULT_OPTIONS.merge(options)
+    create  = lambda do |name, *args|
+      options[name] || options[(name.to_s + "_class").to_sym].new(*args)
+    end
+    build_path = lambda do |suffix|
+      if @path
+        File.join(@tmpdir, suffix)
+      else
+        nil
+      end
+    end
+    @path = options[:path]
+    if @path
+      @path   = File.expand_path(@path)
+      @tmpdir = @path + "#{Process.pid}-#{rand(100000)}"
+      FileUtils.mkdir_p(@tmpdir)
+    end
+
+    @fulltext_writer     = create.call(:fulltext_writer, :path     => build_path["fulltext"])
+    @suffix_array_writer = create.call(:suffix_array_writer, :path => build_path["suffixes"])
+    @doc_map_writer      = create.call(:doc_map_writer, :path      => build_path["docmap"])
+
+    default_analyzer = (klass = options[:default_analyzer_class]) ? klass.new : nil
+    @field_infos     = create.call(:field_infos, :default_analyzer => default_analyzer)
+    @num_documents   = 0
+    @field_map       = Hash.new{|h,k| h[k.to_sym] = h.size}
+    @field_map[:uri] # init
+  end
+
+  def add_document(doc_hash)
+    uri = doc_hash[:uri] || @num_documents.to_s
+    @fulltext_writer.add_document(@num_documents, doc_hash.merge(:uri => uri), 
+                                  @field_map, @field_infos, @suffix_array_writer, @doc_map_writer)
+    @num_documents += 1
+  end
+
+  def merge(fragment_directory)
+    raise "Cannot import old data unless the destination Fragment is empty." unless @num_documents == 0
+    # TODO: use a FragmentReader to access old data
+    fulltext_reader     = FulltextReader.new(:path => "#{fragment_directory}/fulltext")
+    suffix_array_reader = SuffixArrayReader.new(fulltext_reader, nil, 
+                                                :path => "#{fragment_directory}/suffixes")
+    doc_map_reader      = DocumentMapReader.new(:path => "#{fragment_directory}/docmap")
+    @fulltext_writer.merge(fulltext_reader)
+    @suffix_array_writer.merge(suffix_array_reader)
+    @doc_map_writer.merge(doc_map_reader)
+    #FIXME: .num_documents will be wrong if some URIs were repeated
+    @num_documents = doc_map_reader.num_documents
+    File.open(File.join(fragment_directory, "fieldmap"), "rb") do |f|
+      i = 0
+      f.each_line{|l| @field_map[l.chomp.to_sym] = i; i+= 1}
+    end
+  end
+
+  def fields
+    @field_map.sort_by{|field, fid| fid}.map{|field, fid| field}
+  end
+
+  def documents
+    @num_documents
+  end
+
+  def field_id(field)
+    @field_map.has_key?(field) && @field_map[field]
+  end
+
+  def finish!
+    @fulltext_writer.finish!
+    fulltext = @fulltext_writer.data
+    @suffix_array_writer.finish!(fulltext)
+    @doc_map_writer.finish!
+
+    if @path
+      File.open(File.join(@tmpdir, "fieldmap"), "wb") do |f|
+        @field_map.sort_by{|field_name, field_id| field_id}.each do |field_name, field_id| 
+          f.puts field_name
+        end
+        File.rename(@tmpdir, @path)
+      end
+    end
+  end
+end
+end  # FTSearch

data/shoes/ruby/lib/ftsearch/fulltext_reader.rb

@@ -1,52 +1,52 @@
-# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
-#
-
-module FTSearch
-
-class FulltextReader
-  DEFAULT_OPTIONS = {
-    :path => nil,
-    :io   => nil,
-  }
-  def initialize(options = {})
-    options = DEFAULT_OPTIONS.merge(options)
-    unless options[:path] || options[:io]
-      raise ArgumentError, "Need either the path to the suffix array file or an IO."
-    end
-    init_internal_structures(options)
-  end
-
-  def get_data(offset, size)
-    @io.pos = offset
-    @io.read(size)
-  end
-
-  def dump_data(&block)
-    blocksize = 32768
-    @io.pos = 0
-    begin
-      size = @io.stat.size - 1
-    rescue NoMethodError # try with StringIO's interface
-      size = @io.string.size - 1
-    end
-    read = 0
-    #buffer = " " * blocksize
-    while read < size
-      #@io.read([size - read, blocksize].min, buffer)
-      #yield buffer
-      data = @io.read([size - read, blocksize].min)
-      read += data.size
-      yield data
-    end
-  end
-
-  private
-  def init_internal_structures(options)
-    if options[:path]
-      @io = File.open(options[:path], "rb")
-    else
-      @io = options[:io]
-    end
-  end
-end
-end #  FTSearch
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+module FTSearch
+
+class FulltextReader
+  DEFAULT_OPTIONS = {
+    :path => nil,
+    :io   => nil,
+  }
+  def initialize(options = {})
+    options = DEFAULT_OPTIONS.merge(options)
+    unless options[:path] || options[:io]
+      raise ArgumentError, "Need either the path to the suffix array file or an IO."
+    end
+    init_internal_structures(options)
+  end
+
+  def get_data(offset, size)
+    @io.pos = offset
+    @io.read(size)
+  end
+
+  def dump_data(&block)
+    blocksize = 32768
+    @io.pos = 0
+    begin
+      size = @io.stat.size - 1
+    rescue NoMethodError # try with StringIO's interface
+      size = @io.string.size - 1
+    end
+    read = 0
+    #buffer = " " * blocksize
+    while read < size
+      #@io.read([size - read, blocksize].min, buffer)
+      #yield buffer
+      data = @io.read([size - read, blocksize].min)
+      read += data.size
+      yield data
+    end
+  end
+
+  private
+  def init_internal_structures(options)
+    if options[:path]
+      @io = File.open(options[:path], "rb")
+    else
+      @io = options[:io]
+    end
+  end
+end
+end #  FTSearch

data/shoes/ruby/lib/ftsearch/fulltext_writer.rb

@@ -1,75 +1,75 @@
-# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
-#
-
-require 'ftsearch/util'
-
-module FTSearch
-class FulltextWriter
-  include InMemoryWriter
-
-  DEFAULT_OPTIONS = {
-    :path => "fulltext-#{Process.pid}-#{rand(100000)}",
-  }
-
-  attr_reader :path
-
-  def initialize(options = {})
-    options = DEFAULT_OPTIONS.merge(options)
-    @path   = options[:path]
-    initialize_in_memory_buffer
-    if @path
-      @io     = File.open(@path, "wb")
-    else
-      @io     = @memory_io
-    end
-  end
-
-  def merge(fulltext_reader)
-    fulltext_reader.dump_data do |data|
-      @io.write data
-    end
-  end
-
-  def add_document(doc_id, doc_hash, field_mapping, field_infos, suffix_array_writer, doc_map_writer)
-    write_document_header(doc_id, doc_hash, field_mapping, field_infos)
-    doc_map_writer.add_document(doc_id, doc_hash[:uri])
-    doc_hash.each_pair do |field_name, data|
-      if field_id = field_mapping[field_name]
-        field_info = field_infos[field_name]
-        if field_info[:stored]
-          suffix_offset, segment_offset = store_field(doc_id, field_name, field_id, data)
-          if analyzer = field_info[:analyzer]
-            suffix_array_writer.add_suffixes(analyzer, data, suffix_offset)
-          end
-          doc_map_writer.add_field(segment_offset, doc_id, field_id, data.size)
-        end
-      end
-    end
-  end
-
-  def finish!
-    @io.write "\0"
-    @io.fsync
-    @io.close
-  end
-  
-  private
-  def write_document_header(doc_id, doc_hash, field_mapping, field_infos)
-    stored_fields = doc_hash.select do |field_name, data|
-      field_infos[field_name][:stored]
-    end
-    total_size = stored_fields.inject(0){|s,(_,data)| s + data.size} + stored_fields.size * 9
-    # 9 = field ids plus field size plus trailing \0
-    @io.write [total_size].pack("V")
-  end
-
-  def store_field(doc_id, field_name, field_id, data)
-    @io.write [field_id, data.size].pack("V2")
-    offset = @io.pos
-    @io.write data
-    @io.write "\0"
-
-    [offset, offset - 8]
-  end
-end
-end  # FTSearch
+# Copyright (C) 2006  Mauricio Fernandez <mfp@acm.org>
+#
+
+require 'ftsearch/util'
+
+module FTSearch
+class FulltextWriter
+  include InMemoryWriter
+
+  DEFAULT_OPTIONS = {
+    :path => "fulltext-#{Process.pid}-#{rand(100000)}",
+  }
+
+  attr_reader :path
+
+  def initialize(options = {})
+    options = DEFAULT_OPTIONS.merge(options)
+    @path   = options[:path]
+    initialize_in_memory_buffer
+    if @path
+      @io     = File.open(@path, "wb")
+    else
+      @io     = @memory_io
+    end
+  end
+
+  def merge(fulltext_reader)
+    fulltext_reader.dump_data do |data|
+      @io.write data
+    end
+  end
+
+  def add_document(doc_id, doc_hash, field_mapping, field_infos, suffix_array_writer, doc_map_writer)
+    write_document_header(doc_id, doc_hash, field_mapping, field_infos)
+    doc_map_writer.add_document(doc_id, doc_hash[:uri])
+    doc_hash.each_pair do |field_name, data|
+      if field_id = field_mapping[field_name]
+        field_info = field_infos[field_name]
+        if field_info[:stored]
+          suffix_offset, segment_offset = store_field(doc_id, field_name, field_id, data)
+          if analyzer = field_info[:analyzer]
+            suffix_array_writer.add_suffixes(analyzer, data, suffix_offset)
+          end
+          doc_map_writer.add_field(segment_offset, doc_id, field_id, data.size)
+        end
+      end
+    end
+  end
+
+  def finish!
+    @io.write "\0"
+    @io.fsync
+    @io.close
+  end
+  
+  private
+  def write_document_header(doc_id, doc_hash, field_mapping, field_infos)
+    stored_fields = doc_hash.select do |field_name, data|
+      field_infos[field_name][:stored]
+    end
+    total_size = stored_fields.inject(0){|s,(_,data)| s + data.size} + stored_fields.size * 9
+    # 9 = field ids plus field size plus trailing \0
+    @io.write [total_size].pack("V")
+  end
+
+  def store_field(doc_id, field_name, field_id, data)
+    @io.write [field_id, data.size].pack("V2")
+    offset = @io.pos
+    @io.write data
+    @io.write "\0"
+
+    [offset, offset - 8]
+  end
+end
+end  # FTSearch