ts-xml 0.0.1

data/lib/thinking_sphinx/xml/adapters/oracle_adapter.rb

@@ -0,0 +1,123 @@
+module ThinkingSphinx
+  class OracleAdapter < AbstractAdapter
+    def setup
+      create_group_concat_function
+      create_crc32_function
+    end
+    
+    def sphinx_identifier
+      "odbc"
+    end
+    
+    def concatenate(clause, separator = ' ')
+      clause.split(', ').collect { |field| field }.join(" || '#{separator}' || ")
+    end
+    
+    def group_concatenate(clause, separator = ' ')
+      "TO_STRING(CAST(COLLECT(TO_CHAR(#{clause})) AS SYS.ODCIVARCHAR2LIST), '#{separator}')"
+    end
+    
+    def cast_to_string(clause)
+      "TO_CHAR(#{clause})"
+    end
+    
+    def cast_to_datetime(clause)
+      "(TO_DATE(TO_CHAR(#{clause}, 'YYYY-MON-DD HH24.MI.SS'), 'YYYY-MON-DD HH24.MI.SS') - TO_DATE('01-JAN-1970','DD-MON-YYYY')) * (86400)"
+    end
+    
+    def cast_to_unsigned(clause)
+      "CAST(#{clause} AS NUMBER(10,0))"
+    end
+    
+    def convert_nulls(clause, default = '')
+      return clause if default == ''
+      default = "'#{default}'" if default.is_a?(String)
+      "NVL(#{clause},#{default})"
+    end
+    
+    def boolean(value)
+      value ? '1' : '0'
+    end
+    
+    # TODO
+    def crc(clause, blank_to_null = false)
+      "CRC32(#{clause})"
+    end
+    
+    def utf8_query_pre
+      nil
+    end
+    
+    def time_difference(diff)
+      "SYSDATE - #{diff}/(86400)"
+    end
+    
+    def select_each(query)
+      cursor = connection.raw_connection.exec(query)
+      col_names = cursor.get_col_names.collect(&:downcase)
+      while values = cursor.fetch
+        hash_values = Hash[*col_names.zip(values).flatten]
+        yield hash_values
+      end
+    ensure
+      cursor.close if cursor
+    end
+    
+    private
+    
+    # Requires Oracle 10g+
+    # Return only first 4000 bytes
+    def create_group_concat_function
+      connection.execute <<-SQL
+        CREATE OR REPLACE FUNCTION to_string (
+          nt_in IN SYS.ODCIVARCHAR2LIST,
+          delimiter_in IN VARCHAR2 DEFAULT ','
+        ) RETURN VARCHAR2 IS
+          v_idx PLS_INTEGER;
+          v_str VARCHAR2(4000);
+          v_dlm VARCHAR2(1);
+        BEGIN
+          v_idx := nt_in.FIRST;
+          WHILE v_idx IS NOT NULL LOOP
+            v_str := SUBSTRB(v_str || v_dlm || nt_in(v_idx), 1, 4000);
+            v_dlm := delimiter_in;
+            v_idx := nt_in.NEXT(v_idx);
+          END LOOP;
+          RETURN v_str;
+        END to_string;
+      SQL
+    end
+    
+    # Requires Oracle 10g+
+    def create_crc32_function
+      connection.execute <<-SQL
+        CREATE OR REPLACE FUNCTION crc32(
+          word IN VARCHAR2
+        ) RETURN NUMBER IS
+          code NUMBER(4,0);
+          i NUMBER(10,0);
+          j NUMBER(1,0);
+          tmp NUMBER(10,0);
+          tmp_a NUMBER(10,0);
+          tmp_b NUMBER(10,0);
+        BEGIN
+          tmp := 4294967295;
+          i := 0;
+          WHILE i < length(word) LOOP
+            code := ascii(SUBSTR(word, i + 1, 1));
+            tmp := tmp - 2 * to_number(bitand(tmp, code)) + code;
+            j := 0;
+            WHILE j < 8 LOOP
+              tmp_a := floor(tmp / 2);
+              tmp_b := 3988292384 * to_number(bitand(tmp, 1));
+              tmp := tmp_a - 2 * to_number(bitand(tmp_a, tmp_b)) + tmp_b;
+              j := j + 1;
+            END LOOP;
+            i := i + 1;
+          END LOOP;
+          RETURN tmp - 2 * to_number(bitand(tmp, 4294967295)) + 4294967295;
+        END crc32;
+      SQL
+    end
+  end
+end

data/lib/thinking_sphinx/xml/adapters/sqlite3_adapter.rb

@@ -0,0 +1,95 @@
+module ThinkingSphinx
+  class SQLite3Adapter < AbstractAdapter
+    def setup
+      create_group_concat_function
+      create_crc32_function
+    end
+    
+    def sphinx_identifier
+      "sqlite3"
+    end
+    
+    def concatenate(clause, separator = ' ')
+      clause.split(', ').collect { |field| field }.join(" || '#{separator}' || ")
+    end
+    
+    def group_concatenate(clause, separator = ' ')
+      "GROUP_CONCAT(#{clause}, '#{separator}')"
+    end
+    
+    def cast_to_string(clause)
+      "CAST(#{clause} AS TEXT)"
+    end
+    
+    def cast_to_datetime(clause)
+      "STRFTIME('%s', #{clause})"
+    end
+    
+    def cast_to_unsigned(clause)
+      "CAST(#{clause} AS INTEGER)"
+    end
+    
+    def convert_nulls(clause, default = '')
+      default = "'#{default}'" if default.is_a?(String)
+      "IFNULL(#{clause},#{default})"
+    end
+    
+    def boolean(value)
+      value ? "'t'" : "'f'"
+    end
+    
+    def crc(clause, blank_to_null = false)
+      "CRC32(#{clause})"
+    end
+    
+    def utf8_query_pre
+      nil
+    end
+    
+    def time_difference(diff)
+      "STRFTIME('%s','now') - STRFTIME('%s', #{diff})"
+    end
+    
+    private
+    
+    def create_group_concat_function
+      connection.raw_connection.create_aggregate("group_concat", 2) do
+        step do |func, value, separator|
+          if !value.null? && !separator.null?
+            if func[:concat].nil?
+              func[:concat] = value.to_s
+            else
+              func[:concat] << separator.to_s
+              func[:concat] << value.to_s
+            end
+          end
+        end
+        finalize do |func|
+          func.result = func[:concat]
+        end
+      end
+    end
+    
+    def create_crc32_function
+      connection.raw_connection.create_function("crc32", 1) do |func, value|
+        r = 0
+        if !value.null?
+          c = value.to_s
+          n = c.length
+          r = 0xFFFFFFFF
+          n.times do |i|
+            r ^= c[i]
+            8.times do
+              if (r & 1) != 0
+                r = (r >> 1) ^ 0xEDB88320
+              else
+                r >>= 1
+              end
+            end
+          end
+        end
+        func.result = r ^ 0xFFFFFFFF
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/xml/source.rb

@@ -0,0 +1,46 @@
+module ThinkingSphinx
+  class Source
+    def to_riddle_for_core(offset, position)
+      source = Riddle::Configuration::XMLSource.new(
+        "#{index.core_name}_#{position}", "xmlpipe2"
+      )
+      
+      set_source_xml_settings       source, offset
+      set_source_attributes         source, offset
+      set_source_settings           source
+      
+      source
+    end
+    
+    def to_riddle_for_delta(offset, position)
+      source = Riddle::Configuration::XMLSource.new(
+        "#{index.delta_name}_#{position}", "xmlpipe2"
+      )
+      source.parent = "#{index.core_name}_#{position}"
+      
+      set_source_xml_settings       source, offset
+      set_source_attributes         source, offset
+      set_source_settings           source
+      
+      source
+    end
+    
+    private
+    
+    def set_source_xml_settings(source, offset)
+      env = ThinkingSphinx::Configuration.instance.environment
+      database = ENV['DATABASE'] || 'mysql'
+      source.xmlpipe_command = "rake -s ts:xml NAME=#{source.name} OFFSET=#{offset} RAILS_ENV=#{env} MERB_ENV=#{env} DATABASE=#{database}"
+      @fields.each do |field|
+        source.xmlpipe_field << field.unique_name
+      end
+    end
+    
+    def set_source_attributes(source, offset)
+      attributes.each do |attrib|
+        type = attrib.type_to_config.to_s.sub("sql_", "xmlpipe_")
+        source.send(type) << attrib.unique_name
+      end
+    end
+  end
+end

data/lib/thinking_sphinx/xml/tasks.rb

@@ -0,0 +1,95 @@
+require 'thinking_sphinx'
+require 'thinking_sphinx/xml'
+
+namespace :thinking_sphinx do
+  # http://www.sphinxsearch.com/docs/current.html#xmlpipe2
+  desc "Streams XML data to STDOUT"
+  task :xml => :app_env do
+    # ThinkingSphinx::Configuration.instance.load_models
+    source_name = ENV["NAME"]
+    # STDERR.puts "Source name: #{source_name}"
+    source_name =~ /^(.+)_(delta|core)_(\d+)$/
+    sphinx_name = $1
+    delta = $2 == "delta"
+    index_name  = "#{$1}_#{$2}"
+    offset      = ENV["OFFSET"]
+    ThinkingSphinx.context.prepare
+    ThinkingSphinx.context.define_indexes
+    index = nil
+    model_klass = ThinkingSphinx.context.indexed_models.collect{ |m| m.constantize }.detect do |model|
+      index = model.sphinx_indexes.detect { |i| i.name == sphinx_name }
+    end
+    # STDERR.puts model_klass.name
+    # STDERR.puts index
+    model_klass.sphinx_database_adapter.setup
+    
+    source = index.sources.first
+    sql_query_range = source.to_sql_query_range(:delta => delta)
+    sql_query = source.to_sql(:offset => offset, :delta => delta)
+    
+    # get $start, $end
+    # STDERR.puts sql_query_range
+    range_start, range_end = model_klass.connection.select_rows(sql_query_range).first.collect(&:to_i)
+    sql_query = sql_query.gsub('$start', range_start.to_s).gsub('$end', range_end.to_s)
+    
+    # fetch sql and generate xml
+    puts %{<?xml version="1.0" encoding="utf-8"?>}
+    puts %{<sphinx:docset>}
+    
+    # results = model_klass.connection.select_all(query)
+    # STDERR.puts "fetching #{start_id}.. "
+    # STDERR.puts sql_query
+    model_klass.sphinx_database_adapter.select_each(sql_query) do |values|
+      pk_name = model_klass.primary_key_for_sphinx.to_s
+      id = values.delete(pk_name)
+      puts %{<sphinx:document id="#{id.to_i}">}
+      values.each do |k, v|
+        attribute = source.attributes.detect { |a| a.unique_name == k.to_sym }
+        # STDERR.puts attribute
+        case attribute && attribute.type
+        when :boolean
+          v = ['Y', 'T', '1', 1, true].include?(v) ? 1 : 0
+        when :integer, :datetime
+          v = v.to_i
+        when :float
+          v = v.to_f
+        else
+          v = v.to_s.to_xs
+        end
+        puts %{<#{k}>#{v}</#{k}>}
+      end
+      # Add MVA attributes (:ranged_query, :query)
+      internal_id = values["sphinx_internal_id"]
+      multi_attributes = source.attributes.select { |a| a.type == :multi && !a.include_as_association? }
+      multi_attributes.each do |attribute|
+        k = attribute.unique_name
+        mva_query = attribute.send(:query, offset)
+        mva_query_range = attribute.send(:range_query)
+        # STDERR.puts "MVA_ATTRS: #{params.inspect}"
+        # mva_definition, mva_query, mva_query_range = params.split(";")
+        if mva_query
+          mva_query = mva_query.gsub('$start', internal_id.to_s).gsub('$end', internal_id.to_s)
+          printf %{<#{k}>}
+          not_first = false
+          model_klass.sphinx_database_adapter.select_each(mva_query) do |values|
+            if not_first
+              printf ","
+            else
+              not_first = true
+            end
+            printf values.values.first.to_s.to_xs
+          end
+          puts %{</#{k}>}
+        end
+      end
+      puts %{</sphinx:document>}
+    end
+    
+    puts %{</sphinx:docset>}
+  end
+end
+
+namespace :ts do
+  desc "Streams XML data to STDOUT"
+  task :xml => "thinking_sphinx:xml"
+end

data/spec/cucumber_env.rb

@@ -0,0 +1,22 @@
+require 'rubygems'
+require 'cucumber'
+require 'spec'
+require 'fileutils'
+require 'ginger'
+require 'will_paginate'
+require 'active_record'
+
+$:.unshift File.dirname(__FILE__) + '/../../lib'
+Dir[File.join(File.dirname(__FILE__), '../../vendor/*/lib')].each do |path|
+  $:.unshift path
+end
+
+require 'cucumber/thinking_sphinx/internal_world'
+
+world = Cucumber::ThinkingSphinx::InternalWorld.new
+world.configure_database
+
+require "thinking_sphinx"
+require "thinking_sphinx/xml"
+
+world.setup

metadata

@@ -0,0 +1,183 @@
+--- !ruby/object:Gem::Specification 
+name: ts-xml
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Edgars Beigarts
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-12-09 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thinking-sphinx
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.3.8
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.9
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: yard
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: cucumber
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Support for Oracle, SQLite3 using xmlpipe2 for Thinking Sphinx
+email: 1@wb4.lv
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.textile
+files: 
+- LICENSE
+- README.textile
+- lib/thinking_sphinx/xml.rb
+- lib/thinking_sphinx/xml/adapters/abstract_adapter.rb
+- lib/thinking_sphinx/xml/adapters/oracle_adapter.rb
+- lib/thinking_sphinx/xml/adapters/sqlite3_adapter.rb
+- lib/thinking_sphinx/xml/source.rb
+- lib/thinking_sphinx/xml/tasks.rb
+has_rdoc: true
+homepage: http://github.com/ebeigarts/ts-xml
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Thinking Sphinx - XML
+test_files: 
+- features/alternate_primary_key.feature
+- features/attribute_transformation.feature
+- features/attribute_updates.feature
+- features/deleting_instances.feature
+- features/direct_attributes.feature
+- features/excerpts.feature
+- features/extensible_delta_indexing.feature
+- features/facets.feature
+- features/facets_across_model.feature
+- features/handling_edits.feature
+- features/retry_stale_indexes.feature
+- features/searching_across_models.feature
+- features/searching_by_index.feature
+- features/searching_by_model.feature
+- features/searching_with_find_arguments.feature
+- features/sphinx_detection.feature
+- features/sphinx_scopes.feature
+- features/step_definitions/alpha_steps.rb
+- features/step_definitions/beta_steps.rb
+- features/step_definitions/common_steps.rb
+- features/step_definitions/extensible_delta_indexing_steps.rb
+- features/step_definitions/facet_steps.rb
+- features/step_definitions/find_arguments_steps.rb
+- features/step_definitions/gamma_steps.rb
+- features/step_definitions/scope_steps.rb
+- features/step_definitions/search_steps.rb
+- features/step_definitions/sphinx_steps.rb
+- features/sti_searching.feature
+- features/support/database.example.yml
+- features/support/db/fixtures/alphas.rb
+- features/support/db/fixtures/authors.rb
+- features/support/db/fixtures/betas.rb
+- features/support/db/fixtures/boxes.rb
+- features/support/db/fixtures/categories.rb
+- features/support/db/fixtures/cats.rb
+- features/support/db/fixtures/comments.rb
+- features/support/db/fixtures/developers.rb
+- features/support/db/fixtures/dogs.rb
+- features/support/db/fixtures/extensible_betas.rb
+- features/support/db/fixtures/foxes.rb
+- features/support/db/fixtures/gammas.rb
+- features/support/db/fixtures/people.rb
+- features/support/db/fixtures/posts.rb
+- features/support/db/fixtures/robots.rb
+- features/support/db/fixtures/tags.rb
+- features/support/db/migrations/create_alphas.rb
+- features/support/db/migrations/create_animals.rb
+- features/support/db/migrations/create_authors.rb
+- features/support/db/migrations/create_authors_posts.rb
+- features/support/db/migrations/create_betas.rb
+- features/support/db/migrations/create_boxes.rb
+- features/support/db/migrations/create_categories.rb
+- features/support/db/migrations/create_comments.rb
+- features/support/db/migrations/create_developers.rb
+- features/support/db/migrations/create_extensible_betas.rb
+- features/support/db/migrations/create_gammas.rb
+- features/support/db/migrations/create_people.rb
+- features/support/db/migrations/create_posts.rb
+- features/support/db/migrations/create_robots.rb
+- features/support/db/migrations/create_taggings.rb
+- features/support/db/migrations/create_tags.rb
+- features/support/env.rb
+- features/support/lib/generic_delta_handler.rb
+- features/support/models/alpha.rb
+- features/support/models/animal.rb
+- features/support/models/author.rb
+- features/support/models/beta.rb
+- features/support/models/box.rb
+- features/support/models/cat.rb
+- features/support/models/category.rb
+- features/support/models/comment.rb
+- features/support/models/developer.rb
+- features/support/models/dog.rb
+- features/support/models/extensible_beta.rb
+- features/support/models/fox.rb
+- features/support/models/gamma.rb
+- features/support/models/person.rb
+- features/support/models/post.rb
+- features/support/models/robot.rb
+- features/support/models/tag.rb
+- features/support/models/tagging.rb
+- spec/cucumber_env.rb