thinking-sphinx-099 1.2.12

Sign up to get free protection for your applications and to get access to all the features.
Files changed (95) hide show
  1. data/LICENCE +20 -0
  2. data/README.textile +157 -0
  3. data/VERSION.yml +4 -0
  4. data/lib/thinking_sphinx.rb +211 -0
  5. data/lib/thinking_sphinx/active_record.rb +307 -0
  6. data/lib/thinking_sphinx/active_record/attribute_updates.rb +48 -0
  7. data/lib/thinking_sphinx/active_record/delta.rb +87 -0
  8. data/lib/thinking_sphinx/active_record/has_many_association.rb +28 -0
  9. data/lib/thinking_sphinx/active_record/scopes.rb +39 -0
  10. data/lib/thinking_sphinx/adapters/abstract_adapter.rb +42 -0
  11. data/lib/thinking_sphinx/adapters/mysql_adapter.rb +54 -0
  12. data/lib/thinking_sphinx/adapters/postgresql_adapter.rb +136 -0
  13. data/lib/thinking_sphinx/association.rb +164 -0
  14. data/lib/thinking_sphinx/attribute.rb +342 -0
  15. data/lib/thinking_sphinx/class_facet.rb +15 -0
  16. data/lib/thinking_sphinx/configuration.rb +282 -0
  17. data/lib/thinking_sphinx/core/array.rb +7 -0
  18. data/lib/thinking_sphinx/core/string.rb +15 -0
  19. data/lib/thinking_sphinx/deltas.rb +30 -0
  20. data/lib/thinking_sphinx/deltas/datetime_delta.rb +50 -0
  21. data/lib/thinking_sphinx/deltas/default_delta.rb +68 -0
  22. data/lib/thinking_sphinx/deltas/delayed_delta.rb +30 -0
  23. data/lib/thinking_sphinx/deltas/delayed_delta/delta_job.rb +24 -0
  24. data/lib/thinking_sphinx/deltas/delayed_delta/flag_as_deleted_job.rb +27 -0
  25. data/lib/thinking_sphinx/deltas/delayed_delta/job.rb +26 -0
  26. data/lib/thinking_sphinx/deploy/capistrano.rb +100 -0
  27. data/lib/thinking_sphinx/excerpter.rb +22 -0
  28. data/lib/thinking_sphinx/facet.rb +125 -0
  29. data/lib/thinking_sphinx/facet_search.rb +134 -0
  30. data/lib/thinking_sphinx/field.rb +82 -0
  31. data/lib/thinking_sphinx/index.rb +99 -0
  32. data/lib/thinking_sphinx/index/builder.rb +286 -0
  33. data/lib/thinking_sphinx/index/faux_column.rb +110 -0
  34. data/lib/thinking_sphinx/property.rb +162 -0
  35. data/lib/thinking_sphinx/rails_additions.rb +150 -0
  36. data/lib/thinking_sphinx/search.rb +707 -0
  37. data/lib/thinking_sphinx/search_methods.rb +421 -0
  38. data/lib/thinking_sphinx/source.rb +150 -0
  39. data/lib/thinking_sphinx/source/internal_properties.rb +46 -0
  40. data/lib/thinking_sphinx/source/sql.rb +128 -0
  41. data/lib/thinking_sphinx/tasks.rb +165 -0
  42. data/rails/init.rb +14 -0
  43. data/spec/lib/thinking_sphinx/active_record/delta_spec.rb +130 -0
  44. data/spec/lib/thinking_sphinx/active_record/has_many_association_spec.rb +49 -0
  45. data/spec/lib/thinking_sphinx/active_record/scopes_spec.rb +96 -0
  46. data/spec/lib/thinking_sphinx/active_record_spec.rb +364 -0
  47. data/spec/lib/thinking_sphinx/association_spec.rb +239 -0
  48. data/spec/lib/thinking_sphinx/attribute_spec.rb +500 -0
  49. data/spec/lib/thinking_sphinx/configuration_spec.rb +268 -0
  50. data/spec/lib/thinking_sphinx/core/array_spec.rb +9 -0
  51. data/spec/lib/thinking_sphinx/core/string_spec.rb +9 -0
  52. data/spec/lib/thinking_sphinx/excerpter_spec.rb +49 -0
  53. data/spec/lib/thinking_sphinx/facet_search_spec.rb +176 -0
  54. data/spec/lib/thinking_sphinx/facet_spec.rb +333 -0
  55. data/spec/lib/thinking_sphinx/field_spec.rb +154 -0
  56. data/spec/lib/thinking_sphinx/index/builder_spec.rb +455 -0
  57. data/spec/lib/thinking_sphinx/index/faux_column_spec.rb +30 -0
  58. data/spec/lib/thinking_sphinx/index_spec.rb +45 -0
  59. data/spec/lib/thinking_sphinx/rails_additions_spec.rb +203 -0
  60. data/spec/lib/thinking_sphinx/search_methods_spec.rb +152 -0
  61. data/spec/lib/thinking_sphinx/search_spec.rb +1092 -0
  62. data/spec/lib/thinking_sphinx/source_spec.rb +227 -0
  63. data/spec/lib/thinking_sphinx_spec.rb +162 -0
  64. data/tasks/distribution.rb +50 -0
  65. data/tasks/rails.rake +1 -0
  66. data/tasks/testing.rb +83 -0
  67. data/vendor/after_commit/LICENSE +20 -0
  68. data/vendor/after_commit/README +16 -0
  69. data/vendor/after_commit/Rakefile +22 -0
  70. data/vendor/after_commit/init.rb +8 -0
  71. data/vendor/after_commit/lib/after_commit.rb +45 -0
  72. data/vendor/after_commit/lib/after_commit/active_record.rb +114 -0
  73. data/vendor/after_commit/lib/after_commit/connection_adapters.rb +103 -0
  74. data/vendor/after_commit/test/after_commit_test.rb +53 -0
  75. data/vendor/delayed_job/lib/delayed/job.rb +251 -0
  76. data/vendor/delayed_job/lib/delayed/message_sending.rb +7 -0
  77. data/vendor/delayed_job/lib/delayed/performable_method.rb +55 -0
  78. data/vendor/delayed_job/lib/delayed/worker.rb +54 -0
  79. data/vendor/riddle/lib/riddle.rb +30 -0
  80. data/vendor/riddle/lib/riddle/client.rb +735 -0
  81. data/vendor/riddle/lib/riddle/client/filter.rb +53 -0
  82. data/vendor/riddle/lib/riddle/client/message.rb +70 -0
  83. data/vendor/riddle/lib/riddle/client/response.rb +94 -0
  84. data/vendor/riddle/lib/riddle/configuration.rb +33 -0
  85. data/vendor/riddle/lib/riddle/configuration/distributed_index.rb +49 -0
  86. data/vendor/riddle/lib/riddle/configuration/index.rb +146 -0
  87. data/vendor/riddle/lib/riddle/configuration/indexer.rb +19 -0
  88. data/vendor/riddle/lib/riddle/configuration/remote_index.rb +17 -0
  89. data/vendor/riddle/lib/riddle/configuration/searchd.rb +46 -0
  90. data/vendor/riddle/lib/riddle/configuration/section.rb +43 -0
  91. data/vendor/riddle/lib/riddle/configuration/source.rb +23 -0
  92. data/vendor/riddle/lib/riddle/configuration/sql_source.rb +39 -0
  93. data/vendor/riddle/lib/riddle/configuration/xml_source.rb +28 -0
  94. data/vendor/riddle/lib/riddle/controller.rb +53 -0
  95. metadata +172 -0
@@ -0,0 +1,53 @@
1
+ module Riddle
2
+ class Client
3
+ # Used for querying Sphinx.
4
+ class Filter
5
+ attr_accessor :attribute, :values, :exclude
6
+
7
+ # Attribute name, values (which can be an array or a range), and whether
8
+ # the filter should be exclusive.
9
+ def initialize(attribute, values, exclude=false)
10
+ @attribute, @values, @exclude = attribute, values, exclude
11
+ end
12
+
13
+ def exclude?
14
+ self.exclude
15
+ end
16
+
17
+ # Returns the message for this filter to send to the Sphinx service
18
+ def query_message
19
+ message = Message.new
20
+
21
+ message.append_string self.attribute.to_s
22
+ case self.values
23
+ when Range
24
+ if self.values.first.is_a?(Float) && self.values.last.is_a?(Float)
25
+ message.append_int FilterTypes[:float_range]
26
+ message.append_floats self.values.first, self.values.last
27
+ else
28
+ message.append_int FilterTypes[:range]
29
+ message.append_64bit_ints self.values.first, self.values.last
30
+ end
31
+ when Array
32
+ message.append_int FilterTypes[:values]
33
+ message.append_int self.values.length
34
+ # using to_f is a hack from the php client - to workaround 32bit
35
+ # signed ints on x32 platforms
36
+ message.append_64bit_ints *self.values.collect { |val|
37
+ case val
38
+ when TrueClass
39
+ 1
40
+ when FalseClass
41
+ 0
42
+ else
43
+ val
44
+ end
45
+ }
46
+ end
47
+ message.append_int self.exclude? ? 1 : 0
48
+
49
+ message.to_s
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,70 @@
1
+ module Riddle
2
+ class Client
3
+ # This class takes care of the translation of ints, strings and arrays to
4
+ # the format required by the Sphinx service.
5
+ class Message
6
+ def initialize
7
+ @message = ""
8
+ @size_method = @message.respond_to?(:bytesize) ? :bytesize : :length
9
+ end
10
+
11
+ # Append raw data (only use if you know what you're doing)
12
+ def append(*args)
13
+ args.each { |arg| @message << arg }
14
+ end
15
+
16
+ # Append a string's length, then the string itself
17
+ def append_string(str)
18
+ string = str.respond_to?(:force_encoding) ?
19
+ str.dup.force_encoding('ASCII-8BIT') : str
20
+
21
+ @message << [string.send(@size_method)].pack('N') + string
22
+ end
23
+
24
+ # Append an integer
25
+ def append_int(int)
26
+ @message << [int].pack('N')
27
+ end
28
+
29
+ def append_64bit_int(int)
30
+ @message << [int >> 32, int & 0xFFFFFFFF].pack('NN')
31
+ end
32
+
33
+ # Append a float
34
+ def append_float(float)
35
+ @message << [float].pack('f').unpack('L*').pack("N")
36
+ end
37
+
38
+ def append_boolean(bool)
39
+ append_int(bool ? 1 : 0)
40
+ end
41
+
42
+ # Append multiple integers
43
+ def append_ints(*ints)
44
+ ints.each { |int| append_int(int) }
45
+ end
46
+
47
+ def append_64bit_ints(*ints)
48
+ ints.each { |int| append_64bit_int(int) }
49
+ end
50
+
51
+ # Append multiple floats
52
+ def append_floats(*floats)
53
+ floats.each { |float| append_float(float) }
54
+ end
55
+
56
+ # Append an array of strings - first appends the length of the array,
57
+ # then each item's length and value.
58
+ def append_array(array)
59
+ append_int(array.length)
60
+
61
+ array.each { |item| append_string(item) }
62
+ end
63
+
64
+ # Returns the entire message
65
+ def to_s
66
+ @message
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,94 @@
1
+ module Riddle
2
+ class Client
3
+ # Used to interrogate responses from the Sphinx daemon. Keep in mind none
4
+ # of the methods here check whether the data they're grabbing are what the
5
+ # user expects - it just assumes the user knows what the data stream is
6
+ # made up of.
7
+ class Response
8
+ # Create with the data to interpret
9
+ def initialize(str)
10
+ @str = str
11
+ @marker = 0
12
+ end
13
+
14
+ # Return the next string value in the stream
15
+ def next
16
+ len = next_int
17
+ result = @str[@marker, len]
18
+ @marker += len
19
+
20
+ return result
21
+ end
22
+
23
+ # Return the next integer value from the stream
24
+ def next_int
25
+ int = @str[@marker, 4].unpack('N*').first
26
+ @marker += 4
27
+
28
+ return int
29
+ end
30
+
31
+ def next_64bit_int
32
+ high, low = @str[@marker, 8].unpack('N*N*')[0..1]
33
+ @marker += 8
34
+
35
+ return (high << 32) + low
36
+ end
37
+
38
+ # Return the next float value from the stream
39
+ def next_float
40
+ float = @str[@marker, 4].unpack('N*').pack('L').unpack('f*').first
41
+ @marker += 4
42
+
43
+ return float
44
+ end
45
+
46
+ # Returns an array of string items
47
+ def next_array
48
+ count = next_int
49
+ items = []
50
+ for i in 0...count
51
+ items << self.next
52
+ end
53
+
54
+ return items
55
+ end
56
+
57
+ # Returns an array of int items
58
+ def next_int_array
59
+ count = next_int
60
+ items = []
61
+ for i in 0...count
62
+ items << self.next_int
63
+ end
64
+
65
+ return items
66
+ end
67
+
68
+ def next_float_array
69
+ count = next_int
70
+ items = []
71
+ for i in 0...count
72
+ items << self.next_float
73
+ end
74
+
75
+ return items
76
+ end
77
+
78
+ def next_64bit_int_array
79
+ count = next_int
80
+ items = []
81
+ for i in 0...count
82
+ items << self.next_64bit_int
83
+ end
84
+
85
+ return items
86
+ end
87
+
88
+ # Returns the length of the streamed data
89
+ def length
90
+ @str.length
91
+ end
92
+ end
93
+ end
94
+ end
@@ -0,0 +1,33 @@
1
+ require 'riddle/configuration/section'
2
+
3
+ require 'riddle/configuration/distributed_index'
4
+ require 'riddle/configuration/index'
5
+ require 'riddle/configuration/indexer'
6
+ require 'riddle/configuration/remote_index'
7
+ require 'riddle/configuration/searchd'
8
+ require 'riddle/configuration/source'
9
+ require 'riddle/configuration/sql_source'
10
+ require 'riddle/configuration/xml_source'
11
+
12
+ module Riddle
13
+ class Configuration
14
+ class ConfigurationError < StandardError #:nodoc:
15
+ end
16
+
17
+ attr_reader :indexes, :searchd
18
+ attr_accessor :indexer
19
+
20
+ def initialize
21
+ @indexer = Riddle::Configuration::Indexer.new
22
+ @searchd = Riddle::Configuration::Searchd.new
23
+ @indexes = []
24
+ end
25
+
26
+ def render
27
+ (
28
+ [@indexer.render, @searchd.render] +
29
+ @indexes.collect { |index| index.render }
30
+ ).join("\n")
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,49 @@
1
+ module Riddle
2
+ class Configuration
3
+ class DistributedIndex < Riddle::Configuration::Section
4
+ self.settings = [:type, :local, :agent, :agent_blackhole,
5
+ :agent_connect_timeout, :agent_query_timeout]
6
+
7
+ attr_accessor :name, :local_indexes, :remote_indexes, :agent_blackhole,
8
+ :agent_connect_timeout, :agent_query_timeout
9
+
10
+ def initialize(name)
11
+ @name = name
12
+ @local_indexes = []
13
+ @remote_indexes = []
14
+ @agent_blackhole = []
15
+ end
16
+
17
+ def type
18
+ "distributed"
19
+ end
20
+
21
+ def local
22
+ self.local_indexes
23
+ end
24
+
25
+ def agent
26
+ agents = remote_indexes.collect { |index| index.remote }.uniq
27
+ agents.collect { |agent|
28
+ agent + ":" + remote_indexes.select { |index|
29
+ index.remote == agent
30
+ }.collect { |index| index.name }.join(",")
31
+ }
32
+ end
33
+
34
+ def render
35
+ raise ConfigurationError unless valid?
36
+
37
+ (
38
+ ["index #{name}", "{"] +
39
+ settings_body +
40
+ ["}", ""]
41
+ ).join("\n")
42
+ end
43
+
44
+ def valid?
45
+ @local_indexes.length > 0 || @remote_indexes.length > 0
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,146 @@
1
+ module Riddle
2
+ class Configuration
3
+ class Index < Riddle::Configuration::Section
4
+ self.settings = [:source, :path, :docinfo, :mlock, :morphology,
5
+ :min_stemming_len, :stopwords, :wordforms, :exceptions, :min_word_len,
6
+ :charset_type, :charset_table, :ignore_chars, :min_prefix_len,
7
+ :min_infix_len, :prefix_fields, :infix_fields, :enable_star,
8
+ :ngram_len, :ngram_chars, :phrase_boundary, :phrase_boundary_step,
9
+ :html_strip, :html_index_attrs, :html_remove_elements, :preopen,
10
+ :ondisk_dict, :inplace_enable, :inplace_hit_gap, :inplace_docinfo_gap,
11
+ :inplace_reloc_factor, :inplace_write_factor, :index_exact_words]
12
+
13
+ attr_accessor :name, :parent, :sources, :path, :docinfo, :mlock,
14
+ :morphologies, :min_stemming_len, :stopword_files, :wordform_files,
15
+ :exception_files, :min_word_len, :charset_type, :charset_table,
16
+ :ignore_characters, :min_prefix_len, :min_infix_len,
17
+ :prefix_field_names, :infix_field_names, :enable_star, :ngram_len,
18
+ :ngram_characters, :phrase_boundaries, :phrase_boundary_step,
19
+ :html_strip, :html_index_attrs, :html_remove_element_tags, :preopen,
20
+ :ondisk_dict, :inplace_enable, :inplace_hit_gap, :inplace_docinfo_gap,
21
+ :inplace_reloc_factor, :inplace_write_factor, :index_exact_words
22
+
23
+ def initialize(name, *sources)
24
+ @name = name
25
+ @sources = sources
26
+ @morphologies = []
27
+ @stopword_files = []
28
+ @wordform_files = []
29
+ @exception_files = []
30
+ @ignore_characters = []
31
+ @prefix_field_names = []
32
+ @infix_field_names = []
33
+ @ngram_characters = []
34
+ @phrase_boundaries = []
35
+ @html_remove_element_tags = []
36
+ end
37
+
38
+ def source
39
+ @sources.collect { |s| s.name }
40
+ end
41
+
42
+ def morphology
43
+ nil_join @morphologies, ", "
44
+ end
45
+
46
+ def morphology=(morphology)
47
+ @morphologies = nil_split morphology, /,\s?/
48
+ end
49
+
50
+ def stopwords
51
+ nil_join @stopword_files, " "
52
+ end
53
+
54
+ def stopwords=(stopwords)
55
+ @stopword_files = nil_split stopwords, ' '
56
+ end
57
+
58
+ def wordforms
59
+ nil_join @wordform_files, " "
60
+ end
61
+
62
+ def wordforms=(wordforms)
63
+ @wordform_files = nil_split wordforms, ' '
64
+ end
65
+
66
+ def exceptions
67
+ nil_join @exception_files, " "
68
+ end
69
+
70
+ def exceptions=(exceptions)
71
+ @exception_files = nil_split exceptions, ' '
72
+ end
73
+
74
+ def ignore_chars
75
+ nil_join @ignore_characters, ", "
76
+ end
77
+
78
+ def ignore_chars=(ignore_chars)
79
+ @ignore_characters = nil_split ignore_chars, /,\s?/
80
+ end
81
+
82
+ def prefix_fields
83
+ nil_join @prefix_field_names, ", "
84
+ end
85
+
86
+ def infix_fields
87
+ nil_join @infix_field_names, ", "
88
+ end
89
+
90
+ def ngram_chars
91
+ nil_join @ngram_characters, ", "
92
+ end
93
+
94
+ def ngram_chars=(ngram_chars)
95
+ @ngram_characters = nil_split ngram_chars, /,\s?/
96
+ end
97
+
98
+ def phrase_boundary
99
+ nil_join @phrase_boundaries, ", "
100
+ end
101
+
102
+ def phrase_boundary=(phrase_boundary)
103
+ @phrase_boundaries = nil_split phrase_boundary, /,\s?/
104
+ end
105
+
106
+ def html_remove_elements
107
+ nil_join @html_remove_element_tags, ", "
108
+ end
109
+
110
+ def html_remove_elements=(html_remove_elements)
111
+ @html_remove_element_tags = nil_split html_remove_elements, /,\s?/
112
+ end
113
+
114
+ def render
115
+ raise ConfigurationError, "#{@name} #{@sources.inspect} #{@path} #{@parent}" unless valid?
116
+
117
+ inherited_name = "#{name}"
118
+ inherited_name << " : #{parent}" if parent
119
+ (
120
+ @sources.collect { |s| s.render } +
121
+ ["index #{inherited_name}", "{"] +
122
+ settings_body +
123
+ ["}", ""]
124
+ ).join("\n")
125
+ end
126
+
127
+ def valid?
128
+ (!@name.nil?) && (!( @sources.length == 0 || @path.nil? ) || !@parent.nil?)
129
+ end
130
+
131
+ private
132
+
133
+ def nil_split(string, pattern)
134
+ (string || "").split(pattern)
135
+ end
136
+
137
+ def nil_join(array, delimiter)
138
+ if array.length == 0
139
+ nil
140
+ else
141
+ array.join(delimiter)
142
+ end
143
+ end
144
+ end
145
+ end
146
+ end