picky 0.12.3 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/lib/deployment.rb +2 -2
  2. data/lib/picky/application.rb +172 -12
  3. data/lib/picky/cacher/generator.rb +1 -1
  4. data/lib/picky/calculations/location.rb +9 -1
  5. data/lib/picky/character_substituters/west_european.rb +1 -1
  6. data/lib/picky/configuration/index.rb +1 -1
  7. data/lib/picky/cores.rb +1 -1
  8. data/lib/picky/extensions/array.rb +1 -1
  9. data/lib/picky/extensions/hash.rb +1 -1
  10. data/lib/picky/extensions/module.rb +1 -1
  11. data/lib/picky/extensions/object.rb +1 -1
  12. data/lib/picky/extensions/symbol.rb +1 -1
  13. data/lib/picky/generator.rb +2 -2
  14. data/lib/picky/helpers/cache.rb +7 -5
  15. data/lib/picky/helpers/gc.rb +2 -0
  16. data/lib/picky/helpers/measuring.rb +2 -0
  17. data/lib/picky/index/bundle.rb +1 -1
  18. data/lib/picky/index_api.rb +33 -15
  19. data/lib/picky/indexed/bundle.rb +1 -1
  20. data/lib/picky/indexed/index.rb +1 -1
  21. data/lib/picky/indexed/wrappers/bundle/location.rb +1 -1
  22. data/lib/picky/indexers/no_source_specified_error.rb +1 -1
  23. data/lib/picky/indexes_api.rb +1 -1
  24. data/lib/picky/indexing/bundle.rb +1 -1
  25. data/lib/picky/indexing/index.rb +1 -1
  26. data/lib/picky/loader.rb +1 -1
  27. data/lib/picky/loggers/search.rb +1 -1
  28. data/lib/picky/performant.rb +3 -0
  29. data/lib/picky/query/allocation.rb +1 -1
  30. data/lib/picky/query/allocations.rb +1 -1
  31. data/lib/picky/query/base.rb +48 -16
  32. data/lib/picky/query/combination.rb +1 -1
  33. data/lib/picky/query/combinations.rb +1 -1
  34. data/lib/picky/query/full.rb +7 -2
  35. data/lib/picky/query/live.rb +9 -7
  36. data/lib/picky/query/qualifiers.rb +6 -2
  37. data/lib/picky/query/solr.rb +1 -1
  38. data/lib/picky/query/token.rb +2 -1
  39. data/lib/picky/query/tokens.rb +4 -1
  40. data/lib/picky/query/weigher.rb +1 -1
  41. data/lib/picky/query/weights.rb +1 -1
  42. data/lib/picky/rack/harakiri.rb +14 -5
  43. data/lib/picky/results/base.rb +1 -1
  44. data/lib/picky/routing.rb +1 -1
  45. data/lib/picky/solr/schema_generator.rb +2 -1
  46. data/lib/picky/sources/base.rb +39 -25
  47. data/lib/picky/sources/couch.rb +22 -8
  48. data/lib/picky/sources/csv.rb +29 -6
  49. data/lib/picky/sources/db.rb +46 -30
  50. data/lib/picky/sources/delicious.rb +12 -2
  51. data/lib/picky/sources/wrappers/base.rb +3 -1
  52. data/lib/picky/tokenizers/base.rb +1 -1
  53. data/project_prototype/Gemfile +1 -1
  54. data/project_prototype/app/README +0 -1
  55. data/spec/lib/calculations/location_spec.rb +28 -16
  56. data/spec/lib/index_api_spec.rb +64 -0
  57. data/spec/lib/indexed/index_spec.rb +2 -2
  58. data/spec/lib/indexed/wrappers/exact_first_spec.rb +2 -2
  59. data/spec/lib/indexing/index_spec.rb +2 -2
  60. data/spec/lib/rack/harakiri_spec.rb +22 -10
  61. metadata +7 -4
@@ -1,11 +1,32 @@
1
1
  module Sources
2
2
 
3
- # Describes a database source. Just give it a select statement
3
+ # Describes a database source. Needs a SELECT statement
4
4
  # (with id in it), and a file option or the options from an AR config file.
5
5
  #
6
+ # The select statement can be as complicated as you want,
7
+ # as long as it has an id in it and as long as it can be
8
+ # used in a CREATE TABLE AS statement.
9
+ # (working on that last one)
10
+ #
11
+ # Examples:
12
+ # Sources::DB.new('SELECT id, title, author, year FROM books') # Uses the config from app/db.yml by default.
13
+ # Sources::DB.new('SELECT id, title, author, year FROM books', file: 'app/some_db.yml')
14
+ # Sources::DB.new('SELECT b.id, b.title, b.author, b.publishing_year as year FROM books b INNER JOIN ON ...', file: 'app/some_db.yml')
15
+ # Sources::DB.new('SELECT id, title, author, year FROM books', adapter: 'mysql', host:'localhost', ...)
16
+ #
6
17
  class DB < Base
7
18
 
8
- attr_reader :select_statement, :database, :connection_options
19
+ # The select statement that was passed in.
20
+ #
21
+ attr_reader :select_statement
22
+
23
+ # The database adapter.
24
+ #
25
+ attr_reader :database
26
+
27
+ # The database connection options that were either passed in or loaded from the given file.
28
+ #
29
+ attr_reader :connection_options
9
30
 
10
31
  def initialize select_statement, options = { file: 'app/db.yml' }
11
32
  @select_statement = select_statement
@@ -13,17 +34,10 @@ module Sources
13
34
  @options = options
14
35
  end
15
36
 
16
- # Get a configured Database backend.
17
- #
18
- # Options:
19
- # Either
20
- # * file => 'some/filename.yml' # With an active record configuration.
21
- # Or
22
- # * The configuration as a hash.
23
- #
24
- # TODO Do not use ActiveRecord directly.
25
- #
26
- def create_database_adapter
37
+ # Creates a database adapter for use with this source.
38
+ def create_database_adapter # :nodoc:
39
+ # TODO Do not use ActiveRecord directly.
40
+ #
27
41
  adapter_class = Class.new ActiveRecord::Base
28
42
  adapter_class.abstract_class = true
29
43
  adapter_class
@@ -37,7 +51,7 @@ module Sources
37
51
  # Or
38
52
  # * The configuration as a hash.
39
53
  #
40
- def configure options
54
+ def configure options # :nodoc:
41
55
  @connection_options = if filename = options[:file]
42
56
  File.open(File.join(PICKY_ROOT, filename)) { |file| YAML::load(file) }
43
57
  else
@@ -48,13 +62,17 @@ module Sources
48
62
 
49
63
  # Connect the backend.
50
64
  #
65
+ # Will raise unless connection options have been given.
66
+ #
51
67
  def connect_backend
52
68
  configure @options
53
69
  raise "Database backend not configured" unless connection_options
54
70
  database.establish_connection connection_options
55
71
  end
56
72
 
57
- # Take the snapshot.
73
+ # Take a snapshot of the data.
74
+ #
75
+ # Uses CREATE TABLE AS with the given SELECT statement to create a snapshot of the data.
58
76
  #
59
77
  def take_snapshot type
60
78
  connect_backend
@@ -75,23 +93,19 @@ module Sources
75
93
 
76
94
  # Counts all the entries that are used for the index.
77
95
  #
78
- def count type
96
+ def count type # :nodoc:
79
97
  connect_backend
80
98
 
81
99
  database.connection.select_value("SELECT COUNT(id) FROM #{snapshot_table_name(type)}").to_i
82
100
  end
83
101
 
84
- # Ok here?
85
102
  #
86
- def snapshot_table_name type
103
+ #
104
+ def snapshot_table_name type # :nodoc:
87
105
  "#{type.name}_type_index"
88
106
  end
89
107
 
90
- # Harvests the data to index, chunked.
91
- #
92
- # Subclasses should override harvest_statement to define how their data is found.
93
- # Example:
94
- # "SELECT indexed_id, value FROM bla_table st WHERE kind = 'bla'"
108
+ # Harvests the data to index in chunks.
95
109
  #
96
110
  def harvest type, category
97
111
  connect_backend
@@ -105,9 +119,9 @@ module Sources
105
119
  end
106
120
  end
107
121
 
108
- # Gets database from the backend.
122
+ # Gets the data from the backend.
109
123
  #
110
- def get_data type, category, offset
124
+ def get_data type, category, offset # :nodoc:
111
125
  database.connection.execute harvest_statement_with_offset(type, category, offset)
112
126
  end
113
127
 
@@ -115,7 +129,7 @@ module Sources
115
129
  #
116
130
  # TODO Use the adapter for this.
117
131
  #
118
- def harvest_statement_with_offset type, category, offset
132
+ def harvest_statement_with_offset type, category, offset # :nodoc:
119
133
  statement = harvest_statement type, category
120
134
 
121
135
  statement += statement.include?('WHERE') ? ' AND' : ' WHERE'
@@ -123,15 +137,17 @@ module Sources
123
137
  "#{statement} st.id > #{offset} LIMIT #{chunksize}"
124
138
  end
125
139
 
126
- # Base harvest statement for dbs.
140
+ # The harvest statement used to pull data from the snapshot table.
127
141
  #
128
- def harvest_statement type, category
142
+ def harvest_statement type, category # :nodoc:
129
143
  "SELECT indexed_id, #{category.from} FROM #{snapshot_table_name(type)} st"
130
144
  end
131
145
 
132
- # Override in subclasses.
146
+ # The amount of records that are loaded each chunk.
133
147
  #
134
- def chunksize
148
+ def chunksize # :nodoc:
149
+ # TODO Make parametrizable.
150
+ #
135
151
  25_000
136
152
  end
137
153
 
@@ -1,5 +1,15 @@
1
1
  module Sources
2
2
 
3
+ # Describes a Delicious (http://deli.cio.us) source.
4
+ #
5
+ # This source has a fixed set of categories:
6
+ # * title
7
+ # * tags
8
+ # * url
9
+ #
10
+ # Examples:
11
+ # Sources::CSV.new('usrnam', 'paswrd')
12
+ #
3
13
  class Delicious < Base
4
14
 
5
15
  def initialize username, password
@@ -7,7 +17,7 @@ module Sources
7
17
  @username = username
8
18
  @password = password
9
19
  end
10
- def check_gem
20
+ def check_gem # :nodoc:
11
21
  require 'www/delicious'
12
22
  rescue LoadError
13
23
  puts "Delicious gem missing!\nTo use the delicious source, you need to:\n 1. Add the following line to Gemfile:\n gem 'www-delicious'\n 2. Then, run:\n bundle update\n"
@@ -28,7 +38,7 @@ module Sources
28
38
 
29
39
  #
30
40
  #
31
- def get_data
41
+ def get_data # :nodoc:
32
42
  @generated_id ||= 0
33
43
  @posts ||= WWW::Delicious.new(@username, @password).posts_recent(count: 100)
34
44
  @posts.each do |post|
@@ -1,6 +1,8 @@
1
1
  module Sources
2
2
 
3
- module Wrappers
3
+ # TODO Document.
4
+ #
5
+ module Wrappers # :nodoc:all
4
6
 
5
7
  class Base
6
8
 
@@ -1,4 +1,4 @@
1
- module Tokenizers
1
+ module Tokenizers # :nodoc:all
2
2
 
3
3
  # Defines tokenizing processes used both in indexing and querying.
4
4
  #
@@ -2,7 +2,7 @@ source :gemcutter
2
2
 
3
3
  # Gems required by Picky.
4
4
  #
5
- gem 'picky', '~> 0.12.0'
5
+ gem 'picky', '~> 1.0.0'
6
6
  gem 'rake'
7
7
  gem 'bundler'
8
8
  gem 'rack', '~> 1.2.1'
@@ -2,5 +2,4 @@ All your application definitions go here.
2
2
 
3
3
  /app needs a few files defined by picky:
4
4
  * application.rb - how to handle indexing, queries, routing.
5
- * application.ru - rackup.
6
5
  * logging.rb
@@ -2,32 +2,44 @@ require 'spec_helper'
2
2
 
3
3
  describe Calculations::Location do
4
4
 
5
- context 'without margin' do
5
+ context 'with precision 1' do
6
6
  before(:each) do
7
- @calculation = Calculations::Location.new 3, 1
8
- @calculation.minimum = 5
7
+ @calculation = Calculations::Location.new 1.5, 1
8
+ @calculation.minimum = 42.7
9
9
  end
10
- describe 'reposition' do
11
- it 'calculates correctly' do
12
- @calculation.recalculate(13).should == 5
10
+ describe 'recalculate' do
11
+ it 'sets the minimum close value to the minimum minus user grid' do
12
+ @calculation.recalculate(41.2).should == 1
13
13
  end
14
- it 'calculates correctly' do
15
- @calculation.recalculate(5).should == 1
14
+ it 'sets the minimum value to 1 plus precision' do
15
+ @calculation.recalculate(42.7).should == 2
16
+ end
17
+ it 'sets the minimum value plus 2/3 of the grid size to 2 plus 1 grid length' do
18
+ @calculation.recalculate(43.7).should == 3
19
+ end
20
+ it 'sets the minimum value plus 20/3 of the grid size to 2 plus 10 grid length' do
21
+ @calculation.recalculate(52.7).should == 12
16
22
  end
17
23
  end
18
24
  end
19
25
 
20
- context 'without margin' do
26
+ context 'with precision 3' do
21
27
  before(:each) do
22
- @calculation = Calculations::Location.new 3, 3
23
- @calculation.minimum = 5
28
+ @calculation = Calculations::Location.new 1.5, 3
29
+ @calculation.minimum = 42.7
24
30
  end
25
- describe 'reposition' do
26
- it 'calculates correctly' do
27
- @calculation.recalculate(13).should == 12
31
+ describe 'recalculate' do
32
+ it 'sets the minimum close value to the minimum minus user grid' do
33
+ @calculation.recalculate(41.2).should == 1
34
+ end
35
+ it 'sets the minimum value to 1 plus precision' do
36
+ @calculation.recalculate(42.7).should == 4
37
+ end
38
+ it 'sets the minimum value plus 2/3 of the grid size plus 1 plus precision plus 1 grid length' do
39
+ @calculation.recalculate(43.7).should == 6
28
40
  end
29
- it 'calculates correctly' do
30
- @calculation.recalculate(5).should == 3
41
+ it 'sets the minimum value plus 20/3 of the grid size to 2 plus 10 grid length' do
42
+ @calculation.recalculate(52.7).should == 27
31
43
  end
32
44
  end
33
45
  end
@@ -0,0 +1,64 @@
1
+ # encoding: utf-8
2
+ #
3
+ require 'spec_helper'
4
+
5
+ describe IndexAPI do
6
+
7
+ context 'initializer' do
8
+ it 'works' do
9
+ lambda { IndexAPI.new :some_index_name, :some_source }.should_not raise_error
10
+ end
11
+ it 'registers with the indexes' do
12
+ @api = IndexAPI.allocate
13
+
14
+ Indexes.should_receive(:register).once.with @api
15
+
16
+ @api.send :initialize, :some_index_name, :some_source
17
+ end
18
+ end
19
+
20
+ context 'unit' do
21
+ before(:each) do
22
+ @api = IndexAPI.new :some_index_name, :some_source
23
+ end
24
+
25
+ describe 'define_category' do
26
+ context 'with block' do
27
+ it 'returns itself' do
28
+ @api.define_category(:some_name){ |indexing, indexed| }.should == @api
29
+ end
30
+ it 'takes a string' do
31
+ lambda { @api.define_category('some_name'){ |indexing, indexed| } }.should_not raise_error
32
+ end
33
+ it 'yields both the indexing category and the indexed category' do
34
+ @api.define_category(:some_name) do |indexing, indexed|
35
+ indexing.should be_kind_of(Indexing::Category)
36
+ indexed.should be_kind_of(Indexed::Category)
37
+ end
38
+ end
39
+ it 'yields the indexing category which has the given name' do
40
+ @api.define_category(:some_name) do |indexing, indexed|
41
+ indexing.name.should == :some_name
42
+ end
43
+ end
44
+ it 'yields the indexed category which has the given name' do
45
+ @api.define_category(:some_name) do |indexing, indexed|
46
+ indexed.name.should == :some_name
47
+ end
48
+ end
49
+ end
50
+ context 'without block' do
51
+ it 'works' do
52
+ lambda { @api.define_category(:some_name) }.should_not raise_error
53
+ end
54
+ it 'takes a string' do
55
+ lambda { @api.define_category('some_name').should == @api }.should_not raise_error
56
+ end
57
+ it 'returns itself' do
58
+ @api.define_category(:some_name).should == @api
59
+ end
60
+ end
61
+ end
62
+ end
63
+
64
+ end
@@ -7,8 +7,8 @@ describe Indexed::Index do
7
7
  @categories = stub :categories
8
8
 
9
9
  @index = Indexed::Index.new :some_name
10
- @index.add_category :some_category_name1
11
- @index.add_category :some_category_name2
10
+ @index.define_category :some_category_name1
11
+ @index.define_category :some_category_name2
12
12
 
13
13
  @index.stub! :categories => @categories
14
14
  end
@@ -16,7 +16,7 @@ describe Indexed::Wrappers::ExactFirst do
16
16
  #
17
17
  # it "wraps each category" do
18
18
  # type = Index::Type.new :type_name
19
- # type.add_category :some_category
19
+ # type.define_category :some_category
20
20
  #
21
21
  # Index::Wrappers::ExactFirst.wrap type
22
22
  #
@@ -24,7 +24,7 @@ describe Indexed::Wrappers::ExactFirst do
24
24
  # end
25
25
  it "returns the type" do
26
26
  type = Indexed::Index.new :type_name
27
- type.add_category :some_category
27
+ type.define_category :some_category
28
28
 
29
29
  Indexed::Wrappers::ExactFirst.wrap(type).should == type
30
30
  end
@@ -9,8 +9,8 @@ describe Indexing::Index do
9
9
  @categories = stub :categories
10
10
 
11
11
  @index = Indexing::Index.new :some_name, @source
12
- @index.add_category :some_category_name1
13
- @index.add_category :some_category_name2
12
+ @index.define_category :some_category_name1
13
+ @index.define_category :some_category_name2
14
14
 
15
15
  @index.stub! :categories => @categories
16
16
  end
@@ -7,21 +7,33 @@ describe Rack::Harakiri do
7
7
  end
8
8
  context "defaults" do
9
9
  before(:each) do
10
- @harakiri = Rack::Harakiri.new @app
10
+ @ronin = Rack::Harakiri.new @app
11
11
  end
12
- it "should quit after an amount of requests" do
13
- @harakiri.quit_after_requests.should == 50
12
+ it "should quit after a default amount of requests" do
13
+ @ronin.instance_variable_get(:@quit_after_requests).should == 50
14
+ end
15
+ describe 'harakiri?' do
16
+ it "should be true after 50 harakiri calls" do
17
+ 50.times { @ronin.harakiri }
18
+
19
+ @ronin.harakiri?.should == true
20
+ end
21
+ it "should not be true after just 49 harakiri calls" do
22
+ 49.times { @ronin.harakiri }
23
+
24
+ @ronin.harakiri?.should == false
25
+ end
14
26
  end
15
27
  describe "harakiri" do
16
28
  it "should kill the process after 50 harakiri calls" do
17
29
  Process.should_receive(:kill).once
18
30
 
19
- 50.times { @harakiri.harakiri }
31
+ 50.times { @ronin.harakiri }
20
32
  end
21
33
  it "should not kill the process after 49 harakiri calls" do
22
34
  Process.should_receive(:kill).never
23
35
 
24
- 49.times { @harakiri.harakiri }
36
+ 49.times { @ronin.harakiri }
25
37
  end
26
38
  end
27
39
  describe "call" do
@@ -30,27 +42,27 @@ describe Rack::Harakiri do
30
42
  @app.stub! :harakiri
31
43
  end
32
44
  it "calls harakiri" do
33
- @harakiri.should_receive(:harakiri).once.with
45
+ @ronin.should_receive(:harakiri).once.with
34
46
 
35
- @harakiri.call :env
47
+ @ronin.call :env
36
48
  end
37
49
  it "calls the app" do
38
50
  @app.should_receive(:call).once.with :env
39
51
 
40
- @harakiri.call :env
52
+ @ronin.call :env
41
53
  end
42
54
  end
43
55
  end
44
56
  context "with harakiri set" do
45
57
  before(:each) do
46
58
  Rack::Harakiri.after = 100
47
- @harakiri = Rack::Harakiri.new @app
59
+ @ronin = Rack::Harakiri.new @app
48
60
  end
49
61
  after(:each) do
50
62
  Rack::Harakiri.after = nil
51
63
  end
52
64
  it "should quit after an amount of requests" do
53
- @harakiri.quit_after_requests.should == 100
65
+ @ronin.instance_variable_get(:@quit_after_requests).should == 100
54
66
  end
55
67
  end
56
68
 
metadata CHANGED
@@ -3,10 +3,10 @@ name: picky
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease: false
5
5
  segments:
6
+ - 1
6
7
  - 0
7
- - 12
8
- - 3
9
- version: 0.12.3
8
+ - 0
9
+ version: 1.0.0
10
10
  platform: ruby
11
11
  authors:
12
12
  - Florian Hanke
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2010-11-28 00:00:00 +01:00
17
+ date: 2010-12-09 00:00:00 +01:00
18
18
  default_executable: picky
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
@@ -103,6 +103,7 @@ files:
103
103
  - lib/picky/indexing/indexes.rb
104
104
  - lib/picky/loader.rb
105
105
  - lib/picky/loggers/search.rb
106
+ - lib/picky/performant.rb
106
107
  - lib/picky/query/allocation.rb
107
108
  - lib/picky/query/allocations.rb
108
109
  - lib/picky/query/base.rb
@@ -189,6 +190,7 @@ files:
189
190
  - spec/lib/index/file/marshal_spec.rb
190
191
  - spec/lib/index/file/text_spec.rb
191
192
  - spec/lib/index/files_spec.rb
193
+ - spec/lib/index_api_spec.rb
192
194
  - spec/lib/indexed/bundle_spec.rb
193
195
  - spec/lib/indexed/categories_spec.rb
194
196
  - spec/lib/indexed/category_spec.rb
@@ -294,6 +296,7 @@ test_files:
294
296
  - spec/lib/index/file/marshal_spec.rb
295
297
  - spec/lib/index/file/text_spec.rb
296
298
  - spec/lib/index/files_spec.rb
299
+ - spec/lib/index_api_spec.rb
297
300
  - spec/lib/indexed/bundle_spec.rb
298
301
  - spec/lib/indexed/categories_spec.rb
299
302
  - spec/lib/indexed/category_spec.rb