jgre-couchrest 0.12.6
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +176 -0
- data/README.md +68 -0
- data/Rakefile +66 -0
- data/THANKS.md +18 -0
- data/examples/model/example.rb +138 -0
- data/examples/word_count/markov +38 -0
- data/examples/word_count/views/books/chunked-map.js +3 -0
- data/examples/word_count/views/books/united-map.js +1 -0
- data/examples/word_count/views/markov/chain-map.js +6 -0
- data/examples/word_count/views/markov/chain-reduce.js +7 -0
- data/examples/word_count/views/word_count/count-map.js +6 -0
- data/examples/word_count/views/word_count/count-reduce.js +3 -0
- data/examples/word_count/word_count.rb +46 -0
- data/examples/word_count/word_count_query.rb +40 -0
- data/examples/word_count/word_count_views.rb +26 -0
- data/lib/couchrest.rb +139 -0
- data/lib/couchrest/commands/generate.rb +71 -0
- data/lib/couchrest/commands/push.rb +103 -0
- data/lib/couchrest/core/database.rb +241 -0
- data/lib/couchrest/core/design.rb +89 -0
- data/lib/couchrest/core/document.rb +94 -0
- data/lib/couchrest/core/model.rb +613 -0
- data/lib/couchrest/core/server.rb +51 -0
- data/lib/couchrest/core/view.rb +4 -0
- data/lib/couchrest/helper/pager.rb +103 -0
- data/lib/couchrest/helper/streamer.rb +44 -0
- data/lib/couchrest/monkeypatches.rb +38 -0
- data/spec/couchrest/core/couchrest_spec.rb +201 -0
- data/spec/couchrest/core/database_spec.rb +629 -0
- data/spec/couchrest/core/design_spec.rb +131 -0
- data/spec/couchrest/core/document_spec.rb +213 -0
- data/spec/couchrest/core/model_spec.rb +859 -0
- data/spec/couchrest/helpers/pager_spec.rb +122 -0
- data/spec/couchrest/helpers/streamer_spec.rb +23 -0
- data/spec/fixtures/attachments/README +3 -0
- data/spec/fixtures/attachments/couchdb.png +0 -0
- data/spec/fixtures/attachments/test.html +11 -0
- data/spec/fixtures/views/lib.js +3 -0
- data/spec/fixtures/views/test_view/lib.js +3 -0
- data/spec/fixtures/views/test_view/only-map.js +4 -0
- data/spec/fixtures/views/test_view/test-map.js +3 -0
- data/spec/fixtures/views/test_view/test-reduce.js +3 -0
- data/spec/spec.opts +6 -0
- data/spec/spec_helper.rb +20 -0
- data/utils/remap.rb +27 -0
- data/utils/subset.rb +30 -0
- metadata +143 -0
@@ -0,0 +1,38 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require File.expand_path(File.dirname(__FILE__)) + '/../../couchrest'
|
4
|
+
|
5
|
+
cr = CouchRest.new("http://127.0.0.1:5984")
|
6
|
+
@db = cr.database('word-count-example')
|
7
|
+
@word_memoizer = {}
|
8
|
+
|
9
|
+
def probable_follower_for(word)
|
10
|
+
@word_memoizer[word] ||= @db.view('markov/chain-reduce', :startkey => [word,nil], :endkey => [word,{}],:group_level => 2)
|
11
|
+
|
12
|
+
# puts
|
13
|
+
# puts "search #{word} #{wprobs[word]['rows'].length}"
|
14
|
+
# @word_memoizer[word]['rows'].sort_by{|r|r['value']}.each{|r|puts [r['value'],r['key']].inspect}
|
15
|
+
|
16
|
+
rows = @word_memoizer[word]['rows'].select{|r|(r['key'][1]!='')}.sort_by{|r|r['value']}
|
17
|
+
row = rows[(-1*[rows.length,5].min)..-1].sort_by{rand}[0]
|
18
|
+
row ? row['key'][1] : nil
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
word = ARGV[0]
|
23
|
+
words = [word]
|
24
|
+
|
25
|
+
while word
|
26
|
+
$stdout.print ' ' if words.length > 1
|
27
|
+
$stdout.print word
|
28
|
+
$stdout.flush
|
29
|
+
word = probable_follower_for(word)
|
30
|
+
words << word
|
31
|
+
end
|
32
|
+
|
33
|
+
$stdout.print '.'
|
34
|
+
$stdout.flush
|
35
|
+
puts
|
36
|
+
|
37
|
+
# `say #{words.join(' ')}`
|
38
|
+
|
@@ -0,0 +1 @@
|
|
1
|
+
function(doc){if(doc.text && doc.text.match(/united/)) emit([doc.title, doc.chunk],null)}
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'couchrest'
|
3
|
+
|
4
|
+
couch = CouchRest.new("http://127.0.0.1:5984")
|
5
|
+
db = couch.database('word-count-example')
|
6
|
+
db.delete! rescue nil
|
7
|
+
db = couch.create_db('word-count-example')
|
8
|
+
|
9
|
+
books = {
|
10
|
+
'outline-of-science.txt' => 'http://www.gutenberg.org/files/20417/20417.txt',
|
11
|
+
'ulysses.txt' => 'http://www.gutenberg.org/dirs/etext03/ulyss12.txt',
|
12
|
+
'america.txt' => 'http://www.gutenberg.org/files/16960/16960.txt',
|
13
|
+
'da-vinci.txt' => 'http://www.gutenberg.org/dirs/etext04/7ldv110.txt'
|
14
|
+
}
|
15
|
+
|
16
|
+
books.each do |file, url|
|
17
|
+
pathfile = File.join(File.dirname(__FILE__),file)
|
18
|
+
`curl #{url} > #{pathfile}` unless File.exists?(pathfile)
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
books.keys.each do |book|
|
23
|
+
title = book.split('.')[0]
|
24
|
+
puts title
|
25
|
+
File.open(File.join(File.dirname(__FILE__),book),'r') do |file|
|
26
|
+
lines = []
|
27
|
+
chunk = 0
|
28
|
+
while line = file.gets
|
29
|
+
lines << line
|
30
|
+
if lines.length > 10
|
31
|
+
db.save({
|
32
|
+
:title => title,
|
33
|
+
:chunk => chunk,
|
34
|
+
:text => lines.join('')
|
35
|
+
})
|
36
|
+
chunk += 1
|
37
|
+
puts chunk
|
38
|
+
lines = []
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# puts "The books have been stored in your CouchDB. To initiate the MapReduce process, visit http://127.0.0.1:5984/_utils/ in your browser and click 'word-count-example', then select view 'words' or 'count'. The process could take about 15 minutes on an average MacBook."
|
45
|
+
|
46
|
+
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'couchrest'
|
3
|
+
|
4
|
+
couch = CouchRest.new("http://127.0.0.1:5984")
|
5
|
+
db = couch.database('word-count-example')
|
6
|
+
|
7
|
+
puts "Now that we've parsed all those books into CouchDB, the queries we can run are incredibly flexible."
|
8
|
+
puts "\nThe simplest query we can run is the total word count for all words in all documents:"
|
9
|
+
puts "this will take a few minutes the first time. if it times out, just rerun this script in a few few minutes."
|
10
|
+
puts db.view('word_count/words').inspect
|
11
|
+
|
12
|
+
puts "\nWe can also narrow the query down to just one word, across all documents. Here is the count for 'flight' in all three books:"
|
13
|
+
|
14
|
+
word = 'flight'
|
15
|
+
params = {
|
16
|
+
:startkey => [word],
|
17
|
+
:endkey => [word,{}]
|
18
|
+
}
|
19
|
+
|
20
|
+
puts db.view('word_count/words',params).inspect
|
21
|
+
|
22
|
+
puts "\nWe scope the query using startkey and endkey params to take advantage of CouchDB's collation ordering. Here are the params for the last query:"
|
23
|
+
puts params.inspect
|
24
|
+
|
25
|
+
puts "\nWe can also count words on a per-title basis."
|
26
|
+
|
27
|
+
title = 'da-vinci'
|
28
|
+
params = {
|
29
|
+
:key => [word, title]
|
30
|
+
}
|
31
|
+
|
32
|
+
puts db.view('word_count/words',params).inspect
|
33
|
+
|
34
|
+
|
35
|
+
puts "\nHere are the params for 'flight' in the da-vinci book:"
|
36
|
+
puts params.inspect
|
37
|
+
puts
|
38
|
+
puts 'The url looks like this:'
|
39
|
+
puts 'http://127.0.0.1:5984/word-count-example/_view/word_count/count?key=["flight","da-vinci"]'
|
40
|
+
puts "\nTry dropping that in your browser..."
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'couchrest'
|
3
|
+
|
4
|
+
couch = CouchRest.new("http://127.0.0.1:5984")
|
5
|
+
db = couch.database('word-count-example')
|
6
|
+
|
7
|
+
word_count = {
|
8
|
+
:map => 'function(doc){
|
9
|
+
var words = doc.text.split(/\W/);
|
10
|
+
words.forEach(function(word){
|
11
|
+
if (word.length > 0) emit([word,doc.title],1);
|
12
|
+
});
|
13
|
+
}',
|
14
|
+
:reduce => 'function(key,combine){
|
15
|
+
return sum(combine);
|
16
|
+
}'
|
17
|
+
}
|
18
|
+
|
19
|
+
db.delete db.get("_design/word_count") rescue nil
|
20
|
+
|
21
|
+
db.save({
|
22
|
+
"_id" => "_design/word_count",
|
23
|
+
:views => {
|
24
|
+
:words => word_count
|
25
|
+
}
|
26
|
+
})
|
data/lib/couchrest.rb
ADDED
@@ -0,0 +1,139 @@
|
|
1
|
+
# Copyright 2008 J. Chris Anderson
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "rubygems"
|
16
|
+
require 'json'
|
17
|
+
require 'rest_client'
|
18
|
+
|
19
|
+
$:.unshift File.dirname(__FILE__) unless
|
20
|
+
$:.include?(File.dirname(__FILE__)) ||
|
21
|
+
$:.include?(File.expand_path(File.dirname(__FILE__)))
|
22
|
+
|
23
|
+
|
24
|
+
require 'couchrest/monkeypatches'
|
25
|
+
|
26
|
+
# = CouchDB, close to the metal
|
27
|
+
module CouchRest
|
28
|
+
VERSION = '0.12.6'
|
29
|
+
|
30
|
+
autoload :Server, 'couchrest/core/server'
|
31
|
+
autoload :Database, 'couchrest/core/database'
|
32
|
+
autoload :Document, 'couchrest/core/document'
|
33
|
+
autoload :Design, 'couchrest/core/design'
|
34
|
+
autoload :View, 'couchrest/core/view'
|
35
|
+
autoload :Model, 'couchrest/core/model'
|
36
|
+
autoload :Pager, 'couchrest/helper/pager'
|
37
|
+
autoload :FileManager, 'couchrest/helper/file_manager'
|
38
|
+
autoload :Streamer, 'couchrest/helper/streamer'
|
39
|
+
|
40
|
+
# The CouchRest module methods handle the basic JSON serialization
|
41
|
+
# and deserialization, as well as query parameters. The module also includes
|
42
|
+
# some helpers for tasks like instantiating a new Database or Server instance.
|
43
|
+
class << self
|
44
|
+
|
45
|
+
# todo, make this parse the url and instantiate a Server or Database instance
|
46
|
+
# depending on the specificity.
|
47
|
+
def new(*opts)
|
48
|
+
Server.new(*opts)
|
49
|
+
end
|
50
|
+
|
51
|
+
def parse url
|
52
|
+
case url
|
53
|
+
when /^http:\/\/(.*)\/(.*)\/(.*)/
|
54
|
+
host = $1
|
55
|
+
db = $2
|
56
|
+
docid = $3
|
57
|
+
when /^http:\/\/(.*)\/(.*)/
|
58
|
+
host = $1
|
59
|
+
db = $2
|
60
|
+
when /^http:\/\/(.*)/
|
61
|
+
host = $1
|
62
|
+
when /(.*)\/(.*)\/(.*)/
|
63
|
+
host = $1
|
64
|
+
db = $2
|
65
|
+
docid = $3
|
66
|
+
when /(.*)\/(.*)/
|
67
|
+
host = $1
|
68
|
+
db = $2
|
69
|
+
else
|
70
|
+
db = url
|
71
|
+
end
|
72
|
+
|
73
|
+
db = nil if db && db.empty?
|
74
|
+
|
75
|
+
{
|
76
|
+
:host => host || "127.0.0.1:5984",
|
77
|
+
:database => db,
|
78
|
+
:doc => docid
|
79
|
+
}
|
80
|
+
end
|
81
|
+
|
82
|
+
# set proxy for RestClient to use
|
83
|
+
def proxy url
|
84
|
+
RestClient.proxy = url
|
85
|
+
end
|
86
|
+
|
87
|
+
# ensure that a database exists
|
88
|
+
# creates it if it isn't already there
|
89
|
+
# returns it after it's been created
|
90
|
+
def database! url
|
91
|
+
parsed = parse url
|
92
|
+
cr = CouchRest.new(parsed[:host])
|
93
|
+
cr.database!(parsed[:database])
|
94
|
+
end
|
95
|
+
|
96
|
+
def database url
|
97
|
+
parsed = parse url
|
98
|
+
cr = CouchRest.new(parsed[:host])
|
99
|
+
cr.database(parsed[:database])
|
100
|
+
end
|
101
|
+
|
102
|
+
def put uri, doc = nil
|
103
|
+
payload = doc.to_json if doc
|
104
|
+
JSON.parse(RestClient.put(uri, payload))
|
105
|
+
end
|
106
|
+
|
107
|
+
def get uri
|
108
|
+
JSON.parse(RestClient.get(uri), :max_nesting => false)
|
109
|
+
end
|
110
|
+
|
111
|
+
def post uri, doc = nil
|
112
|
+
payload = doc.to_json if doc
|
113
|
+
JSON.parse(RestClient.post(uri, payload))
|
114
|
+
end
|
115
|
+
|
116
|
+
def delete uri
|
117
|
+
JSON.parse(RestClient.delete(uri))
|
118
|
+
end
|
119
|
+
|
120
|
+
def copy uri, destination
|
121
|
+
JSON.parse(RestClient.copy(uri, {'Destination' => destination}))
|
122
|
+
end
|
123
|
+
|
124
|
+
def move uri, destination
|
125
|
+
JSON.parse(RestClient.move(uri, {'Destination' => destination}))
|
126
|
+
end
|
127
|
+
|
128
|
+
def paramify_url url, params = {}
|
129
|
+
if params && !params.empty?
|
130
|
+
query = params.collect do |k,v|
|
131
|
+
v = v.to_json if %w{key startkey endkey}.include?(k.to_s)
|
132
|
+
"#{k}=#{CGI.escape(v.to_s)}"
|
133
|
+
end.join("&")
|
134
|
+
url = "#{url}?#{query}"
|
135
|
+
end
|
136
|
+
url
|
137
|
+
end
|
138
|
+
end # class << self
|
139
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
|
3
|
+
module CouchRest
|
4
|
+
module Commands
|
5
|
+
module Generate
|
6
|
+
|
7
|
+
def self.run(options)
|
8
|
+
directory = options[:directory]
|
9
|
+
design_names = options[:trailing_args]
|
10
|
+
|
11
|
+
FileUtils.mkdir_p(directory)
|
12
|
+
filename = File.join(directory, "lib.js")
|
13
|
+
self.write(filename, <<-FUNC)
|
14
|
+
// Put global functions here.
|
15
|
+
// Include in your views with
|
16
|
+
//
|
17
|
+
// //include-lib
|
18
|
+
FUNC
|
19
|
+
|
20
|
+
design_names.each do |design_name|
|
21
|
+
subdirectory = File.join(directory, design_name)
|
22
|
+
FileUtils.mkdir_p(subdirectory)
|
23
|
+
filename = File.join(subdirectory, "sample-map.js")
|
24
|
+
self.write(filename, <<-FUNC)
|
25
|
+
function(doc) {
|
26
|
+
// Keys is first letter of _id
|
27
|
+
emit(doc._id[0], doc);
|
28
|
+
}
|
29
|
+
FUNC
|
30
|
+
|
31
|
+
filename = File.join(subdirectory, "sample-reduce.js")
|
32
|
+
self.write(filename, <<-FUNC)
|
33
|
+
function(keys, values) {
|
34
|
+
// Count the number of keys starting with this letter
|
35
|
+
return values.length;
|
36
|
+
}
|
37
|
+
FUNC
|
38
|
+
|
39
|
+
filename = File.join(subdirectory, "lib.js")
|
40
|
+
self.write(filename, <<-FUNC)
|
41
|
+
// Put functions specific to '#{design_name}' here.
|
42
|
+
// Include in your views with
|
43
|
+
//
|
44
|
+
// //include-lib
|
45
|
+
FUNC
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.help
|
50
|
+
helpstring = <<-GEN
|
51
|
+
|
52
|
+
Usage: couchview generate directory design1 design2 design3 ...
|
53
|
+
|
54
|
+
Couchview will create directories and example views for the design documents you specify.
|
55
|
+
|
56
|
+
GEN
|
57
|
+
helpstring.gsub(/^ /, '')
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.write(filename, contents)
|
61
|
+
puts "Writing #{filename}"
|
62
|
+
File.open(filename, "w") do |f|
|
63
|
+
# Remove leading spaces
|
64
|
+
contents.gsub!(/^ ( )?/, '')
|
65
|
+
f.write contents
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
module CouchRest
|
2
|
+
|
3
|
+
module Commands
|
4
|
+
|
5
|
+
module Push
|
6
|
+
|
7
|
+
def self.run(options)
|
8
|
+
directory = options[:directory]
|
9
|
+
database = options[:trailing_args].first
|
10
|
+
|
11
|
+
fm = CouchRest::FileManager.new(database)
|
12
|
+
fm.loud = options[:loud]
|
13
|
+
|
14
|
+
if options[:loud]
|
15
|
+
puts "Pushing views from directory #{directory} to database #{fm.db}"
|
16
|
+
end
|
17
|
+
|
18
|
+
fm.push_views(directory)
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.help
|
22
|
+
helpstring = <<-GEN
|
23
|
+
|
24
|
+
== Pushing views with Couchview ==
|
25
|
+
|
26
|
+
Usage: couchview push directory dbname
|
27
|
+
|
28
|
+
Couchview expects a specific filesystem layout for your CouchDB views (see
|
29
|
+
example below). It also supports advanced features like inlining of library
|
30
|
+
code (so you can keep DRY) as well as avoiding unnecessary document
|
31
|
+
modification.
|
32
|
+
|
33
|
+
Couchview also solves a problem with CouchDB's view API, which only provides
|
34
|
+
access to the final reduce side of any views which have both a map and a
|
35
|
+
reduce function defined. The intermediate map results are often useful for
|
36
|
+
development and production. CouchDB is smart enough to reuse map indexes for
|
37
|
+
functions duplicated across views within the same design document.
|
38
|
+
|
39
|
+
For views with a reduce function defined, Couchview creates both a reduce view
|
40
|
+
and a map-only view, so that you can browse and query the map side as well as
|
41
|
+
the reduction, with no performance penalty.
|
42
|
+
|
43
|
+
== Example ==
|
44
|
+
|
45
|
+
couchview push foo-project/bar-views baz-database
|
46
|
+
|
47
|
+
This will push the views defined in foo-project/bar-views into a database
|
48
|
+
called baz-database. Couchview expects the views to be defined in files with
|
49
|
+
names like:
|
50
|
+
|
51
|
+
foo-project/bar-views/my-design/viewname-map.js
|
52
|
+
foo-project/bar-views/my-design/viewname-reduce.js
|
53
|
+
foo-project/bar-views/my-design/noreduce-map.js
|
54
|
+
|
55
|
+
Pushed to => http://127.0.0.1:5984/baz-database/_design/my-design
|
56
|
+
|
57
|
+
And the design document:
|
58
|
+
{
|
59
|
+
"views" : {
|
60
|
+
"viewname-map" : {
|
61
|
+
"map" : "### contents of view-name-map.js ###"
|
62
|
+
},
|
63
|
+
"viewname-reduce" : {
|
64
|
+
"map" : "### contents of view-name-map.js ###",
|
65
|
+
"reduce" : "### contents of view-name-reduce.js ###"
|
66
|
+
},
|
67
|
+
"noreduce-map" : {
|
68
|
+
"map" : "### contents of noreduce-map.js ###"
|
69
|
+
}
|
70
|
+
}
|
71
|
+
}
|
72
|
+
|
73
|
+
Couchview will create a design document for each subdirectory of the views
|
74
|
+
directory specified on the command line.
|
75
|
+
|
76
|
+
== Library Inlining ==
|
77
|
+
|
78
|
+
Couchview can optionally inline library code into your views so you only have
|
79
|
+
to maintain it in one place. It looks for any files named lib.* in your
|
80
|
+
design-doc directory (for doc specific libs) and in the parent views directory
|
81
|
+
(for project global libs). These libraries are only inserted into views which
|
82
|
+
include the text
|
83
|
+
|
84
|
+
// !include lib
|
85
|
+
|
86
|
+
or
|
87
|
+
|
88
|
+
# !include lib
|
89
|
+
|
90
|
+
Couchview is a result of scratching my own itch. I'd be happy to make it more
|
91
|
+
general, so please contact me at jchris@grabb.it if you'd like to see anything
|
92
|
+
added or changed.
|
93
|
+
|
94
|
+
GEN
|
95
|
+
helpstring.gsub(/^ /, '')
|
96
|
+
end
|
97
|
+
|
98
|
+
end
|
99
|
+
|
100
|
+
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|