logworm_amqp 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +52 -0
- data/Manifest +14 -0
- data/README.md +3 -0
- data/Rakefile +10 -0
- data/lib/base/config.rb +49 -0
- data/lib/base/db.rb +145 -0
- data/lib/base/query_builder.rb +115 -0
- data/lib/logworm_amqp.rb +3 -0
- data/logworm_amqp.gemspec +60 -0
- data/spec/base_spec.rb +143 -0
- data/spec/builder_spec.rb +26 -0
- data/spec/config_spec.rb +36 -0
- data/spec/spec.opts +4 -0
- data/spec/spec_helper.rb +8 -0
- data/tests/builder_test.rb +52 -0
- metadata +231 -0
data/CHANGELOG
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
v0.8.0
|
2
|
+
Dropped http for amqp for high scalability and reduce latency
|
3
|
+
|
4
|
+
v0.7.7
|
5
|
+
DB.config now receives an optional app parameter, for the cases where you want to call command-line tools from a directory other than the app's directory... or when you have more than one Heroku remote/app from the same directory
|
6
|
+
|
7
|
+
v0.7.6
|
8
|
+
QueryBuilder now allows Time objects as arguments for timeframe
|
9
|
+
|
10
|
+
v0.7.5
|
11
|
+
:force_ts now default in QueryBuilder
|
12
|
+
|
13
|
+
v0.7.4
|
14
|
+
Eliminated default host for database. Must be specified in the configuration environment.
|
15
|
+
|
16
|
+
v0.7.3
|
17
|
+
Cleaner way to compose the URL for the DB (now logworm://key:secret@host/token/token_secret/)
|
18
|
+
|
19
|
+
v0.7.1
|
20
|
+
Added DB.from_config_or_die, to throw exception if DB cannot be initialized
|
21
|
+
|
22
|
+
v0.7.0
|
23
|
+
Changes to run as a Heroku add-on
|
24
|
+
Configuration now stored as a URL
|
25
|
+
Now requires Heroku gem
|
26
|
+
|
27
|
+
v0.6.1
|
28
|
+
fixed query builder
|
29
|
+
|
30
|
+
v0.6.0
|
31
|
+
added query builder
|
32
|
+
|
33
|
+
v0.5.1
|
34
|
+
removed memcache dependency, added memcache-client
|
35
|
+
|
36
|
+
v0.5.0
|
37
|
+
removed utils, moved them to separate gem
|
38
|
+
|
39
|
+
v0.4.1
|
40
|
+
added start and end options to lw-compute and lw-tail
|
41
|
+
|
42
|
+
v0.4.0
|
43
|
+
added lw-compute tool, to run aggregate queries
|
44
|
+
|
45
|
+
v0.3.0
|
46
|
+
added lw-heroku tool, to push configuration variables to heroku
|
47
|
+
|
48
|
+
v0.2.0
|
49
|
+
removed app/ libraries. Added tail as a utility, lw-tail as a command
|
50
|
+
|
51
|
+
v0.1.0
|
52
|
+
initial version.
|
data/Manifest
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
CHANGELOG
|
2
|
+
Manifest
|
3
|
+
README.md
|
4
|
+
Rakefile
|
5
|
+
lib/base/config.rb
|
6
|
+
lib/base/db.rb
|
7
|
+
lib/base/query_builder.rb
|
8
|
+
lib/logworm_amqp.rb
|
9
|
+
spec/base_spec.rb
|
10
|
+
spec/builder_spec.rb
|
11
|
+
spec/config_spec.rb
|
12
|
+
spec/spec.opts
|
13
|
+
spec/spec_helper.rb
|
14
|
+
tests/builder_test.rb
|
data/README.md
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
require 'echoe'
|
2
|
+
Echoe.new('logworm_amqp', '0.8.0') do |p|
|
3
|
+
p.description = "logworm logging tool"
|
4
|
+
p.url = "http://www.logworm.com"
|
5
|
+
p.author = "Pomelo, LLC"
|
6
|
+
p.email = "schapira@pomelollc.com"
|
7
|
+
p.ignore_pattern = ["tmp/*", "script/*"]
|
8
|
+
p.development_dependencies = ["memcache-client", "hpricot", "oauth", "heroku", "minion"]
|
9
|
+
p.runtime_dependencies = ["memcache-client", "hpricot", "oauth", "heroku", "minion"]
|
10
|
+
end
|
data/lib/base/config.rb
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
|
3
|
+
module Logworm
|
4
|
+
class ConfigFileNotFound < Exception ; end
|
5
|
+
|
6
|
+
class Config
|
7
|
+
|
8
|
+
include ::Singleton
|
9
|
+
|
10
|
+
FILENAME = ".logworm"
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
reset
|
14
|
+
end
|
15
|
+
|
16
|
+
def reset
|
17
|
+
@file_found = false
|
18
|
+
@url = nil
|
19
|
+
end
|
20
|
+
|
21
|
+
def read
|
22
|
+
begin
|
23
|
+
f = File.new("./" + FILENAME, 'r')
|
24
|
+
@url = f.readline.strip
|
25
|
+
@file_found = true
|
26
|
+
rescue Errno::ENOENT => e
|
27
|
+
end
|
28
|
+
self
|
29
|
+
end
|
30
|
+
|
31
|
+
def url
|
32
|
+
@url
|
33
|
+
end
|
34
|
+
|
35
|
+
def file_found?
|
36
|
+
@file_found and (!@url.nil? and @url != "")
|
37
|
+
end
|
38
|
+
|
39
|
+
def save(url)
|
40
|
+
File.open("./" + FILENAME, 'w') do |f|
|
41
|
+
f.puts url
|
42
|
+
end rescue Exception
|
43
|
+
%x[echo #{FILENAME} >> .gitignore]
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
|
data/lib/base/db.rb
ADDED
@@ -0,0 +1,145 @@
|
|
1
|
+
require 'oauth'
|
2
|
+
require 'json'
|
3
|
+
require 'minion'
|
4
|
+
require 'hmac-sha1'
|
5
|
+
require 'cgi'
|
6
|
+
require 'base64'
|
7
|
+
|
8
|
+
module Logworm
|
9
|
+
class ForbiddenAccessException < Exception ; end
|
10
|
+
class DatabaseException < Exception ; end
|
11
|
+
class InvalidQueryException < Exception ; end
|
12
|
+
|
13
|
+
class DB
|
14
|
+
|
15
|
+
URL_FORMAT = /logworm:\/\/([^:]+):([^@]+)@([^\/]+)\/([^\/]+)\/([^\/]+)\//
|
16
|
+
# URI: logworm://<consumer_key>:<consumer_secret>@db.logworm.com/<access_token>/<access_token_secret>/
|
17
|
+
|
18
|
+
attr_reader :host, :consumer_key, :consumer_secret, :token, :token_secret
|
19
|
+
|
20
|
+
def initialize(url)
|
21
|
+
match = DB.parse_url(url)
|
22
|
+
raise ForbiddenAccessException.new("Incorrect URL Format #{url}") unless match and match.size == 6
|
23
|
+
@consumer_key, @consumer_secret, @host, @token, @token_secret = match[1..5]
|
24
|
+
@connection = OAuth::AccessToken.new(OAuth::Consumer.new(@consumer_key, @consumer_secret), @token, @token_secret)
|
25
|
+
Minion.amqp_url = "amqp://logworm-producer:4tX.z.rC@pomelo-1.dotcloud.com/"
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.with_tokens(token, token_secret)
|
29
|
+
consumer_key = ENV["#{ENV['APP_ID']}_APPS_KEY"]
|
30
|
+
consumer_secret = ENV["#{ENV['APP_ID']}_APPS_SECRET"]
|
31
|
+
host = ENV["#{ENV['APP_ID']}_DB_HOST"]
|
32
|
+
DB.new(DB.make_url(host, consumer_key, consumer_secret, token, token_secret))
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.from_config(app = nil)
|
36
|
+
# Try with URL from the environment. This will certainly be the case when running on Heroku, in production.
|
37
|
+
return DB.new(ENV['LOGWORM_URL']) if ENV['LOGWORM_URL'] and DB.parse_url(ENV['LOGWORM_URL'])
|
38
|
+
|
39
|
+
# If no env. found, try with configuration file, unless app specified
|
40
|
+
config = Logworm::Config.instance
|
41
|
+
config.read
|
42
|
+
unless app
|
43
|
+
return DB.new(config.url) if config.file_found? and DB.parse_url(config.url)
|
44
|
+
end
|
45
|
+
|
46
|
+
# Try with Heroku configuration otherwise
|
47
|
+
cmd = "heroku config --long #{app ? " --app #{app}" : ""}"
|
48
|
+
config_vars = %x[#{cmd}] || ""
|
49
|
+
m = config_vars.match(Regexp.new("LOGWORM_URL\\s+=>\\s+([^\\n]+)"))
|
50
|
+
if m and DB.parse_url(m[1])
|
51
|
+
config.save(m[1]) unless (config.file_found? and app) # Do not overwrite if --app is provided
|
52
|
+
return DB.new(m[1])
|
53
|
+
end
|
54
|
+
|
55
|
+
nil
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.from_config_or_die(app = nil)
|
59
|
+
db = self.from_config(app)
|
60
|
+
raise "The application is not properly configured. Either use 'heroku addon:add' to add logworm to your app, or save your project's credentials into the .logworm file" unless db
|
61
|
+
db
|
62
|
+
end
|
63
|
+
|
64
|
+
def self.make_url(host, consumer_key, consumer_secret, token, token_secret)
|
65
|
+
"logworm://#{consumer_key}:#{consumer_secret}@#{host}/#{token}/#{token_secret}/"
|
66
|
+
end
|
67
|
+
|
68
|
+
def url()
|
69
|
+
DB.make_url(@host, @consumer_key, @consumer_secret, @token, @token_secret)
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.example_url
|
73
|
+
self.make_url("db.logworm.com", "Ub5sOstT9w", "GZi0HciTVcoFHEoIZ7", "OzO71hEvWYDmncbf3C", "J7wq4X06MihhZgqDeB")
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
def tables()
|
78
|
+
db_call(:get, "#{host_with_protocol}/") || []
|
79
|
+
end
|
80
|
+
|
81
|
+
def query(table, cond)
|
82
|
+
db_call(:post, "#{host_with_protocol}/queries", {:table => table, :query => cond})
|
83
|
+
end
|
84
|
+
|
85
|
+
|
86
|
+
|
87
|
+
def results(uri)
|
88
|
+
res = db_call(:get, uri)
|
89
|
+
raise InvalidQueryException.new("#{res['error']}") if res['error']
|
90
|
+
res["results"] = JSON.parse(res["results"])
|
91
|
+
res
|
92
|
+
end
|
93
|
+
|
94
|
+
def signature(base_string, consumer_secret)
|
95
|
+
secret="#{escape(consumer_secret)}&"
|
96
|
+
Base64.encode64(HMAC::SHA1.digest(secret,base_string)).chomp.gsub(/\n/,'')
|
97
|
+
end
|
98
|
+
|
99
|
+
def escape(value)
|
100
|
+
CGI.escape(value.to_s).gsub("%7E", '~').gsub("+", "%20")
|
101
|
+
end
|
102
|
+
|
103
|
+
|
104
|
+
def batch_log(entries)
|
105
|
+
#db_call(:post, "#{host_with_protocol}/log", {:entries => $lr_queue.to_json})
|
106
|
+
content = $lr_queue.to_json
|
107
|
+
sig= signature(content, @token_secret )
|
108
|
+
start = Time.now
|
109
|
+
Minion.enqueue("lw.logging", {:entries => content, :consumer_key => @token, :signature => sig })
|
110
|
+
$stderr.puts "#{Time.now - start}"
|
111
|
+
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
|
116
|
+
|
117
|
+
private
|
118
|
+
def db_call(method, uri, params = {})
|
119
|
+
begin
|
120
|
+
res = @connection.send(method, uri, params)
|
121
|
+
rescue SocketError
|
122
|
+
raise DatabaseException
|
123
|
+
end
|
124
|
+
raise InvalidQueryException.new("#{res.body}") if res.code.to_i == 400
|
125
|
+
raise ForbiddenAccessException if res.code.to_i == 403
|
126
|
+
raise DatabaseException if res.code.to_i == 404
|
127
|
+
raise DatabaseException.new("Server returned: #{res.body}") if res.code.to_i == 500
|
128
|
+
begin
|
129
|
+
JSON.parse(res.body)
|
130
|
+
rescue Exception => e
|
131
|
+
raise DatabaseException.new("Database reponse cannot be parsed: #{e}")
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def self.parse_url(url)
|
136
|
+
url.match(URL_FORMAT)
|
137
|
+
end
|
138
|
+
|
139
|
+
def host_with_protocol
|
140
|
+
"http://#{@host}"
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
end
|
145
|
+
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
###
|
5
|
+
# Receives a hash with options, and provides a to_json method that returns the query ready to be sent to the logworm server
|
6
|
+
# Switches (all optional)
|
7
|
+
# :fields => String with a comma-separated list of fields (quoted or not), or Array of Strings
|
8
|
+
# :force_ts => Boolean, specifies whether _ts should be added to the list of fields
|
9
|
+
# :aggregate_function => String
|
10
|
+
# :aggregate_argument => String
|
11
|
+
# :aggregate_group => String with a comma-separated list of fields (quoted or not), or Array of Strings
|
12
|
+
# :conditions => String with comma-separated conditions (in MongoDB syntax), or Array of Strings
|
13
|
+
# :start => String or Integer (for year)
|
14
|
+
# :end => String or Integer (for year)
|
15
|
+
# :limit => String or Integer
|
16
|
+
###
|
17
|
+
module Logworm
|
18
|
+
class QueryBuilder
|
19
|
+
|
20
|
+
attr_accessor :fields, :groups, :aggregate, :conditions, :tf, :limit
|
21
|
+
|
22
|
+
def initialize(options = {})
|
23
|
+
@options = options
|
24
|
+
@options.merge(:force_ts => true) unless @options.include? :force_ts
|
25
|
+
@query = build()
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_json
|
29
|
+
@query
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
def build()
|
34
|
+
query_opts = []
|
35
|
+
|
36
|
+
###
|
37
|
+
# Fields : Array, or Comma-separated string
|
38
|
+
###
|
39
|
+
@fields = to_array(@options[:fields])
|
40
|
+
query_opts << '"fields":' + (@options[:force_ts] ? @fields + ["_ts"] : @fields).to_json if @fields.size > 0
|
41
|
+
|
42
|
+
###
|
43
|
+
# Aggregate
|
44
|
+
# aggregate_function: String
|
45
|
+
# aggregate_argument: String (or empty)
|
46
|
+
# aggregate_group: String or Array
|
47
|
+
###
|
48
|
+
@groups = to_array(@options[:aggregate_group])
|
49
|
+
@aggregate = {}
|
50
|
+
@aggregate[:function] = @options[:aggregate_function] if is_set?(@options[:aggregate_function])
|
51
|
+
@aggregate[:argument] = @options[:aggregate_argument] if is_set?(@options[:aggregate_argument])
|
52
|
+
@aggregate[:group_by] = groups[0] if groups.size == 1
|
53
|
+
@aggregate[:group_by] = groups if groups.size > 1
|
54
|
+
query_opts << '"aggregate":' + @aggregate.to_json if @aggregate.keys.size > 0
|
55
|
+
|
56
|
+
if @fields.size > 0 and @aggregate.keys.size > 0
|
57
|
+
raise Logworm::InvalidQueryException.new("Queries cannot contain both fields and aggregates")
|
58
|
+
end
|
59
|
+
|
60
|
+
###
|
61
|
+
# Conditions : Array, or Comma-separated string
|
62
|
+
# ['"a":10' , '"b":20']
|
63
|
+
# "a:10", "b":20
|
64
|
+
###
|
65
|
+
@conditions = to_string(@options[:conditions])
|
66
|
+
query_opts << '"conditions":{' + conditions + "}" if conditions.size > 0
|
67
|
+
|
68
|
+
###
|
69
|
+
# Timeframe: String
|
70
|
+
###
|
71
|
+
@tf = {}
|
72
|
+
@tf[:start] = unquote(@options[:start]).to_s if is_set?(@options[:start]) or is_set?(@options[:start], Integer, 0)
|
73
|
+
@tf[:start] = unquote(@options[:start].strftime("%Y-%m-%dT%H:%M:%SZ")).to_s if is_set?(@options[:start], Time)
|
74
|
+
@tf[:end] = unquote(@options[:end]).to_s if is_set?(@options[:end]) or is_set?(@options[:end], Integer, 0)
|
75
|
+
@tf[:end] = unquote(@options[:end].strftime("%Y-%m-%dT%H:%M:%SZ")).to_s if is_set?(@options[:end], Time)
|
76
|
+
query_opts << '"timeframe":' + @tf.to_json if @tf.keys.size > 0
|
77
|
+
|
78
|
+
###
|
79
|
+
# Limit
|
80
|
+
# String or Integer
|
81
|
+
###
|
82
|
+
if (is_set?(@options[:limit], Integer, 200) or is_set?(@options[:limit], String, ""))
|
83
|
+
@limit = @options[:limit].to_s
|
84
|
+
query_opts << '"limit":' + @limit
|
85
|
+
end
|
86
|
+
|
87
|
+
# And the string
|
88
|
+
"{#{query_opts.join(", ")}}"
|
89
|
+
end
|
90
|
+
|
91
|
+
def to_array(arg)
|
92
|
+
return [] if arg.nil?
|
93
|
+
return arg if arg.is_a? Array
|
94
|
+
return arg.split(",").map {|e| unquote(e.strip)} if arg.is_a? String and arg.split != ""
|
95
|
+
[]
|
96
|
+
end
|
97
|
+
|
98
|
+
def to_string(arg)
|
99
|
+
return "" if arg.nil?
|
100
|
+
return arg.split(",").map {|e| e.strip}.join(",") if arg.is_a? String
|
101
|
+
return arg.join(",") if arg.is_a? Array and arg.size > 0
|
102
|
+
""
|
103
|
+
end
|
104
|
+
|
105
|
+
def unquote(str)
|
106
|
+
return str unless str.is_a? String
|
107
|
+
str.gsub(/^"/, '').gsub(/"$/,'')
|
108
|
+
end
|
109
|
+
|
110
|
+
def is_set?(elt, klass = String, empty_val = "")
|
111
|
+
elt and elt.is_a?(klass) and elt != empty_val
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
end
|
data/lib/logworm_amqp.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{logworm_amqp}
|
5
|
+
s.version = "0.8.0"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["Pomelo, LLC"]
|
9
|
+
s.date = %q{2010-07-31}
|
10
|
+
s.description = %q{logworm logging tool}
|
11
|
+
s.email = %q{schapira@pomelollc.com}
|
12
|
+
s.extra_rdoc_files = ["CHANGELOG", "README.md", "lib/base/config.rb", "lib/base/db.rb", "lib/base/query_builder.rb", "lib/logworm_amqp.rb"]
|
13
|
+
s.files = ["CHANGELOG", "Manifest", "README.md", "Rakefile", "lib/base/config.rb", "lib/base/db.rb", "lib/base/query_builder.rb", "lib/logworm_amqp.rb", "spec/base_spec.rb", "spec/builder_spec.rb", "spec/config_spec.rb", "spec/spec.opts", "spec/spec_helper.rb", "tests/builder_test.rb", "logworm_amqp.gemspec"]
|
14
|
+
s.homepage = %q{http://www.logworm.com}
|
15
|
+
s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Logworm_amqp", "--main", "README.md"]
|
16
|
+
s.require_paths = ["lib"]
|
17
|
+
s.rubyforge_project = %q{logworm_amqp}
|
18
|
+
s.rubygems_version = %q{1.3.7}
|
19
|
+
s.summary = %q{logworm logging tool}
|
20
|
+
|
21
|
+
if s.respond_to? :specification_version then
|
22
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
23
|
+
s.specification_version = 3
|
24
|
+
|
25
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
26
|
+
s.add_runtime_dependency(%q<memcache-client>, [">= 0"])
|
27
|
+
s.add_runtime_dependency(%q<hpricot>, [">= 0"])
|
28
|
+
s.add_runtime_dependency(%q<oauth>, [">= 0"])
|
29
|
+
s.add_runtime_dependency(%q<heroku>, [">= 0"])
|
30
|
+
s.add_runtime_dependency(%q<minion>, [">= 0"])
|
31
|
+
s.add_development_dependency(%q<memcache-client>, [">= 0"])
|
32
|
+
s.add_development_dependency(%q<hpricot>, [">= 0"])
|
33
|
+
s.add_development_dependency(%q<oauth>, [">= 0"])
|
34
|
+
s.add_development_dependency(%q<heroku>, [">= 0"])
|
35
|
+
s.add_development_dependency(%q<minion>, [">= 0"])
|
36
|
+
else
|
37
|
+
s.add_dependency(%q<memcache-client>, [">= 0"])
|
38
|
+
s.add_dependency(%q<hpricot>, [">= 0"])
|
39
|
+
s.add_dependency(%q<oauth>, [">= 0"])
|
40
|
+
s.add_dependency(%q<heroku>, [">= 0"])
|
41
|
+
s.add_dependency(%q<minion>, [">= 0"])
|
42
|
+
s.add_dependency(%q<memcache-client>, [">= 0"])
|
43
|
+
s.add_dependency(%q<hpricot>, [">= 0"])
|
44
|
+
s.add_dependency(%q<oauth>, [">= 0"])
|
45
|
+
s.add_dependency(%q<heroku>, [">= 0"])
|
46
|
+
s.add_dependency(%q<minion>, [">= 0"])
|
47
|
+
end
|
48
|
+
else
|
49
|
+
s.add_dependency(%q<memcache-client>, [">= 0"])
|
50
|
+
s.add_dependency(%q<hpricot>, [">= 0"])
|
51
|
+
s.add_dependency(%q<oauth>, [">= 0"])
|
52
|
+
s.add_dependency(%q<heroku>, [">= 0"])
|
53
|
+
s.add_dependency(%q<minion>, [">= 0"])
|
54
|
+
s.add_dependency(%q<memcache-client>, [">= 0"])
|
55
|
+
s.add_dependency(%q<hpricot>, [">= 0"])
|
56
|
+
s.add_dependency(%q<oauth>, [">= 0"])
|
57
|
+
s.add_dependency(%q<heroku>, [">= 0"])
|
58
|
+
s.add_dependency(%q<minion>, [">= 0"])
|
59
|
+
end
|
60
|
+
end
|
data/spec/base_spec.rb
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'webmock'
|
3
|
+
|
4
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
5
|
+
|
6
|
+
$: << File.dirname(__FILE__) + '/../lib'
|
7
|
+
require 'logworm.rb'
|
8
|
+
|
9
|
+
describe Logworm::DB, " initialization" do
|
10
|
+
before do
|
11
|
+
File.delete(".logworm") if File.exist?(".logworm")
|
12
|
+
Logworm::Config.instance.reset
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should only accept proper URLs" do
|
16
|
+
lambda {Logworm::DB.new('')}.should raise_exception(Logworm::ForbiddenAccessException)
|
17
|
+
lambda {Logworm::DB.new('http://www.test.com')}.should raise_exception(Logworm::ForbiddenAccessException)
|
18
|
+
lambda {Logworm::DB.new('logworm://a:b@xxx/c/d')}.should raise_exception(Logworm::ForbiddenAccessException)
|
19
|
+
lambda {Logworm::DB.new('logworm://a:b@/c/d/')}.should raise_exception(Logworm::ForbiddenAccessException)
|
20
|
+
lambda {Logworm::DB.new('logworm://a:b@sda//d/')}.should raise_exception(Logworm::ForbiddenAccessException)
|
21
|
+
lambda {Logworm::DB.new('logworm://:b@sda//d/')}.should raise_exception(Logworm::ForbiddenAccessException)
|
22
|
+
lambda {Logworm::DB.new('logworm://a:b@xxx/c/d/')}.should_not raise_exception(Logworm::ForbiddenAccessException)
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should be able to parse a proper logworm URL" do
|
26
|
+
db = Logworm::DB.new('logworm://a:b@localhost:9401/c/d/')
|
27
|
+
db.host.should == "localhost:9401"
|
28
|
+
db.consumer_key.should == "a"
|
29
|
+
db.consumer_secret.should == "b"
|
30
|
+
db.token.should == "c"
|
31
|
+
db.token_secret.should == "d"
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should be able to read its configuration from a file" do
|
35
|
+
File.open(".logworm", "w") do |f|
|
36
|
+
f.puts 'logworm://a:b@localhost:9401/c/d/'
|
37
|
+
end
|
38
|
+
db = Logworm::DB.from_config
|
39
|
+
db.host.should == "localhost:9401"
|
40
|
+
db.consumer_key.should == "a"
|
41
|
+
db.consumer_secret.should == "b"
|
42
|
+
db.token.should == "c"
|
43
|
+
db.token_secret.should == "d"
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should fail if no logworm file (and no current Heroku application)" do
|
47
|
+
db = Logworm::DB.from_config
|
48
|
+
db.should == nil
|
49
|
+
end
|
50
|
+
|
51
|
+
# Note that this will fail unless it's run from the command line!
|
52
|
+
it "should not be nil if we pass a proper app parameter" do
|
53
|
+
db = Logworm::DB.from_config("lw-client")
|
54
|
+
db.should_not == nil
|
55
|
+
db.host.should == "db.logworm.com"
|
56
|
+
end
|
57
|
+
|
58
|
+
# Note that this will fail unless it's run from the command line!
|
59
|
+
it "should not use a config file if app is passed" do
|
60
|
+
File.open(".logworm", "w") do |f|
|
61
|
+
f.puts 'logworm://a:b@xxx:9401/c/d/'
|
62
|
+
end
|
63
|
+
db = Logworm::DB.from_config("lw-client")
|
64
|
+
db.host.should == "db.logworm.com" # The one from the app, not the config file
|
65
|
+
end
|
66
|
+
|
67
|
+
# Note that this will fail unless it's run from the command line!
|
68
|
+
it "should not overwrite a config file if app is passed" do
|
69
|
+
File.open(".logworm", "w") do |f|
|
70
|
+
f.puts 'logworm://a:b@xxx:9401/c/d/'
|
71
|
+
end
|
72
|
+
|
73
|
+
db = Logworm::DB.from_config("lw-client")
|
74
|
+
Logworm::Config.instance.reset
|
75
|
+
Logworm::Config.instance.read
|
76
|
+
Logworm::Config.instance.url.should == 'logworm://a:b@xxx:9401/c/d/'
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
|
81
|
+
describe Logworm::DB, " functioning" do
|
82
|
+
|
83
|
+
host = "http://localhost:9401"
|
84
|
+
|
85
|
+
before(:all) do
|
86
|
+
@db = Logworm::DB.new('logworm://a:b@localhost:9401/c/d/')
|
87
|
+
end
|
88
|
+
|
89
|
+
it "should offer a call to get the list of tables --> /" do
|
90
|
+
@db.should_receive(:db_call).with(:get, "#{host}/")
|
91
|
+
@db.tables
|
92
|
+
end
|
93
|
+
|
94
|
+
it "should just parse and return the results of the call to get tables" do
|
95
|
+
return_body = [
|
96
|
+
{"tablename" => "table1", "url" => "/table1", "last_write" => "2010-03-20 18:10:22", "rows" => 50},
|
97
|
+
{"tablename" => "table2", "url" => "/table1", "last_write" => "2010-03-20 18:10:22", "rows" => 50}]
|
98
|
+
stub_request(:get, "#{host}/").to_return(:body => return_body.to_json)
|
99
|
+
@db.tables.should == return_body
|
100
|
+
end
|
101
|
+
|
102
|
+
it "should support a call to start a query --> POST /queries" do
|
103
|
+
@db.should_receive(:db_call).with(:post, "#{host}/queries", {:table => "tbl1", :query => "a good query"})
|
104
|
+
@db.query("tbl1", "a good query")
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should just parse and return the results of the call to query" do
|
108
|
+
return_body = {"id" => 10, "query" => "q", "self_uri" => "/queries/10", "results_uri" => "/queries/10/results"}
|
109
|
+
stub_request(:post, "#{host}/queries").with(:body => "query=q&table=table1").to_return(:body => return_body.to_json)
|
110
|
+
@db.query("table1", "q").should == return_body
|
111
|
+
end
|
112
|
+
|
113
|
+
it "should support a call to retrieve the results of a query --> GET /queries/10/results" do
|
114
|
+
@db.should_receive(:db_call).with(:get, "#{host}/queries/10/results")
|
115
|
+
@db.results("#{host}/queries/10/results") rescue Exception # Returns an error when trying to parse results
|
116
|
+
end
|
117
|
+
|
118
|
+
it "should just parse and return the results of the call to retrieve results, but also add results field" do
|
119
|
+
results = [{"a" => 10, "b" => "2"}, {"a" => "x"}]
|
120
|
+
return_body = {"id" => 10, "execution_time" => "5",
|
121
|
+
"query_url" => "#{host}/queries/10", "results_url" => "#{host}/queries/10/results",
|
122
|
+
"results" => results.to_json}
|
123
|
+
stub_request(:get, "#{host}/queries/10/results").to_return(:body => return_body.to_json)
|
124
|
+
@db.results("#{host}/queries/10/results").should == return_body.merge("results" => results)
|
125
|
+
end
|
126
|
+
|
127
|
+
it "should raise ForbiddenAccessException if 403" do
|
128
|
+
stub_request(:get, "#{host}/").to_return(:status => 403)
|
129
|
+
lambda {@db.tables}.should raise_exception(Logworm::ForbiddenAccessException)
|
130
|
+
end
|
131
|
+
|
132
|
+
it "should raise InvalidQueryException if query is not valid" do
|
133
|
+
stub_request(:post, "#{host}/queries").to_return(:status => 400, :body => "Query error")
|
134
|
+
lambda {@db.query("tbl1", "bad query")}.should raise_exception(Logworm::InvalidQueryException)
|
135
|
+
end
|
136
|
+
|
137
|
+
it "should raise DatabaseException if response from server is not JSON" do
|
138
|
+
stub_request(:get, "#{host}/").to_return(:body => "blah")
|
139
|
+
lambda {@db.tables}.should raise_exception(Logworm::DatabaseException)
|
140
|
+
end
|
141
|
+
|
142
|
+
|
143
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
4
|
+
|
5
|
+
$: << File.dirname(__FILE__) + '/../lib'
|
6
|
+
require 'logworm.rb'
|
7
|
+
|
8
|
+
describe Logworm::QueryBuilder, " timeframes" do
|
9
|
+
|
10
|
+
it " should accept Strings as time" do
|
11
|
+
Logworm::QueryBuilder.new(:start => "2010-01-01").to_json.should == '{"timeframe":{"start":"2010-01-01"}}'
|
12
|
+
Logworm::QueryBuilder.new(:end => "2010-01-01").to_json.should == '{"timeframe":{"end":"2010-01-01"}}'
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should accept an Integer as time, to mean the year" do
|
16
|
+
Logworm::QueryBuilder.new(:start => 2010).to_json.should == '{"timeframe":{"start":"2010"}}'
|
17
|
+
Logworm::QueryBuilder.new(:end => 2010).to_json.should == '{"timeframe":{"end":"2010"}}'
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should accept a Time object" do
|
21
|
+
ts = Time.now
|
22
|
+
Logworm::QueryBuilder.new(:start => ts).to_json.should == '{"timeframe":{"start":"' + ts.strftime("%Y-%m-%dT%H:%M:%SZ") + '"}}'
|
23
|
+
Logworm::QueryBuilder.new(:end => ts).to_json.should == '{"timeframe":{"end":"' + ts.strftime("%Y-%m-%dT%H:%M:%SZ") + '"}}'
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
data/spec/config_spec.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'webmock'
|
3
|
+
|
4
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
5
|
+
|
6
|
+
$: << File.dirname(__FILE__) + '/../lib'
|
7
|
+
require 'logworm.rb'
|
8
|
+
|
9
|
+
describe Logworm::Config, " initialization" do
|
10
|
+
|
11
|
+
before do
|
12
|
+
%x[rm .logworm]
|
13
|
+
%x[mv .gitignore .gitignore_old]
|
14
|
+
end
|
15
|
+
|
16
|
+
after do
|
17
|
+
%x[mv .gitignore_old .gitignore]
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should create a new .logworm file on save" do
|
21
|
+
url = "xxx"
|
22
|
+
File.should_not exist(".logworm")
|
23
|
+
Logworm::Config.instance.save(url)
|
24
|
+
File.should exist(".logworm")
|
25
|
+
Logworm::Config.instance.read.should be_file_found
|
26
|
+
Logworm::Config.instance.url.should == url
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should add .logworm to .gitignore" do
|
30
|
+
File.should_not exist(".gitignore")
|
31
|
+
Logworm::Config.instance.save("xxx")
|
32
|
+
File.should exist(".gitignore")
|
33
|
+
File.open('.gitignore').readline.strip.should == ".logworm"
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'test/unit'
|
3
|
+
require '../lib/base/query_builder'
|
4
|
+
|
5
|
+
class BuilderTest < Test::Unit::TestCase
|
6
|
+
|
7
|
+
def setup
|
8
|
+
end
|
9
|
+
|
10
|
+
def teardown
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_empty
|
14
|
+
assert_equal "{}", Logworm::QueryBuilder.new({}).to_json
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_fields
|
18
|
+
assert_equal '{"fields":["a","b"]}', Logworm::QueryBuilder.new(:fields => 'a, b').to_json
|
19
|
+
assert_equal '{"fields":["a","b"]}', Logworm::QueryBuilder.new(:fields => '"a", "b"').to_json
|
20
|
+
assert_equal '{"fields":["a","b"]}', Logworm::QueryBuilder.new(:fields => ["a", "b"]).to_json
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_aggregate
|
24
|
+
q = {:aggregate_function => "count"}
|
25
|
+
assert_equal '{"aggregate":{"function":"count"}}', Logworm::QueryBuilder.new(q).to_json
|
26
|
+
q = {:aggregate_function => "a", :aggregate_argument => "b"}
|
27
|
+
assert_equal '{"aggregate":{"argument":"b","function":"a"}}', Logworm::QueryBuilder.new(q).to_json
|
28
|
+
q = {:aggregate_function => "a", :aggregate_argument => "b", :aggregate_group => "a,b,c"}
|
29
|
+
assert_equal '{"aggregate":{"argument":"b","group_by":["a","b","c"],"function":"a"}}', Logworm::QueryBuilder.new(q).to_json
|
30
|
+
q = {:aggregate_function => "a", :aggregate_argument => "b", :aggregate_group => ["a","b","c"]}
|
31
|
+
assert_equal '{"aggregate":{"argument":"b","group_by":["a","b","c"],"function":"a"}}', Logworm::QueryBuilder.new(q).to_json
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_conditions
|
35
|
+
assert_equal '{"conditions":{"a":10,"b":"c"}}', Logworm::QueryBuilder.new(:conditions => '"a":10, "b":"c"').to_json
|
36
|
+
assert_equal '{"conditions":{"a":10,"b":"c"}}', Logworm::QueryBuilder.new(:conditions => ['"a":10', '"b":"c"']).to_json
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_times
|
40
|
+
assert_equal '{}', Logworm::QueryBuilder.new(:blah => "2009").to_json
|
41
|
+
assert_equal '{"timeframe":{"start":"2009"}}', Logworm::QueryBuilder.new(:start => "2009").to_json
|
42
|
+
assert_equal '{"timeframe":{"end":"2009"}}', Logworm::QueryBuilder.new(:end => "2009").to_json
|
43
|
+
assert_equal '{"timeframe":{"start":"2009","end":"2010"}}', Logworm::QueryBuilder.new(:start => "2009", :end => "2010").to_json
|
44
|
+
assert_equal '{"timeframe":{"start":"2009","end":"2010"}}', Logworm::QueryBuilder.new(:start => 2009, :end => 2010).to_json
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_limit
|
48
|
+
assert_equal '{"limit":10}', Logworm::QueryBuilder.new(:limit => 10).to_json
|
49
|
+
assert_equal '{}', Logworm::QueryBuilder.new(:limit => 200).to_json
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
metadata
ADDED
@@ -0,0 +1,231 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logworm_amqp
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 63
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 8
|
9
|
+
- 0
|
10
|
+
version: 0.8.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Pomelo, LLC
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2010-07-31 00:00:00 -04:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: memcache-client
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 3
|
30
|
+
segments:
|
31
|
+
- 0
|
32
|
+
version: "0"
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: hpricot
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
hash: 3
|
44
|
+
segments:
|
45
|
+
- 0
|
46
|
+
version: "0"
|
47
|
+
type: :runtime
|
48
|
+
version_requirements: *id002
|
49
|
+
- !ruby/object:Gem::Dependency
|
50
|
+
name: oauth
|
51
|
+
prerelease: false
|
52
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
53
|
+
none: false
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
hash: 3
|
58
|
+
segments:
|
59
|
+
- 0
|
60
|
+
version: "0"
|
61
|
+
type: :runtime
|
62
|
+
version_requirements: *id003
|
63
|
+
- !ruby/object:Gem::Dependency
|
64
|
+
name: heroku
|
65
|
+
prerelease: false
|
66
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
67
|
+
none: false
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
hash: 3
|
72
|
+
segments:
|
73
|
+
- 0
|
74
|
+
version: "0"
|
75
|
+
type: :runtime
|
76
|
+
version_requirements: *id004
|
77
|
+
- !ruby/object:Gem::Dependency
|
78
|
+
name: minion
|
79
|
+
prerelease: false
|
80
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ">="
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
hash: 3
|
86
|
+
segments:
|
87
|
+
- 0
|
88
|
+
version: "0"
|
89
|
+
type: :runtime
|
90
|
+
version_requirements: *id005
|
91
|
+
- !ruby/object:Gem::Dependency
|
92
|
+
name: memcache-client
|
93
|
+
prerelease: false
|
94
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
95
|
+
none: false
|
96
|
+
requirements:
|
97
|
+
- - ">="
|
98
|
+
- !ruby/object:Gem::Version
|
99
|
+
hash: 3
|
100
|
+
segments:
|
101
|
+
- 0
|
102
|
+
version: "0"
|
103
|
+
type: :development
|
104
|
+
version_requirements: *id006
|
105
|
+
- !ruby/object:Gem::Dependency
|
106
|
+
name: hpricot
|
107
|
+
prerelease: false
|
108
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
109
|
+
none: false
|
110
|
+
requirements:
|
111
|
+
- - ">="
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
hash: 3
|
114
|
+
segments:
|
115
|
+
- 0
|
116
|
+
version: "0"
|
117
|
+
type: :development
|
118
|
+
version_requirements: *id007
|
119
|
+
- !ruby/object:Gem::Dependency
|
120
|
+
name: oauth
|
121
|
+
prerelease: false
|
122
|
+
requirement: &id008 !ruby/object:Gem::Requirement
|
123
|
+
none: false
|
124
|
+
requirements:
|
125
|
+
- - ">="
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
hash: 3
|
128
|
+
segments:
|
129
|
+
- 0
|
130
|
+
version: "0"
|
131
|
+
type: :development
|
132
|
+
version_requirements: *id008
|
133
|
+
- !ruby/object:Gem::Dependency
|
134
|
+
name: heroku
|
135
|
+
prerelease: false
|
136
|
+
requirement: &id009 !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ">="
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
hash: 3
|
142
|
+
segments:
|
143
|
+
- 0
|
144
|
+
version: "0"
|
145
|
+
type: :development
|
146
|
+
version_requirements: *id009
|
147
|
+
- !ruby/object:Gem::Dependency
|
148
|
+
name: minion
|
149
|
+
prerelease: false
|
150
|
+
requirement: &id010 !ruby/object:Gem::Requirement
|
151
|
+
none: false
|
152
|
+
requirements:
|
153
|
+
- - ">="
|
154
|
+
- !ruby/object:Gem::Version
|
155
|
+
hash: 3
|
156
|
+
segments:
|
157
|
+
- 0
|
158
|
+
version: "0"
|
159
|
+
type: :development
|
160
|
+
version_requirements: *id010
|
161
|
+
description: logworm logging tool
|
162
|
+
email: schapira@pomelollc.com
|
163
|
+
executables: []
|
164
|
+
|
165
|
+
extensions: []
|
166
|
+
|
167
|
+
extra_rdoc_files:
|
168
|
+
- CHANGELOG
|
169
|
+
- README.md
|
170
|
+
- lib/base/config.rb
|
171
|
+
- lib/base/db.rb
|
172
|
+
- lib/base/query_builder.rb
|
173
|
+
- lib/logworm_amqp.rb
|
174
|
+
files:
|
175
|
+
- CHANGELOG
|
176
|
+
- Manifest
|
177
|
+
- README.md
|
178
|
+
- Rakefile
|
179
|
+
- lib/base/config.rb
|
180
|
+
- lib/base/db.rb
|
181
|
+
- lib/base/query_builder.rb
|
182
|
+
- lib/logworm_amqp.rb
|
183
|
+
- spec/base_spec.rb
|
184
|
+
- spec/builder_spec.rb
|
185
|
+
- spec/config_spec.rb
|
186
|
+
- spec/spec.opts
|
187
|
+
- spec/spec_helper.rb
|
188
|
+
- tests/builder_test.rb
|
189
|
+
- logworm_amqp.gemspec
|
190
|
+
has_rdoc: true
|
191
|
+
homepage: http://www.logworm.com
|
192
|
+
licenses: []
|
193
|
+
|
194
|
+
post_install_message:
|
195
|
+
rdoc_options:
|
196
|
+
- --line-numbers
|
197
|
+
- --inline-source
|
198
|
+
- --title
|
199
|
+
- Logworm_amqp
|
200
|
+
- --main
|
201
|
+
- README.md
|
202
|
+
require_paths:
|
203
|
+
- lib
|
204
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
205
|
+
none: false
|
206
|
+
requirements:
|
207
|
+
- - ">="
|
208
|
+
- !ruby/object:Gem::Version
|
209
|
+
hash: 3
|
210
|
+
segments:
|
211
|
+
- 0
|
212
|
+
version: "0"
|
213
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
214
|
+
none: false
|
215
|
+
requirements:
|
216
|
+
- - ">="
|
217
|
+
- !ruby/object:Gem::Version
|
218
|
+
hash: 11
|
219
|
+
segments:
|
220
|
+
- 1
|
221
|
+
- 2
|
222
|
+
version: "1.2"
|
223
|
+
requirements: []
|
224
|
+
|
225
|
+
rubyforge_project: logworm_amqp
|
226
|
+
rubygems_version: 1.3.7
|
227
|
+
signing_key:
|
228
|
+
specification_version: 3
|
229
|
+
summary: logworm logging tool
|
230
|
+
test_files: []
|
231
|
+
|