oozou-fusion_tables 0.2.3.dev.20110408163600
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/CHANGELOG +6 -0
- data/LICENSE +20 -0
- data/README.textile +148 -0
- data/Rakefile +54 -0
- data/TODO +11 -0
- data/VERSION +1 -0
- data/examples/boris_bikes.rb +159 -0
- data/examples/compare_tweets.rb +143 -0
- data/examples/credentials.example.yml +4 -0
- data/fusion_tables.gemspec +78 -0
- data/lib/fusion_tables.rb +6 -0
- data/lib/fusion_tables/client/fusion_tables.rb +77 -0
- data/lib/fusion_tables/data/data.rb +72 -0
- data/lib/fusion_tables/data/table.rb +144 -0
- data/lib/fusion_tables/ext/fusion_tables.rb +96 -0
- data/pkg/fusion_tables-0.1.0.gem +0 -0
- data/pkg/fusion_tables-0.1.1.gem +0 -0
- data/pkg/fusion_tables-0.1.2.gem +0 -0
- data/pkg/fusion_tables-0.2.0.gem +0 -0
- data/pkg/fusion_tables-0.2.1.gem +0 -0
- data/pkg/fusion_tables-0.2.2.gem +0 -0
- data/test/README +3 -0
- data/test/helper.rb +34 -0
- data/test/test_client.rb +22 -0
- data/test/test_config.yml.sample +3 -0
- data/test/test_ext.rb +58 -0
- data/test/test_table.rb +76 -0
- metadata +110 -0
@@ -0,0 +1,78 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{fusion_tables}
|
8
|
+
s.version = "0.2.2"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Simon Tokumine", "Tom Verbeure"]
|
12
|
+
s.date = %q{2010-09-02}
|
13
|
+
s.description = %q{A simple Google Fusion Tables API wrapper. Supports bulk inserts and most API functions}
|
14
|
+
s.email = %q{simon@tinypla.net}
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE",
|
17
|
+
"README.textile",
|
18
|
+
"TODO"
|
19
|
+
]
|
20
|
+
s.files = [
|
21
|
+
".document",
|
22
|
+
".gitignore",
|
23
|
+
"LICENSE",
|
24
|
+
"README.textile",
|
25
|
+
"Rakefile",
|
26
|
+
"TODO",
|
27
|
+
"VERSION",
|
28
|
+
"examples/compare_tweets.rb",
|
29
|
+
"examples/credentials.example.yml",
|
30
|
+
"fusion_tables.gemspec",
|
31
|
+
"lib/fusion_tables.rb",
|
32
|
+
"lib/fusion_tables/client/fusion_tables.rb",
|
33
|
+
"lib/fusion_tables/data/data.rb",
|
34
|
+
"lib/fusion_tables/data/table.rb",
|
35
|
+
"lib/fusion_tables/ext/fusion_tables.rb",
|
36
|
+
"pkg/fusion_tables-0.1.0.gem",
|
37
|
+
"pkg/fusion_tables-0.1.1.gem",
|
38
|
+
"pkg/fusion_tables-0.1.2.gem",
|
39
|
+
"pkg/fusion_tables-0.2.0.gem",
|
40
|
+
"pkg/fusion_tables-0.2.1.gem",
|
41
|
+
"pkg/fusion_tables-0.2.2.gem",
|
42
|
+
"test/README",
|
43
|
+
"test/helper.rb",
|
44
|
+
"test/test_client.rb",
|
45
|
+
"test/test_config.yml.sample",
|
46
|
+
"test/test_ext.rb",
|
47
|
+
"test/test_table.rb"
|
48
|
+
]
|
49
|
+
s.homepage = %q{http://github.com/tokumine/fusion-tables}
|
50
|
+
s.rdoc_options = ["--charset=UTF-8"]
|
51
|
+
s.require_paths = ["lib"]
|
52
|
+
s.rubygems_version = %q{1.3.7}
|
53
|
+
s.summary = %q{Google Fusion Tables API wrapper}
|
54
|
+
s.test_files = [
|
55
|
+
"test/helper.rb",
|
56
|
+
"test/test_client.rb",
|
57
|
+
"test/test_ext.rb",
|
58
|
+
"test/test_table.rb",
|
59
|
+
"examples/compare_tweets.rb"
|
60
|
+
]
|
61
|
+
|
62
|
+
if s.respond_to? :specification_version then
|
63
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
64
|
+
s.specification_version = 3
|
65
|
+
|
66
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
67
|
+
s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
|
68
|
+
s.add_runtime_dependency(%q<gdata_19>, [">= 1.1.2"])
|
69
|
+
else
|
70
|
+
s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
|
71
|
+
s.add_dependency(%q<gdata_19>, [">= 1.1.2"])
|
72
|
+
end
|
73
|
+
else
|
74
|
+
s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
|
75
|
+
s.add_dependency(%q<gdata_19>, [">= 1.1.2"])
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# Copyright (C) 2010 Tom Verbeure, Simon Tokumine
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
module GData
|
16
|
+
module Client
|
17
|
+
class FusionTables < Base
|
18
|
+
|
19
|
+
SERVICE_URL = "http://tables.googlelabs.com/api/query"
|
20
|
+
DATATYPES = %w(number string location datetime)
|
21
|
+
|
22
|
+
def initialize(options = {})
|
23
|
+
options[:clientlogin_service] ||= 'fusiontables'
|
24
|
+
options[:headers] = { 'Content-Type' => 'application/x-www-form-urlencoded' }
|
25
|
+
super(options)
|
26
|
+
end
|
27
|
+
|
28
|
+
def sql_encode(sql)
|
29
|
+
"sql=" + CGI::escape(sql)
|
30
|
+
end
|
31
|
+
|
32
|
+
def sql_get(sql)
|
33
|
+
resp = self.get(SERVICE_URL + "?" + sql_encode(sql))
|
34
|
+
end
|
35
|
+
|
36
|
+
def sql_post(sql)
|
37
|
+
resp = self.post(SERVICE_URL, sql_encode(sql))
|
38
|
+
end
|
39
|
+
|
40
|
+
def sql_put(sql)
|
41
|
+
resp = self.put(SERVICE_URL, sql_encode(sql))
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
# Overrides auth_handler= so if the authentication changes,
|
46
|
+
# the session cookie is cleared.
|
47
|
+
def auth_handler=(handler)
|
48
|
+
@session_cookie = nil
|
49
|
+
return super(handler)
|
50
|
+
end
|
51
|
+
|
52
|
+
# Overrides make_request to handle 500 redirects with a session cookie.
|
53
|
+
def make_request(method, url, body = '', retries = 10)
|
54
|
+
begin
|
55
|
+
response = super(method, url, body)
|
56
|
+
rescue GData::Client::ServerError => e
|
57
|
+
if e.response.status_code == 500 and retries > 0
|
58
|
+
sleep_time = 11 - retries
|
59
|
+
sleep sleep_time # <= Fusion tables has rate of 5 calls per second. Be nice, get longer
|
60
|
+
@session_cookie = e.response.headers['set-cookie']
|
61
|
+
return self.make_request(method, url, body, retries - 1)
|
62
|
+
else
|
63
|
+
return e.response
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
# Custom prepare_headers to include the session cookie if it exists
|
69
|
+
def prepare_headers
|
70
|
+
if @session_cookie
|
71
|
+
@headers['cookie'] = @session_cookie
|
72
|
+
end
|
73
|
+
super
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# Copyright (C) 2010 Simon Tokumine
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
module GData
|
16
|
+
module Client
|
17
|
+
class FusionTables < Base
|
18
|
+
class Data
|
19
|
+
include Enumerable
|
20
|
+
|
21
|
+
attr_reader :headers, :body, :live
|
22
|
+
alias :to_h :body
|
23
|
+
|
24
|
+
# configures headers hash and sets up
|
25
|
+
def initialize options
|
26
|
+
@headers = options[:headers]
|
27
|
+
@body = options[:body]
|
28
|
+
end
|
29
|
+
|
30
|
+
# Reads in CSV
|
31
|
+
def self.parse response
|
32
|
+
body = []
|
33
|
+
headers = []
|
34
|
+
|
35
|
+
first = true
|
36
|
+
if CSV.const_defined? :Reader
|
37
|
+
CSV::Reader.parse(response.body) do |row|
|
38
|
+
if first
|
39
|
+
first = false
|
40
|
+
headers = row.map { |x|x.strip.downcase.gsub(" ","_").to_sym }
|
41
|
+
next
|
42
|
+
end
|
43
|
+
body << Hash[*headers.zip(row).flatten]
|
44
|
+
end
|
45
|
+
else
|
46
|
+
CSV.parse(response.body) do |row|
|
47
|
+
if first
|
48
|
+
first = false
|
49
|
+
headers = row.map { |x|x.strip.downcase.gsub(" ","_").to_sym }
|
50
|
+
next
|
51
|
+
end
|
52
|
+
body << Hash[*headers.zip(row).flatten]
|
53
|
+
end
|
54
|
+
end
|
55
|
+
self.new :headers => headers, :body => body
|
56
|
+
end
|
57
|
+
|
58
|
+
# Implement enumerable
|
59
|
+
def each
|
60
|
+
@body.each { |i| yield i }
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
# Encodes row according to type
|
65
|
+
def encode
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
# Copyright (C) 2010 Simon Tokumine
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
module GData
|
16
|
+
module Client
|
17
|
+
class FusionTables < Base
|
18
|
+
class Table
|
19
|
+
attr_reader :headers, :id, :name
|
20
|
+
|
21
|
+
# configures headers hash and sets up
|
22
|
+
#
|
23
|
+
# eg options: {:table_id => "x", :name => "y"}
|
24
|
+
#
|
25
|
+
def initialize client, options
|
26
|
+
raise ArgumentError, "need ft client" if client.class != GData::Client::FusionTables
|
27
|
+
raise ArgumentError, "need table_id and name hash" if !options.has_key?(:name) || !options.has_key?(:table_id)
|
28
|
+
@client = client
|
29
|
+
@id = options[:table_id]
|
30
|
+
@name = options[:name]
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
# Sets up data types from google
|
35
|
+
#
|
36
|
+
def describe
|
37
|
+
GData::Client::FusionTables::Data.parse(@client.sql_get("DESCRIBE #{@id}")).body
|
38
|
+
end
|
39
|
+
|
40
|
+
# Runs select and returns data obj
|
41
|
+
#
|
42
|
+
# Define columns and SQL conditions separatly
|
43
|
+
#
|
44
|
+
# See http://code.google.com/apis/fusiontables/docs/developers_reference.html#Select
|
45
|
+
#
|
46
|
+
# use columns=ROWID to select row ids
|
47
|
+
#
|
48
|
+
def select columns="*", conditions=nil
|
49
|
+
sql = "SELECT #{columns} FROM #{@id} #{conditions}"
|
50
|
+
GData::Client::FusionTables::Data.parse(@client.sql_get(sql)).body
|
51
|
+
end
|
52
|
+
|
53
|
+
# Returns a count of rows. SQL conditions optional
|
54
|
+
#
|
55
|
+
def count conditions=nil
|
56
|
+
select("count()", conditions).first.values.first.to_i
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
# Outputs data to an array of concatenated INSERT SQL statements
|
61
|
+
#
|
62
|
+
# format should be:
|
63
|
+
#
|
64
|
+
# [{:col_1 => data, :col2 => data}, {:col_1 => data, :col2 => data}]
|
65
|
+
#
|
66
|
+
# Fields are escaped and formatted for FT based on type
|
67
|
+
#
|
68
|
+
def insert data
|
69
|
+
data = [data] unless data.respond_to?(:to_ary)
|
70
|
+
|
71
|
+
# encode values to insert
|
72
|
+
data = encode data
|
73
|
+
|
74
|
+
# Chunk up the data and send
|
75
|
+
chunk = ""
|
76
|
+
data.each_with_index do |d,i|
|
77
|
+
chunk << "INSERT INTO #{@id} (#{ d.keys.join(",") }) VALUES (#{ d.values.join(",") });"
|
78
|
+
if (i+1) % 500 == 0 || (i+1) == data.size
|
79
|
+
begin
|
80
|
+
@client.sql_post(chunk)
|
81
|
+
chunk = ""
|
82
|
+
rescue => e
|
83
|
+
raise "INSERT to table:#{@id} failed on row #{i} with #{e}"
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# Runs update on rows and return data obj
|
90
|
+
# No bulk update, so may aswell drop table and start again
|
91
|
+
def update row_id, data
|
92
|
+
data = encode([data]).first
|
93
|
+
data = data.to_a.map{|x| x.join("=")}.join(", ")
|
94
|
+
|
95
|
+
sql = "UPDATE #{@id} SET #{data} WHERE ROWID = #{row_id}"
|
96
|
+
GData::Client::FusionTables::Data.parse(@client.sql_post(sql)).body
|
97
|
+
end
|
98
|
+
|
99
|
+
# delete row
|
100
|
+
# no bulk delete so may aswell drop table and start again
|
101
|
+
def delete row_id
|
102
|
+
sql = "DELETE FROM #{@id} WHERE rowid='#{row_id}'"
|
103
|
+
GData::Client::FusionTables::Data.parse(@client.sql_post(sql)).body
|
104
|
+
end
|
105
|
+
|
106
|
+
# Delete all the data from one table
|
107
|
+
def truncate!
|
108
|
+
GData::Client::FusionTables::Data.parse(@client.sql_post("DELETE FROM #{@id}")).body
|
109
|
+
end
|
110
|
+
|
111
|
+
def get_headers
|
112
|
+
@headers ||= describe
|
113
|
+
end
|
114
|
+
|
115
|
+
def encode data
|
116
|
+
data.inject([]) do |ar,h|
|
117
|
+
ret = {}
|
118
|
+
h.each do |key, value|
|
119
|
+
ret["'#{key.to_s}'"] = case get_datatype(key)
|
120
|
+
when "number" then "#{value}"
|
121
|
+
when "datetime" then "'#{value.strftime("%m-%d-%Y %H:%M:%S")}'"
|
122
|
+
else "'#{value.gsub(/\\/, '\&\&').gsub(/'/, "''")}'"
|
123
|
+
end
|
124
|
+
end
|
125
|
+
ar << ret
|
126
|
+
ar
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
#
|
131
|
+
# Returns datatype of given column name
|
132
|
+
#
|
133
|
+
def get_datatype column_name
|
134
|
+
get_headers
|
135
|
+
|
136
|
+
@headers.each do |h|
|
137
|
+
return h[:type] if h[:name] == column_name.to_s
|
138
|
+
end
|
139
|
+
raise ArgumentError "The column doesn't exist"
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
# Copyright (C) 2010 Tom Verbeure, Simon Tokumine
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
module GData
|
16
|
+
module Client
|
17
|
+
class FusionTables < Base
|
18
|
+
|
19
|
+
# Show a list of fusion tables
|
20
|
+
def show_tables
|
21
|
+
data = GData::Client::FusionTables::Data.parse(self.sql_get("SHOW TABLES"))
|
22
|
+
data.inject([]) do |x, row|
|
23
|
+
x << GData::Client::FusionTables::Table.new(self, row)
|
24
|
+
x
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
# Create a new table. Return the corresponding table
|
30
|
+
#
|
31
|
+
# Columns specified as [{:name => 'my_col_name', :type => 'my_type'}]
|
32
|
+
#
|
33
|
+
# Type must be one of:
|
34
|
+
#
|
35
|
+
# * number
|
36
|
+
# * string
|
37
|
+
# * location
|
38
|
+
# * datetime
|
39
|
+
#
|
40
|
+
def create_table(table_name, columns)
|
41
|
+
|
42
|
+
# Sanity check name
|
43
|
+
table_name = table_name.strip.gsub(/ /,'_')
|
44
|
+
|
45
|
+
# ensure all column types are valid
|
46
|
+
columns.each do |col|
|
47
|
+
if !DATATYPES.include? col[:type].downcase
|
48
|
+
raise ArgumentError, "Ensure input types are: 'number', 'string', 'location' or 'datetime'"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# generate sql
|
53
|
+
fields = columns.map{ |col| "'#{col[:name]}': #{col[:type].upcase}" }.join(", ")
|
54
|
+
sql = "CREATE TABLE #{table_name} (#{fields})"
|
55
|
+
|
56
|
+
# create table
|
57
|
+
resp = self.sql_post(sql)
|
58
|
+
raise "unknown column type" if resp.body == "Unknown column type."
|
59
|
+
|
60
|
+
# construct table object and return
|
61
|
+
table_id = resp.body.split("\n")[1].chomp.to_i
|
62
|
+
table = GData::Client::FusionTables::Table.new(self, :table_id => table_id, :name => table_name)
|
63
|
+
table.get_headers
|
64
|
+
table
|
65
|
+
end
|
66
|
+
|
67
|
+
# Drops Fusion Tables
|
68
|
+
#
|
69
|
+
# options can be:
|
70
|
+
#
|
71
|
+
# * an integer for single drop
|
72
|
+
# * array of integers for multi drop
|
73
|
+
# * a regex against table_name for flexible multi_drop
|
74
|
+
#
|
75
|
+
def drop(options)
|
76
|
+
# collect ids
|
77
|
+
ids = []
|
78
|
+
ids << options if options.class == Integer || options.class == String || Fixnum
|
79
|
+
ids = options if options.class == Array
|
80
|
+
|
81
|
+
if options.class == Regexp
|
82
|
+
tables = show_tables
|
83
|
+
ids = tables.map { |table| table.id if options =~ table.name }.compact
|
84
|
+
end
|
85
|
+
|
86
|
+
# drop tables
|
87
|
+
delete_count = 0
|
88
|
+
ids.each do |id|
|
89
|
+
resp = self.sql_post("DROP TABLE #{id}")
|
90
|
+
delete_count += 1 if resp.body.strip.downcase == 'ok'
|
91
|
+
end
|
92
|
+
delete_count
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|