daru-td 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 461d8e3126690c28a90e16d5dc7d27b1e8ba15ae
4
+ data.tar.gz: 88dda02d3ca72acb1b1b5e862447d8cbc6357b39
5
+ SHA512:
6
+ metadata.gz: 644ad8268c5acaf8bfb4a6876e92c22711111c0158e624d6f3c337e6f83dbd18a9857aa17db6a79f3ef9a4f3b0f9ca7fbccd3845a5e1bceb47d5fe2c828c75ab
7
+ data.tar.gz: efa56cd6272259114f7411bc828010ed2d27f5552302921319ca4c5211d3fa48182a91f4f09f0a77069bf8a52b2a000e038eee09b38977a340d82f6d2ba82ccc
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --format documentation
2
+ --color
data/.travis.yml ADDED
@@ -0,0 +1,4 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.2.3
4
+ before_install: gem install bundler -v 1.11.0
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in daru-td.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2015 Kenta Murata
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,61 @@
1
+ # Daru::TD
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/daru/td`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'daru-td'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install daru-td
22
+
23
+ ## Usage
24
+
25
+ ### Create connection
26
+
27
+ ```ruby
28
+ conn = Daru::TD.connect(ENV['TD_API_KEY'])
29
+ ```
30
+
31
+ ### Obtain a list of databases and tables as a data frame
32
+
33
+ ```ruby
34
+ df_databases = conn.databases
35
+ df_tables = conn.tables('db_name')
36
+ ```
37
+
38
+ ### Read the query result
39
+
40
+ ```ruby
41
+ engine = Daru::TD.create_engine('presto:sample_datasets', conn:conn)
42
+ df_result = Daru::TD.read_td_query(<<-SQL, engine)
43
+ select * From nasdaq limit 3
44
+ SQL
45
+ ```
46
+
47
+ ## Development
48
+
49
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
50
+
51
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
52
+
53
+ ## Contributing
54
+
55
+ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/daru-td.
56
+
57
+
58
+ ## License
59
+
60
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
61
+
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "daru/td"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/daru-td.gemspec ADDED
@@ -0,0 +1,29 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'daru/td/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "daru-td"
8
+ spec.version = Daru::TD::VERSION
9
+ spec.authors = ["Kenta Murata"]
10
+ spec.email = ["mrkn@mrkn.jp"]
11
+
12
+ spec.summary = %q{Interactive data analysis with Daru and Treasure Data.}
13
+ spec.description = %q{Interactive data analysis with Daru and Treasure Data.}
14
+ spec.homepage = "https://github.com/mrkn/daru-td"
15
+ spec.license = "MIT"
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ spec.bindir = "exe"
19
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
20
+ spec.require_paths = ["lib"]
21
+
22
+ spec.add_dependency "activesupport"
23
+ spec.add_dependency "daru"
24
+ spec.add_dependency "td"
25
+
26
+ spec.add_development_dependency "bundler", "~> 1.11"
27
+ spec.add_development_dependency "rake", "~> 10.0"
28
+ spec.add_development_dependency "rspec", "~> 3.0"
29
+ end
data/lib/daru-td.rb ADDED
@@ -0,0 +1 @@
1
+ require 'daru/td'
data/lib/daru/td.rb ADDED
@@ -0,0 +1,81 @@
1
+ require 'daru/td/connection'
2
+ require 'daru/td/query_engine'
3
+ require 'daru/td/version'
4
+
5
+ require 'active_support/hash_with_indifferent_access'
6
+ require 'cgi/core'
7
+ require 'uri'
8
+
9
+ module Daru
10
+ module TD
11
+ DEFAULT_ENGINE_TYPE = :presto
12
+
13
+ # @param apikey [String]
14
+ # @param endpoint [String]
15
+ # @param kwargs [Hash]
16
+ def self.connect(apikey=nil, endpoint=nil, **kwargs)
17
+ Connection.new(apikey, endpoint, **kwargs)
18
+ end
19
+
20
+ # Create a handler for query engine based on a URL.
21
+ #
22
+ # The following environment variables are used for default connections:
23
+ #
24
+ # - TD_API_KEY
25
+ # - TD_API_SERVER
26
+ # - HTTP_PROXY
27
+ #
28
+ # @param uri [String] Engine descriptor in the form "type://apikey@host/database?params..."
29
+ # Use shorthand notation "type:database?params..." for the default connection.
30
+ # @param conn [Connection, nil] Handler returned by connect.
31
+ # If not given, the default connection is used.
32
+ # @param header [String, true, false] Prepend comment strings, in the form "-- comment",
33
+ # as a header of queries. Set false to disable header.
34
+ # @param show_progress [Float, true, false] Number of seconds to wait before printing progress.
35
+ # Set false to disable progress entirely.
36
+ # @param clear_progress [true, false] If true, clear progress when query completed.
37
+ # @return [QueryEngine]
38
+ def self.create_engine(uri, conn:nil, header:true, show_progress:5.0, clear_progress:true)
39
+ uri = URI.parse(uri)
40
+ engine_type = (uri.scheme || DEFAULT_ENGINE_TYPE).to_sym
41
+ unless conn
42
+ if uri.host
43
+ apikey, host = uri.userinfo, uri.host
44
+ conn = connect(apikey, "https://#{host}/")
45
+ else
46
+ conn = connect()
47
+ end
48
+ end
49
+ database = uri.path || uri.opaque
50
+ database = database[1..-1] if database.start_with?('/')
51
+ params = {
52
+ type: engine_type
53
+ }
54
+ params.update(parse_query(uri.query)) if uri.query
55
+ return QueryEngine.new(conn, database, params,
56
+ header: header,
57
+ show_progress: show_progress,
58
+ clear_progress: clear_progress)
59
+ end
60
+
61
+ def self.read_td_query(query, engine, **kwargs)
62
+ distributed_join = kwargs.delete(:distributed_join)
63
+ parse_dates = kwargs.delete(:parse_dates)
64
+ header = engine.create_header('read_td_query')
65
+ if engine.type == :presto && distributed_join
66
+ header += "-- set session distributed_join = #{!!distributed_join}\n"
67
+ end
68
+ result = engine.execute(header + query, **kwargs)
69
+ result.to_dataframe(parse_dates: parse_dates)
70
+ end
71
+
72
+ def self.parse_query(query_string)
73
+ CGI.parse(query_string).tap do |hash|
74
+ hash.keys.each do |key|
75
+ hash[key.to_sym] = hash.delete(key)
76
+ end
77
+ end
78
+ end
79
+ private_class_method :parse_query
80
+ end
81
+ end
@@ -0,0 +1,83 @@
1
+ require 'daru'
2
+ require 'td'
3
+ require 'td-client'
4
+
5
+ module Daru
6
+ module TD
7
+ class Connection
8
+ def initialize(apikey=nil, endpoint=nil, **kwargs)
9
+ if apikey.nil? && kwargs[:apikey]
10
+ apikey = kwargs.delete(:apikey)
11
+ end
12
+
13
+ if endpoint
14
+ unless endpoint.end_with?('/')
15
+ endpoint = endpoint + '/'
16
+ end
17
+ kwargs[:endpoint] = endpoint
18
+ end
19
+
20
+ if kwargs[:user_agent].nil?
21
+ versions = [
22
+ "daru/#{Daru::VERSION}",
23
+ "td-client/#{::TD::Client::VERSION}",
24
+ "ruby/#{RUBY_VERSION}"
25
+ ]
26
+ kwargs[:user_agent] = "daru-td/#{Daru::TD::VERSION} (#{versions.join(' ')})"
27
+ end
28
+
29
+ @kwargs = kwargs
30
+ @client = get_client(apikey, **kwargs)
31
+ end
32
+
33
+ attr_reader :client
34
+
35
+ # @return [String] TreasureData API key
36
+ def apikey
37
+ @client.apikey
38
+ end
39
+
40
+ # See https://github.com/treasure-data/td-client-ruby/blob/master/lib/td/client/api.rb#L67
41
+ # @return [String] TreasureData endpoint URL
42
+ def endpoint
43
+ @kwargs[:endpoint] || ENV['TD_API_SERVER'] || ::TD::API::DEFAULT_ENDPOINT
44
+ end
45
+
46
+ def databases
47
+ if (databases = self.client.databases())
48
+ fields = [:name, :count, :permission, :created_at, :updated_at]
49
+ make_dataframe(databases, [:name, :count, :permission, :created_at, :updated_at]) do |db|
50
+ [db.name, db.count, db.permission, db.created_at, db.updated_at]
51
+ end
52
+ else
53
+ Daru::DataFrame.new()
54
+ end
55
+ end
56
+
57
+ def tables(database)
58
+ if (tables = self.client.tables(database))
59
+ fields = [:name, :count, :estimated_storage_size, :last_log_timestamp, :created_at]
60
+ make_dataframe(tables, [:name, :count, :estimated_storage_size, :last_log_timestamp, :created_at]) do |t|
61
+ [t.name, t.count, t.estimated_storage_size, t.last_log_timestamp, t.created_at]
62
+ end
63
+ else
64
+ Daru::DataFrame.new()
65
+ end
66
+ end
67
+
68
+ private
69
+
70
+ def get_client(apikey, **kwargs)
71
+ ::TD::Client.new(apikey, **kwargs)
72
+ end
73
+
74
+ def make_dataframe(enum, fields)
75
+ Daru::DataFrame.new([], order: fields).tap do |df|
76
+ enum.each do |item|
77
+ df.add_row(yield(item))
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,11 @@
1
+ require 'iruby/display'
2
+
3
+ module IRuby
4
+ module Display
5
+ unless self.respond_to?(:clear_output)
6
+ def self.clear_output(wait=false)
7
+ IRuby::Kernel.instance.session.send(:publish, :clear_output, {wait: wait})
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,185 @@
1
+ require 'daru/td/iruby/display'
2
+ require 'daru/td/result_proxy'
3
+
4
+ require 'cgi/util'
5
+ require 'delegate'
6
+ require 'erb'
7
+ require 'strscan'
8
+
9
+ module Daru
10
+ module TD
11
+ class QueryEngine
12
+ class JobWrapper < SimpleDelegator
13
+ attr_accessor :issued_at
14
+
15
+ class TimeoutError < StandardError; end
16
+
17
+ def wait(timeout=nil, wait_interval=2)
18
+ started_at = Time.now
19
+ until finished?
20
+ if !timeout || ((Time.now - started_at).abs > timeout && wait_interval <= timeout)
21
+ sleep wait_interval
22
+ yield self if block_given?
23
+ else
24
+ raise TimeoutError, "timeout"
25
+ end
26
+ update_progress!
27
+ end
28
+ end
29
+ end
30
+
31
+ def initialize(connection, database, params={}, header:false, show_progress:false, clear_progress:false)
32
+ @connection = connection
33
+ @database = database
34
+ @params = params
35
+ @header = header
36
+ if iruby_notebook?
37
+ # Enable progress for IRuby notebook
38
+ @show_progress = show_progress
39
+ @clear_progress = clear_progress
40
+ else
41
+ @show_progress = false
42
+ @clear_progress = false
43
+ end
44
+ end
45
+
46
+ attr_reader :connection, :show_progress, :clear_progress
47
+
48
+ def type
49
+ @params[:type]
50
+ end
51
+
52
+ def create_header(name)
53
+ return '' unless @header
54
+ return "-- #{@header}\n" if String === @header
55
+ "-- #{name}\n"
56
+ end
57
+
58
+ def execute(query, **kwargs)
59
+ params = @params.dup
60
+ params.update(kwargs)
61
+
62
+ # Issue query
63
+ issued_at = Time.now.utc.round
64
+ result_url = params.delete(:result_url)
65
+ priority = params.delete(:priority)
66
+ retry_limit = params.delete(:retry_limit)
67
+ job = JobWrapper.new(connection.client.query(@database, query, result_url, priority, retry_limit, params))
68
+ job.issued_at = issued_at
69
+
70
+ get_result(job, wait: true)
71
+ rescue Interrupt
72
+ job.kill()
73
+ raise
74
+ end
75
+
76
+ def wait_callback(job, cursize=nil)
77
+ display_progress(job, cursize)
78
+ end
79
+
80
+ def job_finished?(job)
81
+ job.update_progress!
82
+ job.finished?
83
+ end
84
+
85
+ def get_result(job, wait: true)
86
+ if wait
87
+ job.wait(nil, 2, &method(:wait_callback))
88
+ end
89
+
90
+ # status check
91
+ unless job.success?
92
+ if job.debug && job.debug['stderr']
93
+ logger.error(job.debug['stderr'])
94
+ end
95
+ raise "job #{job.job_id} #{job.status}"
96
+ end
97
+
98
+ ResultProxy.new(self, job)
99
+ end
100
+
101
+ private
102
+
103
+ def display_progress(job, cursize=nil)
104
+ return unless show_progress
105
+ if show_progress.is_a?(Integer) && job.issued_at
106
+ return if Time.now.getutc < job.issued_at + show_progress
107
+ end
108
+
109
+ IRuby::Display.clear_output(true)
110
+ html = render_progress_html_erb(binding)
111
+ IRuby.display(IRuby.html(html))
112
+ end
113
+
114
+ def iruby_notebook?
115
+ defined?(IRuby) && !$stdout.tty?
116
+ end
117
+
118
+ def render_progress_html_erb(given_binding)
119
+ template = <<-'END_ERB'
120
+ <div style="border-style: dashed; border-width: 1px;">
121
+ <%=html_text("issued at #{job.issued_at.iso8601}") %>
122
+ URL: <a href="<%=job.url %>" target="_blank"><%=job.url %></a><br>
123
+ <% if job.type == :presto %>
124
+ <% if job.debug && job.debug['cmdout'] %>
125
+ <%= html_presto_output(job.debug['cmdout']) %>
126
+ <% end %>
127
+ <% end %>
128
+ <% if job.result_size %>
129
+ Result size: <%=escape_html(job.result_size) %> bytes<br>
130
+ <% end %>
131
+ <% if cursize %>
132
+ Download: <%=escape_html(cursize) %> / <%=escape_html(job.result_size) %> bytes
133
+ (<%=escape_html('%.2f' % [cursize * 100.0 / job.result_size]) %>%)<br>
134
+ <% if cursize >= job.result_size %>
135
+ downloaded at <%=escape_html(Time.now.getutc.round.iso8601) %>
136
+ <% end %>
137
+ <% end %>
138
+ </div>
139
+ END_ERB
140
+ erb = ERB.new(template)
141
+ erb.filename = 'render_progress_html_erb'
142
+ erb.result(given_binding)
143
+ end
144
+
145
+ def html_presto_output(cmdout)
146
+ template = <<-'END_PRESTO_OUTPUT'
147
+ <% # started at %>
148
+ <% cmdout.scan(/started at.*$/) do |text| %>
149
+ <%= html_text(text) %>
150
+ <% end %>
151
+ <% # warning %>
152
+ <pre style="color: #c44;">
153
+ <% cmdout.scan(/^\*{2} .*$/) do |text| %>
154
+ <%= escape_html(text) %>
155
+ <% end %>
156
+ </pre>
157
+ <% # progress %>
158
+ <% progress = cmdout.scan(/\n\d{4}-\d{2}-\d{2}.*(?:\n +.*)+/).last %>
159
+ <% if progress %>
160
+ <pre><%=escape_html(progress) %></pre>
161
+ <% end %>
162
+ <% # rows %>
163
+ <% cmdout.scan(/^\d+ rows/) do |text| %>
164
+ <%= escape_html(text) %><br />
165
+ <% end %>
166
+ <% # finished at %>
167
+ <% cmdout.scan(/finished at.*$/) do |text| %>
168
+ <%= html_text(text) %>
169
+ <% end %>
170
+ END_PRESTO_OUTPUT
171
+ erb = ERB.new(template)
172
+ erb.filename = 'html_presto_output'
173
+ erb.result(binding)
174
+ end
175
+
176
+ def html_text(text)
177
+ %Q[<div style="color: #888;"># #{escape_html(text)}</div>]
178
+ end
179
+
180
+ def escape_html(text)
181
+ CGI.escape_html(text.to_s)
182
+ end
183
+ end
184
+ end
185
+ end
@@ -0,0 +1,149 @@
1
+ require 'daru/td/iruby/display'
2
+
3
+ require 'open-uri'
4
+ require 'uri'
5
+ require 'msgpack'
6
+ require 'zlib'
7
+
8
+ module Daru
9
+ module TD
10
+ class ResultProxy
11
+ def initialize(engine, job)
12
+ @engine = engine
13
+ @job = job
14
+ @http = nil
15
+ end
16
+
17
+ attr_reader :engine, :job
18
+
19
+ def status
20
+ job.status
21
+ end
22
+
23
+ def size
24
+ if !job.finished?
25
+ job.wait()
26
+ end
27
+ job.result_size
28
+ end
29
+
30
+ def description
31
+ if !job.finished?
32
+ job.wait()
33
+ end
34
+ job.hive_result_schema
35
+ end
36
+
37
+ def readpartial(len=16384, outbuf="")
38
+ @result_io ||= open_result_io()
39
+ @result_io.readpartial(len, outbuf)
40
+ end
41
+
42
+ def each_record(&block)
43
+ MessagePack::Unpacker.new(self).each(&block)
44
+ end
45
+
46
+ def to_dataframe(parse_dates: nil)
47
+ fields = description.map {|c| c[0].to_sym }
48
+ Daru::DataFrame.new([], order: fields).tap do |df|
49
+ each_record do |record|
50
+ df.add_row(record)
51
+ end
52
+ if parse_dates
53
+ parse_date_fields(df, parse_dates)
54
+ end
55
+ if engine.clear_progress
56
+ IRuby::Display.clear_output()
57
+ end
58
+ end
59
+ end
60
+
61
+ private
62
+
63
+ def parse_date_fields(df, fields)
64
+ fields.each do |name|
65
+ parsed_values = df[name].map {|v| DateTime.parse(v) }
66
+ df[name] = Daru::Vector.new(parsed_values, name: name)
67
+ end
68
+ df
69
+ end
70
+
71
+ def open_result_io
72
+ Zlib::GzipReader.new(content_downloader())
73
+ end
74
+
75
+ def content_downloader
76
+ ContentDownloader.new(self, job_result_uri, http_header) do |downloader|
77
+ engine.wait_callback(job, downloader.downloaded_size)
78
+ end
79
+ end
80
+
81
+ def job_result_uri
82
+ endpoint_uri = URI.parse(self.engine.connection.endpoint)
83
+ unless endpoint_uri.scheme
84
+ endpoint_uri = URI.parse("https://#{endpoint_uri}")
85
+ end
86
+ URI.join(endpoint_uri, "v3/job/result/#{job.job_id}?format=msgpack.gz")
87
+ end
88
+
89
+ def http_header
90
+ {
91
+ 'Authorization' => "TD1 #{self.engine.connection.apikey}",
92
+ 'Accept-Encoding' => 'deflate, gzip',
93
+ 'User-Agent' => "daru-td/#{Daru::TD::VERSION} (Ruby #{RUBY_VERSION})",
94
+ }
95
+ end
96
+
97
+ class ContentDownloader
98
+ def initialize(result_proxy, url, http_header, &callback)
99
+ @callback = callback
100
+ @url = url
101
+ @http_header = http_header
102
+ @downloaded_size = 0
103
+ end
104
+
105
+ attr_reader :downloaded_size
106
+
107
+ def read(length=nil, outbuf="")
108
+ if closed?
109
+ raise IOError, "read from closed IO"
110
+ end
111
+ if (result = io.read(length, outbuf))
112
+ @downloaded_size += result.bytesize
113
+ callback
114
+ end
115
+ result
116
+ end
117
+
118
+ def readpartial(maxlen, outbuf="")
119
+ if closed?
120
+ raise IOError, "read from closed IO"
121
+ end
122
+ if (result = io.readpartial(maxlen, outbuf))
123
+ @downloaded_size += result.bytesize
124
+ callback
125
+ end
126
+ result
127
+ end
128
+
129
+ def closed?
130
+ @io && @io.closed?
131
+ end
132
+
133
+ def close
134
+ @io && @io.close
135
+ end
136
+
137
+ private
138
+
139
+ def callback
140
+ @callback && @callback.(self)
141
+ end
142
+
143
+ def io
144
+ @io ||= open(@url, @http_header)
145
+ end
146
+ end
147
+ end
148
+ end
149
+ end
@@ -0,0 +1,5 @@
1
+ module Daru
2
+ module TD
3
+ VERSION = "0.1.0"
4
+ end
5
+ end
metadata ADDED
@@ -0,0 +1,146 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: daru-td
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Kenta Murata
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2015-12-24 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: activesupport
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: daru
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: td
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: bundler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.11'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.11'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rake
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '10.0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '10.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rspec
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '3.0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '3.0'
97
+ description: Interactive data analysis with Daru and Treasure Data.
98
+ email:
99
+ - mrkn@mrkn.jp
100
+ executables: []
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - ".gitignore"
105
+ - ".rspec"
106
+ - ".travis.yml"
107
+ - Gemfile
108
+ - LICENSE.txt
109
+ - README.md
110
+ - Rakefile
111
+ - bin/console
112
+ - bin/setup
113
+ - daru-td.gemspec
114
+ - lib/daru-td.rb
115
+ - lib/daru/td.rb
116
+ - lib/daru/td/connection.rb
117
+ - lib/daru/td/iruby/display.rb
118
+ - lib/daru/td/query_engine.rb
119
+ - lib/daru/td/result_proxy.rb
120
+ - lib/daru/td/version.rb
121
+ homepage: https://github.com/mrkn/daru-td
122
+ licenses:
123
+ - MIT
124
+ metadata: {}
125
+ post_install_message:
126
+ rdoc_options: []
127
+ require_paths:
128
+ - lib
129
+ required_ruby_version: !ruby/object:Gem::Requirement
130
+ requirements:
131
+ - - ">="
132
+ - !ruby/object:Gem::Version
133
+ version: '0'
134
+ required_rubygems_version: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ requirements: []
140
+ rubyforge_project:
141
+ rubygems_version: 2.4.5.1
142
+ signing_key:
143
+ specification_version: 4
144
+ summary: Interactive data analysis with Daru and Treasure Data.
145
+ test_files: []
146
+ has_rdoc: