influxdb 0.3.5 → 0.3.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c027fe626b731c0fe89678ae44f7f87a24db7c68
+  data.tar.gz: 929dc5eda07eabfba284891ede0a293a6b0fc49a
+SHA512:
+  metadata.gz: 64db490708b15350c4e2ab2cc76b5f934db510a86eec7be024f9a6c652e26d09fa2570b729ab6987887382769b5b9a1f5bbfdb0a3c8fc96775ce2db7b0a563fb
+  data.tar.gz: b4fc2890772ec74612dac94633e5607904b4aadeef0a3760d698b24cbf16d4a23e6c004edece2b15d8926fa21e028f7479a26d168a03ab59ae4cbaedccc782f5

data/.travis.yml

@@ -2,6 +2,7 @@ sudo: required
 dist: trusty
 language: ruby
 before_install:
+  - gem install bundler
   - gem update bundler
 rvm:
   - 1.9.3
@@ -21,12 +22,13 @@ matrix:
     - rvm: 2.3.1
       env: TEST_TASK=rubocop
     - rvm: jruby-9.0.5.0
-      env: JRUBY_OPTS='-J-Xmx256M'
+      env: JRUBY_OPTS='--client -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-Xss2m -J-Xmx256M'
     - rvm: jruby-head
-      env: JRUBY_OPTS='-J-Xmx256M'
+      env: JRUBY_OPTS='--client -J-XX:+TieredCompilation -J-XX:TieredStopAtLevel=1 -J-Xss2m -J-Xmx256M'
   fail_fast: true
 addons:
   apt:
     packages:
+      - haveged
       - libgmp-dev
 script: bundle exec rake $TEST_TASK

data/CHANGELOG.md

@@ -6,6 +6,11 @@ For the full commit log, [see here](https://github.com/influxdata/influxdb-ruby/
 
 - None.
 
+## v0.3.6, released 2016-07-24
+
+- Added feature for JSON streaming response, via `"chunk_size"` parameter
+  (#155, @mhodson-qxbranch)
+
 ## v0.3.5, released 2016-06-09
 
 - Reintroduced full dependency on "cause" (for Ruby 1.9 compat).

data/Gemfile

@@ -1,5 +1,9 @@
 source "https://rubygems.org"
 
+if RUBY_ENGINE != "jruby" && RUBY_VERSION < "2.0"
+  gem "json", "~> 1.8.3"
+end
+
 gemspec
 
 local_gemfile = 'Gemfile.local'

data/README.md

@@ -18,6 +18,9 @@ Maintained by [@toddboom](https://github.com/toddboom) and [@dmke](https://githu
 
 This gem should work with Ruby 1.9+, but starting with v0.4, we'll likely drop Ruby 1.9 support.
 
+Please note that for Ruby 1.9, you'll need to install the JSON gem in version 1.8.x yourself,
+for example by pinning the version in your `Gemfile` (i.e. `gem "json", "~> 1.8.3"`).
+
 ## Install
 
 ```
@@ -424,7 +427,7 @@ influxdb.write_point(name, data)
 
 ### Reading data
 
-Querying:
+### Querying
 
 ``` ruby
 username = 'foo'
@@ -468,6 +471,8 @@ influxdb.query 'select * from time_series', epoch: 'ms'
 # [{"name"=>"time_series", "tags"=>{"region"=>"uk"}, "values"=>[{"time"=>1438411376000, "count"=>32, "value"=>0.9673}]}]
 ```
 
+#### (De-) Normalization
+
 By default, InfluxDB::Client will denormalize points (received from InfluxDB as columns and rows), if you want to get _raw_ data add `denormalize: false` to initialization options or to query itself:
 
 ``` ruby
@@ -487,6 +492,22 @@ end
 # time_series_1 [ {"region"=>"us"} ] => {"columns"=>["time", "count", "value"], "values"=>[["2015-07-09T09:02:54Z", 55, 0.4343]]}
 ```
 
+#### Streaming response
+
+If you expect large quantities of data in a response, you may want to enable JSON streaming by setting a `chunk_size`:
+
+``` ruby
+influxdb = InfluxDB::Client.new database,
+  username:   username,
+  password:   password,
+  chunk_size: 10000
+```
+
+See the [official documentation](http://docs.influxdata.com/influxdb/v0.13/guides/querying_data/#chunking) for more details.
+
+
+#### Retry
+
 By default, InfluxDB::Client will keep trying to connect to the database when it gets connection denied,
 if you want to retry a finite number of times (or disable retries altogether), you should pass the `:retry`
 value.

data/lib/influxdb/client/http.rb

@@ -4,6 +4,7 @@ require 'net/http'
 require 'net/https'
 
 module InfluxDB
+  # rubocop:disable Metrics/ModuleLength
   # rubocop:disable Metrics/MethodLength
   # rubocop:disable Metrics/AbcSize
   module HTTP # :nodoc:
@@ -85,7 +86,14 @@ module InfluxDB
     end
 
     def handle_successful_response(response, options)
-      parsed_response = JSON.parse(response.body) if response.body
+      if options.fetch(:json_streaming, false)
+        parsed_response = response.body.each_line.with_object({}) do |line, parsed|
+          parsed.merge!(JSON.parse(line)) { |_key, oldval, newval| oldval + newval }
+        end
+      elsif response.body
+        parsed_response = JSON.parse(response.body)
+      end
+
       errors = errors_from_response(parsed_response)
 
       raise InfluxDB::QueryError, errors if errors

data/lib/influxdb/config.rb

@@ -20,6 +20,7 @@ module InfluxDB
                   :read_timeout,
                   :retry,
                   :prefix,
+                  :chunk_size,
                   :denormalize,
                   :epoch
 
@@ -30,6 +31,7 @@ module InfluxDB
       extract_ssl_options!(opts)
       extract_database_options!(opts)
       extract_writer_options!(opts)
+      extract_query_options!(opts)
 
       configure_retry! opts.fetch(:retry, nil)
       configure_hosts! opts[:hosts] || opts[:host] || "localhost".freeze
@@ -110,5 +112,9 @@ module InfluxDB
       @async = opts.fetch :async, false
       @udp   = opts.fetch :udp, false
     end
+
+    def extract_query_options!(opts)
+      @chunk_size = opts.fetch :chunk_size, nil
+    end
   end
 end

data/lib/influxdb/query/core.rb

@@ -14,9 +14,11 @@ module InfluxDB
       # rubocop:disable Metrics/MethodLength
       def query(query, opts = {})
         denormalize = opts.fetch(:denormalize, config.denormalize)
+        json_streaming = !opts.fetch(:chunk_size, config.chunk_size).nil?
+
         params = query_params(query, opts)
         url = full_url("/query".freeze, params)
-        series = fetch_series(get(url, parse: true))
+        series = fetch_series(get(url, parse: true, json_streaming: json_streaming))
 
         if block_given?
           series.each do |s|
@@ -67,9 +69,16 @@ module InfluxDB
       def query_params(query, opts)
         precision   = opts.fetch(:precision, config.time_precision)
         epoch       = opts.fetch(:epoch, config.epoch)
+        chunk_size  = opts.fetch(:chunk_size, config.chunk_size)
 
         params = { q: query, db: config.database, precision: precision }
         params[:epoch] = epoch if epoch
+
+        if chunk_size
+          params[:chunked] = 'true'
+          params[:chunk_size] = chunk_size
+        end
+
         params
       end
 

data/lib/influxdb/version.rb

@@ -1,3 +1,3 @@
 module InfluxDB # :nodoc:
-  VERSION = "0.3.5".freeze
+  VERSION = "0.3.6".freeze
 end

data/spec/influxdb/cases/querying_issue_7000_spec.rb

@@ -0,0 +1,89 @@
+# This test spec addresses closed issue https://github.com/influxdata/influxdb/issues/7000 where
+# it was confirmed that when chunking is enabled, the InfluxDB REST API returns multi-line JSON.
+
+require "spec_helper"
+require "json"
+
+describe InfluxDB::Client do
+  let(:subject) do
+    described_class.new "database", {
+      host: "influxdb.test",
+      port: 9999,
+      username: "username",
+      password: "password",
+      time_precision: "s"
+    }.merge(args)
+  end
+
+  let(:args)          { {} }
+  let(:database)      { subject.config.database }
+  let(:extra_params)  { {} }
+  let(:response)      { "" }
+
+  before do
+    stub_request(:get, "http://influxdb.test:9999/query")
+      .with(query: { q: query, u: "username", p: "password", precision: 's', db: database }.merge(extra_params))
+      .to_return(body: response)
+  end
+
+  describe "#query" do
+    context "with series with different tags (multi-line)" do
+      let(:args) { { chunk_size: 100 } }
+      let(:extra_params) { { chunked: "true", chunk_size: "100" } }
+
+      let(:response_line_1) do
+        { "results" => [{ "series" => [{ "name" => "cpu", "tags" => { "region" => "pl" }, "columns" => %w(time temp value), "values" => [["2015-07-07T15:13:04Z", 34, 0.343443]] }] }] }
+      end
+      let(:response_line_2) do
+        { "results" => [{ "series" => [{ "name" => "cpu", "tags" => { "region" => "us" }, "columns" => %w(time temp value), "values" => [["2015-07-07T14:58:37Z", 92, 0.3445], ["2015-07-07T14:59:09Z", 68, 0.8787]] }] }] }
+      end
+      let(:response) do
+        JSON.generate(response_line_1) + "\n" + JSON.generate(response_line_2)
+      end
+      let(:expected_result) do
+        [{ "name" => "cpu", "tags" => { "region" => "pl" },
+           "values" => [{ "time" => "2015-07-07T15:13:04Z", "temp" => 34, "value" => 0.343443 }] },
+         { "name" => "cpu", "tags" => { "region" => "us" },
+           "values" => [{ "time" => "2015-07-07T14:58:37Z", "temp" => 92, "value" => 0.3445 },
+                        { "time" => "2015-07-07T14:59:09Z", "temp" => 68, "value" => 0.8787 }] }]
+      end
+      let(:query) { 'SELECT * FROM cpu' }
+
+      it "should return array with 2 elements grouped by tags" do
+        expect(subject.query(query)).to eq(expected_result)
+      end
+    end
+
+    context "with multiple series with different tags" do
+      let(:args) { { chunk_size: 100 } }
+      let(:extra_params) { { chunked: "true", chunk_size: "100" } }
+
+      let(:response_line_1) do
+        { "results" => [{ "series" => [{ "name" => "access_times.service_1", "tags" => { "code" => "200", "result" => "failure", "status" => "OK" }, "columns" => %w(time value), "values" => [["2015-07-08T07:15:22Z", 327]] }] }] }
+      end
+      let(:response_line_2) do
+        { "results" => [{ "series" => [{ "name" => "access_times.service_1", "tags" => { "code" => "500", "result" => "failure", "status" => "Internal Server Error" }, "columns" => %w(time value), "values" => [["2015-07-08T06:15:22Z", 873]] }] }] }
+      end
+      let(:response_line_3) do
+        { "results" => [{ "series" => [{ "name" => "access_times.service_2", "tags" => { "code" => "200", "result" => "failure", "status" => "OK" }, "columns" => %w(time value), "values" => [["2015-07-08T07:15:22Z", 943]] }] }] }
+      end
+      let(:response_line_4) do
+        { "results" => [{ "series" => [{ "name" => "access_times.service_2", "tags" => { "code" => "500", "result" => "failure", "status" => "Internal Server Error" }, "columns" => %w(time value), "values" => [["2015-07-08T06:15:22Z", 606]] }] }] }
+      end
+      let(:response) do
+        JSON.generate(response_line_1) + "\n" + JSON.generate(response_line_2) + "\n" + JSON.generate(response_line_3) + "\n" + JSON.generate(response_line_4)
+      end
+      let(:expected_result) do
+        [{ "name" => "access_times.service_1", "tags" => { "code" => "200", "result" => "failure", "status" => "OK" }, "values" => [{ "time" => "2015-07-08T07:15:22Z", "value" => 327 }] },
+         { "name" => "access_times.service_1", "tags" => { "code" => "500", "result" => "failure", "status" => "Internal Server Error" }, "values" => [{ "time" => "2015-07-08T06:15:22Z", "value" => 873 }] },
+         { "name" => "access_times.service_2", "tags" => { "code" => "200", "result" => "failure", "status" => "OK" }, "values" => [{ "time" => "2015-07-08T07:15:22Z", "value" => 943 }] },
+         { "name" => "access_times.service_2", "tags" => { "code" => "500", "result" => "failure", "status" => "Internal Server Error" }, "values" => [{ "time" => "2015-07-08T06:15:22Z", "value" => 606 }] }]
+      end
+      let(:query) { "SELECT * FROM /access_times.*/" }
+
+      it "should return array with 4 elements grouped by name and tags" do
+        expect(subject.query(query)).to eq(expected_result)
+      end
+    end
+  end
+end

data/spec/influxdb/cases/querying_spec.rb

@@ -134,8 +134,7 @@ describe InfluxDB::Client do
            "values" => [{ "time" => 1_438_580_576, "temp" => 34, "value" => 0.343443 }] },
          { "name" => "cpu", "tags" => { "region" => "us" },
            "values" => [{ "time" => 1_438_612_976, "temp" => 92, "value" => 0.3445 },
-                        { "time" => 1_438_612_989, "temp" => 68, "value" => 0.8787 }]
-        }]
+                        { "time" => 1_438_612_989, "temp" => 68, "value" => 0.8787 }] }]
       end
       let(:query) { 'SELECT * FROM cpu' }
 
@@ -143,6 +142,23 @@ describe InfluxDB::Client do
         expect(subject.query(query)).to eq(expected_result)
       end
     end
+
+    context "with chunk_size set to 100" do
+      let(:args) { { chunk_size: 100 } }
+      let(:extra_params) { { chunked: "true", chunk_size: "100" } }
+
+      let(:response) do
+        { "results" => [{ "series" => [{ "name" => "cpu", "tags" => { "region" => "pl" }, "columns" => %w(time temp value), "values" => [[1_438_580_576, 34, 0.343443]] }] }] }
+      end
+      let(:expected_result) do
+        [{ "name" => "cpu", "tags" => { "region" => "pl" }, "values" => [{ "time" => 1_438_580_576, "temp" => 34, "value" => 0.343443 }] }]
+      end
+      let(:query) { 'SELECT * FROM cpu' }
+
+      it "should set 'chunked' and 'chunk_size' parameters" do
+        expect(subject.query(query)).to eq(expected_result)
+      end
+    end
   end
 
   describe "multiple select queries" do

metadata

@@ -1,126 +1,111 @@
 --- !ruby/object:Gem::Specification
 name: influxdb
 version: !ruby/object:Gem::Version
-  version: 0.3.5
-  prerelease: 
+  version: 0.3.6
 platform: ruby
 authors:
 - Todd Persen
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-09 00:00:00.000000000 Z
+date: 2016-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: json
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: cause
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: cause
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.4.0
-  type: :development
+  name: rspec
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 3.4.0
 - !ruby/object:Gem::Dependency
-  name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.24.2
-  type: :development
+  name: webmock
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.24.2
 - !ruby/object:Gem::Dependency
-  name: rubocop
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.39.0
-  type: :development
+  name: rubocop
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.39.0
 description: This is the official Ruby library for InfluxDB.
@@ -130,9 +115,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rubocop.yml
-- .travis.yml
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -159,13 +144,14 @@ files:
 - spec/influxdb/cases/async_client_spec.rb
 - spec/influxdb/cases/query_cluster_spec.rb
 - spec/influxdb/cases/query_continuous_query_spec.rb
-- spec/influxdb/cases/query_core.rb
+- spec/influxdb/cases/query_core_spec.rb
 - spec/influxdb/cases/query_database_spec.rb
 - spec/influxdb/cases/query_retention_policy_spec.rb
 - spec/influxdb/cases/query_series_spec.rb
 - spec/influxdb/cases/query_shard_space_spec.rb
 - spec/influxdb/cases/query_shard_spec.rb
 - spec/influxdb/cases/query_user_spec.rb
+- spec/influxdb/cases/querying_issue_7000_spec.rb
 - spec/influxdb/cases/querying_spec.rb
 - spec/influxdb/cases/retry_requests_spec.rb
 - spec/influxdb/cases/udp_client_spec.rb
@@ -180,45 +166,39 @@ files:
 homepage: http://influxdb.org
 licenses:
 - MIT
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -2716499157754774119
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -2716499157754774119
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.25
-signing_key: 
-specification_version: 3
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
 summary: Ruby library for InfluxDB.
 test_files:
 - spec/influxdb/cases/async_client_spec.rb
 - spec/influxdb/cases/query_cluster_spec.rb
 - spec/influxdb/cases/query_continuous_query_spec.rb
-- spec/influxdb/cases/query_core.rb
+- spec/influxdb/cases/query_core_spec.rb
 - spec/influxdb/cases/query_database_spec.rb
 - spec/influxdb/cases/query_retention_policy_spec.rb
 - spec/influxdb/cases/query_series_spec.rb
 - spec/influxdb/cases/query_shard_space_spec.rb
 - spec/influxdb/cases/query_shard_spec.rb
 - spec/influxdb/cases/query_user_spec.rb
+- spec/influxdb/cases/querying_issue_7000_spec.rb
 - spec/influxdb/cases/querying_spec.rb
 - spec/influxdb/cases/retry_requests_spec.rb
 - spec/influxdb/cases/udp_client_spec.rb