yancya-big_query 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -6
- data/lib/yancya/big_query/dataset.rb +22 -20
- data/lib/yancya/big_query/datasets.rb +48 -51
- data/lib/yancya/big_query/job.rb +67 -38
- data/lib/yancya/big_query/jobs.rb +55 -55
- data/lib/yancya/big_query/project.rb +21 -19
- data/lib/yancya/big_query/projects.rb +10 -13
- data/lib/yancya/big_query/request_executor.rb +3 -1
- data/lib/yancya/big_query/resource.rb +13 -0
- data/lib/yancya/big_query/table.rb +55 -53
- data/lib/yancya/big_query/tabledata.rb +30 -28
- data/lib/yancya/big_query/tables.rb +70 -73
- data/lib/yancya/big_query/version.rb +1 -1
- data/test/test_big_query.rb +23 -4
- metadata +3 -3
- data/lib/yancya/big_query/initializer.rb +0 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 06183ec4fd4344c825bbc907b5e7ac62807cf921
|
4
|
+
data.tar.gz: 7bef7e5f244f514cb418503ab44f57448c5c1ff0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2b25247ed836fc651980505d1b89270d91bd9b1207e91bad8f258d9e945419381001365b1cd5ef2bc45af96929bc1f6a5a4a7521658c073fd36e5f12b6773e23
|
7
|
+
data.tar.gz: a9fd32c9c007d101e9b221fee9e35ff5a78f7c6a3c49adac08e868ba704dc2a49a464e2df86e62dde8dd9343a4eacc41617a8a9cbb3d184db43034d357ed10cb
|
data/README.md
CHANGED
@@ -8,22 +8,20 @@ Google BigQuery API client library
|
|
8
8
|
|
9
9
|
Add this line to your application's Gemfile:
|
10
10
|
|
11
|
-
gem 'big_query'
|
11
|
+
gem 'yancya-big_query'
|
12
12
|
|
13
13
|
And then execute:
|
14
14
|
|
15
15
|
$ bundle
|
16
16
|
|
17
|
-
Or
|
17
|
+
Or install it yourself as:
|
18
18
|
|
19
|
-
$
|
20
|
-
$ cd big_query
|
21
|
-
$ rake install
|
19
|
+
$ gem install yancya-big_query
|
22
20
|
|
23
21
|
## Usage
|
24
22
|
|
25
23
|
```rb
|
26
|
-
bq = BigQuery.new(
|
24
|
+
bq = Yancya::BigQuery.new(
|
27
25
|
key_path: "path_to_secret_key"
|
28
26
|
issuer: "mail@address"
|
29
27
|
)
|
@@ -1,31 +1,33 @@
|
|
1
1
|
require_relative 'tables'
|
2
2
|
|
3
3
|
module Yancya
|
4
|
-
class BigQuery
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
4
|
+
class BigQuery
|
5
|
+
class Dataset
|
6
|
+
def initialize(resource:, bq:)
|
7
|
+
@resource = resource
|
8
|
+
@tables = BigQuery::Tables.new(bq: bq)
|
9
|
+
end
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
11
|
+
def tables
|
12
|
+
@tables.list(
|
13
|
+
project_id: project_id,
|
14
|
+
dataset_id: dataset_id
|
15
|
+
)
|
16
|
+
end
|
16
17
|
|
17
|
-
|
18
|
-
|
19
|
-
|
18
|
+
def project_id
|
19
|
+
dataset_reference["projectId"]
|
20
|
+
end
|
20
21
|
|
21
|
-
|
22
|
-
|
23
|
-
|
22
|
+
def dataset_id
|
23
|
+
dataset_reference["datasetId"]
|
24
|
+
end
|
24
25
|
|
25
|
-
|
26
|
+
private
|
26
27
|
|
27
|
-
|
28
|
-
|
28
|
+
def dataset_reference
|
29
|
+
@resource["datasetReference"]
|
30
|
+
end
|
29
31
|
end
|
30
32
|
end
|
31
33
|
end
|
@@ -1,63 +1,60 @@
|
|
1
|
-
require_relative '
|
2
|
-
require_relative 'request_executor'
|
3
|
-
|
1
|
+
require_relative 'resource'
|
4
2
|
require_relative 'dataset'
|
5
3
|
|
6
4
|
module Yancya
|
7
|
-
class BigQuery
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
)
|
16
|
-
end
|
17
|
-
|
18
|
-
def get(project_id:, dataset_id:)
|
19
|
-
execute(
|
20
|
-
api_method: @bq.api.datasets.get,
|
21
|
-
parameters: {projectId: project_id, datasetId: dataset_id}
|
22
|
-
)
|
23
|
-
end
|
5
|
+
class BigQuery
|
6
|
+
class Datasets < Resource
|
7
|
+
def delete(project_id:, dataset_id:)
|
8
|
+
execute(
|
9
|
+
api_method: @bq.api.datasets.delete,
|
10
|
+
parameters: {projectId: project_id, datasetId: dataset_id, deleteContents: true}
|
11
|
+
)
|
12
|
+
end
|
24
13
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
end
|
14
|
+
def get(project_id:, dataset_id:)
|
15
|
+
execute(
|
16
|
+
api_method: @bq.api.datasets.get,
|
17
|
+
parameters: {projectId: project_id, datasetId: dataset_id}
|
18
|
+
)
|
19
|
+
end
|
32
20
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
21
|
+
def insert(project_id:, dataset:)
|
22
|
+
execute(
|
23
|
+
api_method: @bq.api.datasets.insert,
|
24
|
+
parameters: {projectId: project_id},
|
25
|
+
body_object: dataset
|
26
|
+
)
|
27
|
+
end
|
38
28
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
29
|
+
def list(project_id:)
|
30
|
+
resources = execute(
|
31
|
+
api_method: @bq.api.datasets.list,
|
32
|
+
parameters: {projectId: project_id}
|
43
33
|
)
|
44
|
-
}
|
45
|
-
end
|
46
34
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
35
|
+
resources["datasets"].map{ |resource|
|
36
|
+
Yancya::BigQuery::Dataset.new(
|
37
|
+
resource: resource,
|
38
|
+
bq: @bq
|
39
|
+
)
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
def patch(project_id:, dataset_id:, dataset:)
|
44
|
+
execute(
|
45
|
+
api_method: @bq.api.datasets.patch,
|
46
|
+
parameters: {projectId: project_id, datasetId: dataset_id},
|
47
|
+
body_object: dataset
|
48
|
+
)
|
49
|
+
end
|
54
50
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
51
|
+
def update(project_id:, dataset_id:, dataset:)
|
52
|
+
execute(
|
53
|
+
api_method: @bq.api.datasets.update,
|
54
|
+
parameters: {projectId: project_id, datasetId: dataset_id},
|
55
|
+
body_object: dataset
|
56
|
+
)
|
57
|
+
end
|
61
58
|
end
|
62
59
|
end
|
63
60
|
end
|
data/lib/yancya/big_query/job.rb
CHANGED
@@ -1,53 +1,82 @@
|
|
1
1
|
module Yancya
|
2
|
-
class BigQuery
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
2
|
+
class BigQuery
|
3
|
+
class Job
|
4
|
+
def initialize(resource:, bq:)
|
5
|
+
@resource = resource
|
6
|
+
@jobs = BigQuery::Jobs.new(bq: bq)
|
7
|
+
end
|
7
8
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
9
|
+
def query_results
|
10
|
+
@jobs.get_query_results(
|
11
|
+
project_id: project_id,
|
12
|
+
job_id: job_id
|
13
|
+
).tap do |qr|
|
14
|
+
if qr["pageToken"]
|
15
|
+
warn <<-TEXT
|
16
|
+
This is a slice in all results.
|
17
|
+
You have not took rest results.
|
18
|
+
Perhaps all_query_results is useful.
|
19
|
+
TEXT
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
14
23
|
|
15
|
-
|
16
|
-
|
17
|
-
|
24
|
+
# caution: This method might return huge results
|
25
|
+
def all_query_results
|
26
|
+
loop.reduce([]) do |ary, _|
|
27
|
+
ary.tap do |a|
|
28
|
+
a << @jobs.get_query_results(
|
29
|
+
project_id: project_id,
|
30
|
+
job_id: job_id,
|
31
|
+
page_token: (ary.last||{})["pageToken"]
|
32
|
+
)
|
33
|
+
end
|
18
34
|
|
19
|
-
|
20
|
-
|
21
|
-
|
35
|
+
if (ary.last||{})["pageToken"].nil?
|
36
|
+
break ary
|
37
|
+
else
|
38
|
+
ary
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
22
42
|
|
23
|
-
|
24
|
-
|
25
|
-
|
43
|
+
def reload
|
44
|
+
@jobs.get(project_id: project_id, job_id: job_id)
|
45
|
+
end
|
26
46
|
|
27
|
-
|
28
|
-
|
29
|
-
|
47
|
+
def project_id
|
48
|
+
job_reference["projectId"]
|
49
|
+
end
|
30
50
|
|
31
|
-
|
32
|
-
|
33
|
-
|
51
|
+
def job_id
|
52
|
+
job_reference["jobId"]
|
53
|
+
end
|
34
54
|
|
35
|
-
|
36
|
-
|
37
|
-
|
55
|
+
def query
|
56
|
+
(configuration["query"]||{})["query"]
|
57
|
+
end
|
38
58
|
|
39
|
-
|
59
|
+
def state
|
60
|
+
status["state"]
|
61
|
+
end
|
40
62
|
|
41
|
-
|
42
|
-
|
43
|
-
|
63
|
+
def errors
|
64
|
+
status["errors"]
|
65
|
+
end
|
44
66
|
|
45
|
-
|
46
|
-
|
47
|
-
|
67
|
+
private
|
68
|
+
|
69
|
+
def configuration
|
70
|
+
@resource["configuration"]
|
71
|
+
end
|
72
|
+
|
73
|
+
def status
|
74
|
+
@resource["status"]
|
75
|
+
end
|
48
76
|
|
49
|
-
|
50
|
-
|
77
|
+
def job_reference
|
78
|
+
@resource["jobReference"]
|
79
|
+
end
|
51
80
|
end
|
52
81
|
end
|
53
82
|
end
|
@@ -1,61 +1,61 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require_relative 'initializer'
|
4
|
-
require_relative 'request_executor'
|
1
|
+
require_relative 'resource'
|
5
2
|
require_relative 'job'
|
6
3
|
|
7
4
|
module Yancya
|
8
|
-
class BigQuery
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
5
|
+
class BigQuery
|
6
|
+
class Jobs < Resource
|
7
|
+
def cancel(project_id:, job_id:)
|
8
|
+
execute(
|
9
|
+
api_method: @bq.api.jobs.cancel,
|
10
|
+
parameters: { projectId: project_id, jobId: job_id }
|
11
|
+
)
|
12
|
+
end
|
13
|
+
|
14
|
+
def get(project_id:, job_id:)
|
15
|
+
resource = execute(
|
16
|
+
api_method: @bq.api.jobs.get,
|
17
|
+
parameters: { projectId: project_id, jobId: job_id }
|
18
|
+
)
|
19
|
+
|
20
|
+
BigQuery::Job.new(resource: resource, bq: @bq)
|
21
|
+
end
|
22
|
+
|
23
|
+
def get_query_results(project_id:, job_id:, page_token: nil)
|
24
|
+
execute(
|
25
|
+
api_method: @bq.api.jobs.get_query_results,
|
26
|
+
parameters: {
|
27
|
+
projectId: project_id,
|
28
|
+
jobId: job_id,
|
29
|
+
pageToken: page_token
|
30
|
+
}
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
def insert(project_id:)
|
35
|
+
# TODO: https://developers.google.com/bigquery/docs/reference/v2/jobs/insert
|
36
|
+
raise "This method is not yet working"
|
37
|
+
end
|
38
|
+
|
39
|
+
def list(project_id:)
|
40
|
+
resources = execute(
|
41
|
+
api_method: @bq.api.jobs.list,
|
42
|
+
parameters: { projectId: project_id, projection: "full" }
|
43
|
+
)
|
44
|
+
|
45
|
+
resources["jobs"].map{|job|
|
46
|
+
BigQuery::Job.new(resource: job, bq: @bq)
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
def query(project_id:, sql:)
|
51
|
+
resource = execute(
|
52
|
+
api_method: @bq.api.jobs.query,
|
53
|
+
body_object: { query: sql },
|
54
|
+
parameters: { projectId: project_id }
|
55
|
+
)
|
56
|
+
|
57
|
+
BigQuery::Job.new(resource: resource, bq: @bq)
|
58
|
+
end
|
59
59
|
end
|
60
60
|
end
|
61
61
|
end
|
@@ -3,30 +3,32 @@ require_relative 'datasets'
|
|
3
3
|
require_relative 'request_executor'
|
4
4
|
|
5
5
|
module Yancya
|
6
|
-
class BigQuery
|
7
|
-
|
6
|
+
class BigQuery
|
7
|
+
class Project
|
8
|
+
include RequestExecutor
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
10
|
+
def initialize(resource:, bq:)
|
11
|
+
@resource = resource
|
12
|
+
@jobs = BigQuery::Jobs.new(bq: bq)
|
13
|
+
@datasets = BigQuery::Datasets.new(bq: bq)
|
14
|
+
@bq = bq
|
15
|
+
end
|
15
16
|
|
16
|
-
|
17
|
-
|
18
|
-
|
17
|
+
def query(sql)
|
18
|
+
@jobs.query(project_id: project_id, sql: sql)
|
19
|
+
end
|
19
20
|
|
20
|
-
|
21
|
-
|
22
|
-
|
21
|
+
def jobs
|
22
|
+
@jobs.list(project_id: project_id)
|
23
|
+
end
|
23
24
|
|
24
|
-
|
25
|
-
|
26
|
-
|
25
|
+
def datasets
|
26
|
+
@datasets.list(project_id: project_id)
|
27
|
+
end
|
27
28
|
|
28
|
-
|
29
|
-
|
29
|
+
def project_id
|
30
|
+
@resource["projectReference"]["projectId"]
|
31
|
+
end
|
30
32
|
end
|
31
33
|
end
|
32
34
|
end
|
@@ -1,20 +1,17 @@
|
|
1
|
-
require_relative '
|
2
|
-
require_relative 'initializer'
|
3
|
-
|
1
|
+
require_relative 'resource'
|
4
2
|
require_relative 'project'
|
5
3
|
|
6
4
|
module Yancya
|
7
|
-
class BigQuery
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
api_method: @bq.api.projects.list
|
14
|
-
)
|
5
|
+
class BigQuery
|
6
|
+
class Projects < Resource
|
7
|
+
def list
|
8
|
+
resources = execute(
|
9
|
+
api_method: @bq.api.projects.list
|
10
|
+
)
|
15
11
|
|
16
|
-
|
17
|
-
|
12
|
+
resources["projects"].map do |resource|
|
13
|
+
BigQuery::Project.new(resource: resource, bq: @bq)
|
14
|
+
end
|
18
15
|
end
|
19
16
|
end
|
20
17
|
end
|
@@ -1,12 +1,14 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
1
3
|
module Yancya
|
2
4
|
module RequestExecutor
|
3
5
|
private
|
4
6
|
|
5
7
|
def response_to_json(obj)
|
6
8
|
return {} if obj.response.body.empty?
|
9
|
+
|
7
10
|
JSON.parse(obj.response.body).tap do |resource|
|
8
11
|
raise resource["error"].to_s if resource["error"]
|
9
|
-
break resource
|
10
12
|
end
|
11
13
|
end
|
12
14
|
|
@@ -1,67 +1,69 @@
|
|
1
1
|
module Yancya
|
2
|
-
class BigQuery
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
2
|
+
class BigQuery
|
3
|
+
class Table
|
4
|
+
def initialize(resource:, tables:)
|
5
|
+
@resource = resource
|
6
|
+
@tables = tables
|
7
|
+
end
|
7
8
|
|
8
|
-
|
9
|
-
|
10
|
-
|
9
|
+
def project_id
|
10
|
+
table_reference["projectId"]
|
11
|
+
end
|
11
12
|
|
12
|
-
|
13
|
-
|
14
|
-
|
13
|
+
def dataset_id
|
14
|
+
table_reference["datasetId"]
|
15
|
+
end
|
15
16
|
|
16
|
-
|
17
|
-
|
18
|
-
|
17
|
+
def table_id
|
18
|
+
table_reference["tableId"]
|
19
|
+
end
|
19
20
|
|
20
|
-
|
21
|
-
|
22
|
-
|
21
|
+
def description
|
22
|
+
@resource["description"]
|
23
|
+
end
|
23
24
|
|
24
|
-
|
25
|
-
|
26
|
-
|
25
|
+
def schema
|
26
|
+
@resource["schema"]
|
27
|
+
end
|
27
28
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
def reload
|
30
|
+
@tables.get(
|
31
|
+
project_id: project_id,
|
32
|
+
dataset_id: dataset_id,
|
33
|
+
table_id: table_id
|
34
|
+
)
|
35
|
+
end
|
35
36
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
37
|
+
def delete
|
38
|
+
@tables.delete(
|
39
|
+
project_id: project_id,
|
40
|
+
dataset_id: dataset_id,
|
41
|
+
table_id: table_id
|
42
|
+
)
|
43
|
+
end
|
43
44
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
45
|
+
def patch(table_reference:)
|
46
|
+
@tables.patch(
|
47
|
+
project_id: project_id,
|
48
|
+
dataset_id: dataset_id,
|
49
|
+
table_id: table_id,
|
50
|
+
table: table_reference
|
51
|
+
)
|
52
|
+
end
|
52
53
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
54
|
+
def update(table_reference:)
|
55
|
+
@tables.update(
|
56
|
+
project_id: project_id,
|
57
|
+
dataset_id: dataset_id,
|
58
|
+
table_id: table_id,
|
59
|
+
table: table_reference
|
60
|
+
)
|
61
|
+
end
|
61
62
|
|
62
|
-
|
63
|
-
|
64
|
-
|
63
|
+
private
|
64
|
+
def table_reference
|
65
|
+
@resource["tableReference"]
|
66
|
+
end
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
@@ -1,37 +1,39 @@
|
|
1
1
|
require_relative 'request_executor'
|
2
2
|
|
3
3
|
module Yancya
|
4
|
-
class BigQuery
|
5
|
-
|
4
|
+
class BigQuery
|
5
|
+
class Tabledata
|
6
|
+
include RequestExecutor
|
6
7
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
8
|
+
def initialize(project_id:, dataset_id:, table_id:, bq:)
|
9
|
+
@project_id = project_id
|
10
|
+
@dataset_id = dataset_id
|
11
|
+
@table_id = table_id
|
12
|
+
@bq = bq
|
13
|
+
end
|
13
14
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
15
|
+
def insert_all(rows)
|
16
|
+
execute(
|
17
|
+
api_method: @bq.api.tabledata.insert_all,
|
18
|
+
body_object: {rows: rows},
|
19
|
+
parameters: {
|
20
|
+
datasetId: @dataset_id,
|
21
|
+
projectId: @project_id,
|
22
|
+
tableId: @table_id
|
23
|
+
}
|
24
|
+
)
|
25
|
+
end
|
25
26
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
27
|
+
def list
|
28
|
+
execute(
|
29
|
+
api_method: @bq.api.tabledata.list,
|
30
|
+
parameters: {
|
31
|
+
datasetId: @dataset_id,
|
32
|
+
projectId: @project_id,
|
33
|
+
tableId: @table_id
|
34
|
+
}
|
35
|
+
)
|
36
|
+
end
|
35
37
|
end
|
36
38
|
end
|
37
39
|
end
|
@@ -1,90 +1,87 @@
|
|
1
|
-
require_relative '
|
2
|
-
require_relative 'request_executor'
|
3
|
-
|
1
|
+
require_relative 'resource'
|
4
2
|
require_relative 'table'
|
5
3
|
|
6
4
|
module Yancya
|
7
|
-
class BigQuery
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
)
|
20
|
-
end
|
5
|
+
class BigQuery
|
6
|
+
class Tables < Resource
|
7
|
+
def delete(project_id:, dataset_id:, table_id:)
|
8
|
+
execute(
|
9
|
+
api_method: @bq.api.tables.delete,
|
10
|
+
parameters: {
|
11
|
+
datasetId: dataset_id,
|
12
|
+
projectId: project_id,
|
13
|
+
tableId: table_id,
|
14
|
+
}
|
15
|
+
)
|
16
|
+
end
|
21
17
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
18
|
+
def get(project_id:, dataset_id:, table_id:)
|
19
|
+
resource = execute(
|
20
|
+
api_method: @bq.api.tables.get,
|
21
|
+
parameters: {
|
22
|
+
datasetId: dataset_id,
|
23
|
+
projectId: project_id,
|
24
|
+
tableId: table_id,
|
25
|
+
}
|
26
|
+
)
|
31
27
|
|
32
|
-
|
33
|
-
|
28
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
29
|
+
end
|
34
30
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
31
|
+
def insert(project_id:, dataset_id:, table:)
|
32
|
+
resource = execute(
|
33
|
+
api_method: @bq.api.tables.insert,
|
34
|
+
parameters: {
|
35
|
+
datasetId: dataset_id,
|
36
|
+
projectId: project_id,
|
37
|
+
},
|
38
|
+
body_object: table
|
39
|
+
)
|
44
40
|
|
45
|
-
|
46
|
-
|
41
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
42
|
+
end
|
47
43
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
44
|
+
def list(project_id:, dataset_id:)
|
45
|
+
resources = execute(
|
46
|
+
api_method: @bq.api.tables.list,
|
47
|
+
parameters: {
|
48
|
+
datasetId: dataset_id,
|
49
|
+
projectId: project_id,
|
50
|
+
}
|
51
|
+
)
|
56
52
|
|
57
|
-
|
58
|
-
|
53
|
+
(resources["tables"]||[]).map do |resource|
|
54
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
55
|
+
end
|
59
56
|
end
|
60
|
-
end
|
61
57
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
58
|
+
def patch(project_id:, dataset_id:, table_id:, table:)
|
59
|
+
resource = execute(
|
60
|
+
api_method: @bq.api.tables.patch,
|
61
|
+
parameters: {
|
62
|
+
datasetId: dataset_id,
|
63
|
+
projectId: project_id,
|
64
|
+
tableId: table_id,
|
65
|
+
},
|
66
|
+
body_object: table
|
67
|
+
)
|
72
68
|
|
73
|
-
|
74
|
-
|
69
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
70
|
+
end
|
75
71
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
72
|
+
def update(project_id:, dataset_id:, table_id:, table:)
|
73
|
+
resource = execute(
|
74
|
+
api_method: @bq.api.tables.update,
|
75
|
+
parameters: {
|
76
|
+
datasetId: dataset_id,
|
77
|
+
projectId: project_id,
|
78
|
+
tableId: table_id,
|
79
|
+
},
|
80
|
+
body_object: table
|
81
|
+
)
|
86
82
|
|
87
|
-
|
83
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
84
|
+
end
|
88
85
|
end
|
89
86
|
end
|
90
87
|
end
|
data/test/test_big_query.rb
CHANGED
@@ -147,13 +147,28 @@ class TestBigQuery < Test::Unit::TestCase
|
|
147
147
|
dataset_id: TEST_DATASET_ID,
|
148
148
|
table_id: TEST_TABLE_ID,
|
149
149
|
table: {
|
150
|
-
schema: schema.tap { |hash| hash["fields"] << { name: "hoge", type: "STRING" }
|
150
|
+
schema: schema.tap { |hash| hash["fields"] << { name: "hoge", type: "STRING" } }
|
151
151
|
}
|
152
152
|
)
|
153
153
|
|
154
154
|
assert { table.schema["fields"].map { |field| field["name"] }.include? "hoge" }
|
155
155
|
end
|
156
156
|
|
157
|
+
test "all_query_results" do
|
158
|
+
job = @bq.jobs.query(project_id: @project_id, sql: <<-SQL)
|
159
|
+
SELECT 2 as a, 4 as b, 6 as c
|
160
|
+
SQL
|
161
|
+
|
162
|
+
begin
|
163
|
+
results = job.all_query_results
|
164
|
+
sleep(1)
|
165
|
+
end until results.first["jobComplete"]
|
166
|
+
|
167
|
+
assert do
|
168
|
+
"2,4,6" == results.first["rows"].map{|row| row["f"].map{|col| col["v"]}.join(",")}.first
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
157
172
|
private
|
158
173
|
|
159
174
|
def create_table
|
@@ -167,10 +182,14 @@ class TestBigQuery < Test::Unit::TestCase
|
|
167
182
|
def execute_query(sql)
|
168
183
|
job = @bq.jobs.query(project_id: @project_id, sql: sql)
|
169
184
|
|
170
|
-
|
185
|
+
loop.each_with_index do |_, index|
|
186
|
+
raise 'Job timeout' if index > 60
|
187
|
+
|
171
188
|
result = job.query_results
|
172
|
-
end until result["jobComplete"]
|
173
189
|
|
174
|
-
|
190
|
+
break result if result["jobComplete"]
|
191
|
+
|
192
|
+
sleep(1)
|
193
|
+
end
|
175
194
|
end
|
176
195
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: yancya-big_query
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yancya
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-07-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: google-api-client
|
@@ -86,12 +86,12 @@ files:
|
|
86
86
|
- lib/yancya/big_query.rb
|
87
87
|
- lib/yancya/big_query/dataset.rb
|
88
88
|
- lib/yancya/big_query/datasets.rb
|
89
|
-
- lib/yancya/big_query/initializer.rb
|
90
89
|
- lib/yancya/big_query/job.rb
|
91
90
|
- lib/yancya/big_query/jobs.rb
|
92
91
|
- lib/yancya/big_query/project.rb
|
93
92
|
- lib/yancya/big_query/projects.rb
|
94
93
|
- lib/yancya/big_query/request_executor.rb
|
94
|
+
- lib/yancya/big_query/resource.rb
|
95
95
|
- lib/yancya/big_query/table.rb
|
96
96
|
- lib/yancya/big_query/tabledata.rb
|
97
97
|
- lib/yancya/big_query/tables.rb
|