yancya-big_query 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -6
- data/lib/yancya/big_query/dataset.rb +22 -20
- data/lib/yancya/big_query/datasets.rb +48 -51
- data/lib/yancya/big_query/job.rb +67 -38
- data/lib/yancya/big_query/jobs.rb +55 -55
- data/lib/yancya/big_query/project.rb +21 -19
- data/lib/yancya/big_query/projects.rb +10 -13
- data/lib/yancya/big_query/request_executor.rb +3 -1
- data/lib/yancya/big_query/resource.rb +13 -0
- data/lib/yancya/big_query/table.rb +55 -53
- data/lib/yancya/big_query/tabledata.rb +30 -28
- data/lib/yancya/big_query/tables.rb +70 -73
- data/lib/yancya/big_query/version.rb +1 -1
- data/test/test_big_query.rb +23 -4
- metadata +3 -3
- data/lib/yancya/big_query/initializer.rb +0 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 06183ec4fd4344c825bbc907b5e7ac62807cf921
|
4
|
+
data.tar.gz: 7bef7e5f244f514cb418503ab44f57448c5c1ff0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2b25247ed836fc651980505d1b89270d91bd9b1207e91bad8f258d9e945419381001365b1cd5ef2bc45af96929bc1f6a5a4a7521658c073fd36e5f12b6773e23
|
7
|
+
data.tar.gz: a9fd32c9c007d101e9b221fee9e35ff5a78f7c6a3c49adac08e868ba704dc2a49a464e2df86e62dde8dd9343a4eacc41617a8a9cbb3d184db43034d357ed10cb
|
data/README.md
CHANGED
@@ -8,22 +8,20 @@ Google BigQuery API client library
|
|
8
8
|
|
9
9
|
Add this line to your application's Gemfile:
|
10
10
|
|
11
|
-
gem 'big_query'
|
11
|
+
gem 'yancya-big_query'
|
12
12
|
|
13
13
|
And then execute:
|
14
14
|
|
15
15
|
$ bundle
|
16
16
|
|
17
|
-
Or
|
17
|
+
Or install it yourself as:
|
18
18
|
|
19
|
-
$
|
20
|
-
$ cd big_query
|
21
|
-
$ rake install
|
19
|
+
$ gem install yancya-big_query
|
22
20
|
|
23
21
|
## Usage
|
24
22
|
|
25
23
|
```rb
|
26
|
-
bq = BigQuery.new(
|
24
|
+
bq = Yancya::BigQuery.new(
|
27
25
|
key_path: "path_to_secret_key"
|
28
26
|
issuer: "mail@address"
|
29
27
|
)
|
@@ -1,31 +1,33 @@
|
|
1
1
|
require_relative 'tables'
|
2
2
|
|
3
3
|
module Yancya
|
4
|
-
class BigQuery
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
4
|
+
class BigQuery
|
5
|
+
class Dataset
|
6
|
+
def initialize(resource:, bq:)
|
7
|
+
@resource = resource
|
8
|
+
@tables = BigQuery::Tables.new(bq: bq)
|
9
|
+
end
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
11
|
+
def tables
|
12
|
+
@tables.list(
|
13
|
+
project_id: project_id,
|
14
|
+
dataset_id: dataset_id
|
15
|
+
)
|
16
|
+
end
|
16
17
|
|
17
|
-
|
18
|
-
|
19
|
-
|
18
|
+
def project_id
|
19
|
+
dataset_reference["projectId"]
|
20
|
+
end
|
20
21
|
|
21
|
-
|
22
|
-
|
23
|
-
|
22
|
+
def dataset_id
|
23
|
+
dataset_reference["datasetId"]
|
24
|
+
end
|
24
25
|
|
25
|
-
|
26
|
+
private
|
26
27
|
|
27
|
-
|
28
|
-
|
28
|
+
def dataset_reference
|
29
|
+
@resource["datasetReference"]
|
30
|
+
end
|
29
31
|
end
|
30
32
|
end
|
31
33
|
end
|
@@ -1,63 +1,60 @@
|
|
1
|
-
require_relative '
|
2
|
-
require_relative 'request_executor'
|
3
|
-
|
1
|
+
require_relative 'resource'
|
4
2
|
require_relative 'dataset'
|
5
3
|
|
6
4
|
module Yancya
|
7
|
-
class BigQuery
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
)
|
16
|
-
end
|
17
|
-
|
18
|
-
def get(project_id:, dataset_id:)
|
19
|
-
execute(
|
20
|
-
api_method: @bq.api.datasets.get,
|
21
|
-
parameters: {projectId: project_id, datasetId: dataset_id}
|
22
|
-
)
|
23
|
-
end
|
5
|
+
class BigQuery
|
6
|
+
class Datasets < Resource
|
7
|
+
def delete(project_id:, dataset_id:)
|
8
|
+
execute(
|
9
|
+
api_method: @bq.api.datasets.delete,
|
10
|
+
parameters: {projectId: project_id, datasetId: dataset_id, deleteContents: true}
|
11
|
+
)
|
12
|
+
end
|
24
13
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
end
|
14
|
+
def get(project_id:, dataset_id:)
|
15
|
+
execute(
|
16
|
+
api_method: @bq.api.datasets.get,
|
17
|
+
parameters: {projectId: project_id, datasetId: dataset_id}
|
18
|
+
)
|
19
|
+
end
|
32
20
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
21
|
+
def insert(project_id:, dataset:)
|
22
|
+
execute(
|
23
|
+
api_method: @bq.api.datasets.insert,
|
24
|
+
parameters: {projectId: project_id},
|
25
|
+
body_object: dataset
|
26
|
+
)
|
27
|
+
end
|
38
28
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
29
|
+
def list(project_id:)
|
30
|
+
resources = execute(
|
31
|
+
api_method: @bq.api.datasets.list,
|
32
|
+
parameters: {projectId: project_id}
|
43
33
|
)
|
44
|
-
}
|
45
|
-
end
|
46
34
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
35
|
+
resources["datasets"].map{ |resource|
|
36
|
+
Yancya::BigQuery::Dataset.new(
|
37
|
+
resource: resource,
|
38
|
+
bq: @bq
|
39
|
+
)
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
def patch(project_id:, dataset_id:, dataset:)
|
44
|
+
execute(
|
45
|
+
api_method: @bq.api.datasets.patch,
|
46
|
+
parameters: {projectId: project_id, datasetId: dataset_id},
|
47
|
+
body_object: dataset
|
48
|
+
)
|
49
|
+
end
|
54
50
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
51
|
+
def update(project_id:, dataset_id:, dataset:)
|
52
|
+
execute(
|
53
|
+
api_method: @bq.api.datasets.update,
|
54
|
+
parameters: {projectId: project_id, datasetId: dataset_id},
|
55
|
+
body_object: dataset
|
56
|
+
)
|
57
|
+
end
|
61
58
|
end
|
62
59
|
end
|
63
60
|
end
|
data/lib/yancya/big_query/job.rb
CHANGED
@@ -1,53 +1,82 @@
|
|
1
1
|
module Yancya
|
2
|
-
class BigQuery
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
2
|
+
class BigQuery
|
3
|
+
class Job
|
4
|
+
def initialize(resource:, bq:)
|
5
|
+
@resource = resource
|
6
|
+
@jobs = BigQuery::Jobs.new(bq: bq)
|
7
|
+
end
|
7
8
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
9
|
+
def query_results
|
10
|
+
@jobs.get_query_results(
|
11
|
+
project_id: project_id,
|
12
|
+
job_id: job_id
|
13
|
+
).tap do |qr|
|
14
|
+
if qr["pageToken"]
|
15
|
+
warn <<-TEXT
|
16
|
+
This is a slice in all results.
|
17
|
+
You have not took rest results.
|
18
|
+
Perhaps all_query_results is useful.
|
19
|
+
TEXT
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
14
23
|
|
15
|
-
|
16
|
-
|
17
|
-
|
24
|
+
# caution: This method might return huge results
|
25
|
+
def all_query_results
|
26
|
+
loop.reduce([]) do |ary, _|
|
27
|
+
ary.tap do |a|
|
28
|
+
a << @jobs.get_query_results(
|
29
|
+
project_id: project_id,
|
30
|
+
job_id: job_id,
|
31
|
+
page_token: (ary.last||{})["pageToken"]
|
32
|
+
)
|
33
|
+
end
|
18
34
|
|
19
|
-
|
20
|
-
|
21
|
-
|
35
|
+
if (ary.last||{})["pageToken"].nil?
|
36
|
+
break ary
|
37
|
+
else
|
38
|
+
ary
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
22
42
|
|
23
|
-
|
24
|
-
|
25
|
-
|
43
|
+
def reload
|
44
|
+
@jobs.get(project_id: project_id, job_id: job_id)
|
45
|
+
end
|
26
46
|
|
27
|
-
|
28
|
-
|
29
|
-
|
47
|
+
def project_id
|
48
|
+
job_reference["projectId"]
|
49
|
+
end
|
30
50
|
|
31
|
-
|
32
|
-
|
33
|
-
|
51
|
+
def job_id
|
52
|
+
job_reference["jobId"]
|
53
|
+
end
|
34
54
|
|
35
|
-
|
36
|
-
|
37
|
-
|
55
|
+
def query
|
56
|
+
(configuration["query"]||{})["query"]
|
57
|
+
end
|
38
58
|
|
39
|
-
|
59
|
+
def state
|
60
|
+
status["state"]
|
61
|
+
end
|
40
62
|
|
41
|
-
|
42
|
-
|
43
|
-
|
63
|
+
def errors
|
64
|
+
status["errors"]
|
65
|
+
end
|
44
66
|
|
45
|
-
|
46
|
-
|
47
|
-
|
67
|
+
private
|
68
|
+
|
69
|
+
def configuration
|
70
|
+
@resource["configuration"]
|
71
|
+
end
|
72
|
+
|
73
|
+
def status
|
74
|
+
@resource["status"]
|
75
|
+
end
|
48
76
|
|
49
|
-
|
50
|
-
|
77
|
+
def job_reference
|
78
|
+
@resource["jobReference"]
|
79
|
+
end
|
51
80
|
end
|
52
81
|
end
|
53
82
|
end
|
@@ -1,61 +1,61 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require_relative 'initializer'
|
4
|
-
require_relative 'request_executor'
|
1
|
+
require_relative 'resource'
|
5
2
|
require_relative 'job'
|
6
3
|
|
7
4
|
module Yancya
|
8
|
-
class BigQuery
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
5
|
+
class BigQuery
|
6
|
+
class Jobs < Resource
|
7
|
+
def cancel(project_id:, job_id:)
|
8
|
+
execute(
|
9
|
+
api_method: @bq.api.jobs.cancel,
|
10
|
+
parameters: { projectId: project_id, jobId: job_id }
|
11
|
+
)
|
12
|
+
end
|
13
|
+
|
14
|
+
def get(project_id:, job_id:)
|
15
|
+
resource = execute(
|
16
|
+
api_method: @bq.api.jobs.get,
|
17
|
+
parameters: { projectId: project_id, jobId: job_id }
|
18
|
+
)
|
19
|
+
|
20
|
+
BigQuery::Job.new(resource: resource, bq: @bq)
|
21
|
+
end
|
22
|
+
|
23
|
+
def get_query_results(project_id:, job_id:, page_token: nil)
|
24
|
+
execute(
|
25
|
+
api_method: @bq.api.jobs.get_query_results,
|
26
|
+
parameters: {
|
27
|
+
projectId: project_id,
|
28
|
+
jobId: job_id,
|
29
|
+
pageToken: page_token
|
30
|
+
}
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
def insert(project_id:)
|
35
|
+
# TODO: https://developers.google.com/bigquery/docs/reference/v2/jobs/insert
|
36
|
+
raise "This method is not yet working"
|
37
|
+
end
|
38
|
+
|
39
|
+
def list(project_id:)
|
40
|
+
resources = execute(
|
41
|
+
api_method: @bq.api.jobs.list,
|
42
|
+
parameters: { projectId: project_id, projection: "full" }
|
43
|
+
)
|
44
|
+
|
45
|
+
resources["jobs"].map{|job|
|
46
|
+
BigQuery::Job.new(resource: job, bq: @bq)
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
def query(project_id:, sql:)
|
51
|
+
resource = execute(
|
52
|
+
api_method: @bq.api.jobs.query,
|
53
|
+
body_object: { query: sql },
|
54
|
+
parameters: { projectId: project_id }
|
55
|
+
)
|
56
|
+
|
57
|
+
BigQuery::Job.new(resource: resource, bq: @bq)
|
58
|
+
end
|
59
59
|
end
|
60
60
|
end
|
61
61
|
end
|
@@ -3,30 +3,32 @@ require_relative 'datasets'
|
|
3
3
|
require_relative 'request_executor'
|
4
4
|
|
5
5
|
module Yancya
|
6
|
-
class BigQuery
|
7
|
-
|
6
|
+
class BigQuery
|
7
|
+
class Project
|
8
|
+
include RequestExecutor
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
10
|
+
def initialize(resource:, bq:)
|
11
|
+
@resource = resource
|
12
|
+
@jobs = BigQuery::Jobs.new(bq: bq)
|
13
|
+
@datasets = BigQuery::Datasets.new(bq: bq)
|
14
|
+
@bq = bq
|
15
|
+
end
|
15
16
|
|
16
|
-
|
17
|
-
|
18
|
-
|
17
|
+
def query(sql)
|
18
|
+
@jobs.query(project_id: project_id, sql: sql)
|
19
|
+
end
|
19
20
|
|
20
|
-
|
21
|
-
|
22
|
-
|
21
|
+
def jobs
|
22
|
+
@jobs.list(project_id: project_id)
|
23
|
+
end
|
23
24
|
|
24
|
-
|
25
|
-
|
26
|
-
|
25
|
+
def datasets
|
26
|
+
@datasets.list(project_id: project_id)
|
27
|
+
end
|
27
28
|
|
28
|
-
|
29
|
-
|
29
|
+
def project_id
|
30
|
+
@resource["projectReference"]["projectId"]
|
31
|
+
end
|
30
32
|
end
|
31
33
|
end
|
32
34
|
end
|
@@ -1,20 +1,17 @@
|
|
1
|
-
require_relative '
|
2
|
-
require_relative 'initializer'
|
3
|
-
|
1
|
+
require_relative 'resource'
|
4
2
|
require_relative 'project'
|
5
3
|
|
6
4
|
module Yancya
|
7
|
-
class BigQuery
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
api_method: @bq.api.projects.list
|
14
|
-
)
|
5
|
+
class BigQuery
|
6
|
+
class Projects < Resource
|
7
|
+
def list
|
8
|
+
resources = execute(
|
9
|
+
api_method: @bq.api.projects.list
|
10
|
+
)
|
15
11
|
|
16
|
-
|
17
|
-
|
12
|
+
resources["projects"].map do |resource|
|
13
|
+
BigQuery::Project.new(resource: resource, bq: @bq)
|
14
|
+
end
|
18
15
|
end
|
19
16
|
end
|
20
17
|
end
|
@@ -1,12 +1,14 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
1
3
|
module Yancya
|
2
4
|
module RequestExecutor
|
3
5
|
private
|
4
6
|
|
5
7
|
def response_to_json(obj)
|
6
8
|
return {} if obj.response.body.empty?
|
9
|
+
|
7
10
|
JSON.parse(obj.response.body).tap do |resource|
|
8
11
|
raise resource["error"].to_s if resource["error"]
|
9
|
-
break resource
|
10
12
|
end
|
11
13
|
end
|
12
14
|
|
@@ -1,67 +1,69 @@
|
|
1
1
|
module Yancya
|
2
|
-
class BigQuery
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
2
|
+
class BigQuery
|
3
|
+
class Table
|
4
|
+
def initialize(resource:, tables:)
|
5
|
+
@resource = resource
|
6
|
+
@tables = tables
|
7
|
+
end
|
7
8
|
|
8
|
-
|
9
|
-
|
10
|
-
|
9
|
+
def project_id
|
10
|
+
table_reference["projectId"]
|
11
|
+
end
|
11
12
|
|
12
|
-
|
13
|
-
|
14
|
-
|
13
|
+
def dataset_id
|
14
|
+
table_reference["datasetId"]
|
15
|
+
end
|
15
16
|
|
16
|
-
|
17
|
-
|
18
|
-
|
17
|
+
def table_id
|
18
|
+
table_reference["tableId"]
|
19
|
+
end
|
19
20
|
|
20
|
-
|
21
|
-
|
22
|
-
|
21
|
+
def description
|
22
|
+
@resource["description"]
|
23
|
+
end
|
23
24
|
|
24
|
-
|
25
|
-
|
26
|
-
|
25
|
+
def schema
|
26
|
+
@resource["schema"]
|
27
|
+
end
|
27
28
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
def reload
|
30
|
+
@tables.get(
|
31
|
+
project_id: project_id,
|
32
|
+
dataset_id: dataset_id,
|
33
|
+
table_id: table_id
|
34
|
+
)
|
35
|
+
end
|
35
36
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
37
|
+
def delete
|
38
|
+
@tables.delete(
|
39
|
+
project_id: project_id,
|
40
|
+
dataset_id: dataset_id,
|
41
|
+
table_id: table_id
|
42
|
+
)
|
43
|
+
end
|
43
44
|
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
45
|
+
def patch(table_reference:)
|
46
|
+
@tables.patch(
|
47
|
+
project_id: project_id,
|
48
|
+
dataset_id: dataset_id,
|
49
|
+
table_id: table_id,
|
50
|
+
table: table_reference
|
51
|
+
)
|
52
|
+
end
|
52
53
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
54
|
+
def update(table_reference:)
|
55
|
+
@tables.update(
|
56
|
+
project_id: project_id,
|
57
|
+
dataset_id: dataset_id,
|
58
|
+
table_id: table_id,
|
59
|
+
table: table_reference
|
60
|
+
)
|
61
|
+
end
|
61
62
|
|
62
|
-
|
63
|
-
|
64
|
-
|
63
|
+
private
|
64
|
+
def table_reference
|
65
|
+
@resource["tableReference"]
|
66
|
+
end
|
65
67
|
end
|
66
68
|
end
|
67
69
|
end
|
@@ -1,37 +1,39 @@
|
|
1
1
|
require_relative 'request_executor'
|
2
2
|
|
3
3
|
module Yancya
|
4
|
-
class BigQuery
|
5
|
-
|
4
|
+
class BigQuery
|
5
|
+
class Tabledata
|
6
|
+
include RequestExecutor
|
6
7
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
8
|
+
def initialize(project_id:, dataset_id:, table_id:, bq:)
|
9
|
+
@project_id = project_id
|
10
|
+
@dataset_id = dataset_id
|
11
|
+
@table_id = table_id
|
12
|
+
@bq = bq
|
13
|
+
end
|
13
14
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
15
|
+
def insert_all(rows)
|
16
|
+
execute(
|
17
|
+
api_method: @bq.api.tabledata.insert_all,
|
18
|
+
body_object: {rows: rows},
|
19
|
+
parameters: {
|
20
|
+
datasetId: @dataset_id,
|
21
|
+
projectId: @project_id,
|
22
|
+
tableId: @table_id
|
23
|
+
}
|
24
|
+
)
|
25
|
+
end
|
25
26
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
27
|
+
def list
|
28
|
+
execute(
|
29
|
+
api_method: @bq.api.tabledata.list,
|
30
|
+
parameters: {
|
31
|
+
datasetId: @dataset_id,
|
32
|
+
projectId: @project_id,
|
33
|
+
tableId: @table_id
|
34
|
+
}
|
35
|
+
)
|
36
|
+
end
|
35
37
|
end
|
36
38
|
end
|
37
39
|
end
|
@@ -1,90 +1,87 @@
|
|
1
|
-
require_relative '
|
2
|
-
require_relative 'request_executor'
|
3
|
-
|
1
|
+
require_relative 'resource'
|
4
2
|
require_relative 'table'
|
5
3
|
|
6
4
|
module Yancya
|
7
|
-
class BigQuery
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
)
|
20
|
-
end
|
5
|
+
class BigQuery
|
6
|
+
class Tables < Resource
|
7
|
+
def delete(project_id:, dataset_id:, table_id:)
|
8
|
+
execute(
|
9
|
+
api_method: @bq.api.tables.delete,
|
10
|
+
parameters: {
|
11
|
+
datasetId: dataset_id,
|
12
|
+
projectId: project_id,
|
13
|
+
tableId: table_id,
|
14
|
+
}
|
15
|
+
)
|
16
|
+
end
|
21
17
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
18
|
+
def get(project_id:, dataset_id:, table_id:)
|
19
|
+
resource = execute(
|
20
|
+
api_method: @bq.api.tables.get,
|
21
|
+
parameters: {
|
22
|
+
datasetId: dataset_id,
|
23
|
+
projectId: project_id,
|
24
|
+
tableId: table_id,
|
25
|
+
}
|
26
|
+
)
|
31
27
|
|
32
|
-
|
33
|
-
|
28
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
29
|
+
end
|
34
30
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
31
|
+
def insert(project_id:, dataset_id:, table:)
|
32
|
+
resource = execute(
|
33
|
+
api_method: @bq.api.tables.insert,
|
34
|
+
parameters: {
|
35
|
+
datasetId: dataset_id,
|
36
|
+
projectId: project_id,
|
37
|
+
},
|
38
|
+
body_object: table
|
39
|
+
)
|
44
40
|
|
45
|
-
|
46
|
-
|
41
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
42
|
+
end
|
47
43
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
44
|
+
def list(project_id:, dataset_id:)
|
45
|
+
resources = execute(
|
46
|
+
api_method: @bq.api.tables.list,
|
47
|
+
parameters: {
|
48
|
+
datasetId: dataset_id,
|
49
|
+
projectId: project_id,
|
50
|
+
}
|
51
|
+
)
|
56
52
|
|
57
|
-
|
58
|
-
|
53
|
+
(resources["tables"]||[]).map do |resource|
|
54
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
55
|
+
end
|
59
56
|
end
|
60
|
-
end
|
61
57
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
58
|
+
def patch(project_id:, dataset_id:, table_id:, table:)
|
59
|
+
resource = execute(
|
60
|
+
api_method: @bq.api.tables.patch,
|
61
|
+
parameters: {
|
62
|
+
datasetId: dataset_id,
|
63
|
+
projectId: project_id,
|
64
|
+
tableId: table_id,
|
65
|
+
},
|
66
|
+
body_object: table
|
67
|
+
)
|
72
68
|
|
73
|
-
|
74
|
-
|
69
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
70
|
+
end
|
75
71
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
72
|
+
def update(project_id:, dataset_id:, table_id:, table:)
|
73
|
+
resource = execute(
|
74
|
+
api_method: @bq.api.tables.update,
|
75
|
+
parameters: {
|
76
|
+
datasetId: dataset_id,
|
77
|
+
projectId: project_id,
|
78
|
+
tableId: table_id,
|
79
|
+
},
|
80
|
+
body_object: table
|
81
|
+
)
|
86
82
|
|
87
|
-
|
83
|
+
BigQuery::Table.new(resource: resource, tables: self)
|
84
|
+
end
|
88
85
|
end
|
89
86
|
end
|
90
87
|
end
|
data/test/test_big_query.rb
CHANGED
@@ -147,13 +147,28 @@ class TestBigQuery < Test::Unit::TestCase
|
|
147
147
|
dataset_id: TEST_DATASET_ID,
|
148
148
|
table_id: TEST_TABLE_ID,
|
149
149
|
table: {
|
150
|
-
schema: schema.tap { |hash| hash["fields"] << { name: "hoge", type: "STRING" }
|
150
|
+
schema: schema.tap { |hash| hash["fields"] << { name: "hoge", type: "STRING" } }
|
151
151
|
}
|
152
152
|
)
|
153
153
|
|
154
154
|
assert { table.schema["fields"].map { |field| field["name"] }.include? "hoge" }
|
155
155
|
end
|
156
156
|
|
157
|
+
test "all_query_results" do
|
158
|
+
job = @bq.jobs.query(project_id: @project_id, sql: <<-SQL)
|
159
|
+
SELECT 2 as a, 4 as b, 6 as c
|
160
|
+
SQL
|
161
|
+
|
162
|
+
begin
|
163
|
+
results = job.all_query_results
|
164
|
+
sleep(1)
|
165
|
+
end until results.first["jobComplete"]
|
166
|
+
|
167
|
+
assert do
|
168
|
+
"2,4,6" == results.first["rows"].map{|row| row["f"].map{|col| col["v"]}.join(",")}.first
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
157
172
|
private
|
158
173
|
|
159
174
|
def create_table
|
@@ -167,10 +182,14 @@ class TestBigQuery < Test::Unit::TestCase
|
|
167
182
|
def execute_query(sql)
|
168
183
|
job = @bq.jobs.query(project_id: @project_id, sql: sql)
|
169
184
|
|
170
|
-
|
185
|
+
loop.each_with_index do |_, index|
|
186
|
+
raise 'Job timeout' if index > 60
|
187
|
+
|
171
188
|
result = job.query_results
|
172
|
-
end until result["jobComplete"]
|
173
189
|
|
174
|
-
|
190
|
+
break result if result["jobComplete"]
|
191
|
+
|
192
|
+
sleep(1)
|
193
|
+
end
|
175
194
|
end
|
176
195
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: yancya-big_query
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yancya
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-07-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: google-api-client
|
@@ -86,12 +86,12 @@ files:
|
|
86
86
|
- lib/yancya/big_query.rb
|
87
87
|
- lib/yancya/big_query/dataset.rb
|
88
88
|
- lib/yancya/big_query/datasets.rb
|
89
|
-
- lib/yancya/big_query/initializer.rb
|
90
89
|
- lib/yancya/big_query/job.rb
|
91
90
|
- lib/yancya/big_query/jobs.rb
|
92
91
|
- lib/yancya/big_query/project.rb
|
93
92
|
- lib/yancya/big_query/projects.rb
|
94
93
|
- lib/yancya/big_query/request_executor.rb
|
94
|
+
- lib/yancya/big_query/resource.rb
|
95
95
|
- lib/yancya/big_query/table.rb
|
96
96
|
- lib/yancya/big_query/tabledata.rb
|
97
97
|
- lib/yancya/big_query/tables.rb
|