canvas_data_client 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.rspec +2 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +44 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +7 -0
- data/canvas_data_client.gemspec +28 -0
- data/lib/canvas_data_client.rb +6 -0
- data/lib/canvas_data_client/client.rb +72 -0
- data/lib/canvas_data_client/helpers.rb +5 -0
- data/lib/canvas_data_client/helpers/csv_helper.rb +78 -0
- data/lib/canvas_data_client/helpers/hmac_helper.rb +42 -0
- data/lib/canvas_data_client/helpers/time_helper.rb +6 -0
- data/lib/canvas_data_client/version.rb +3 -0
- metadata +147 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: d190d87207c61e7aa4615e38b96418c598ff1cee
|
4
|
+
data.tar.gz: 77547c95bd694534b07ddc326434223091957412
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 654c8b81bdb6a172068b5d98dd44da5ed3398566bab9363d82d104063d417f0317931124f24cbfe407aef3e187cfe78ce210b91c6e39e7bdc776134c5bbddad3
|
7
|
+
data.tar.gz: 8e210a9a12486f711c0b995d4d89f9f1aa54cb5a4a1c96b7549f51baf0f1e0bd1ca60e699bb53a0e97a3b14d0b7092ed691a36f9fccfc9785ac3ae5db2e9a735
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Ben Young
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
# CanvasDataClient
|
2
|
+
|
3
|
+
This gem is meant to provide an easy-to-use ruby client wrapping the [Canvas](https://canvaslms.com) Hosted Data API.
|
4
|
+
|
5
|
+
It calculates and attaches HMAC signatures for each request, and returns the parsed JSON response.
|
6
|
+
|
7
|
+
## Usage
|
8
|
+
|
9
|
+
The client can be initialized as such:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
client = CanvasDataClient::Client.new(api_key, api_secret)
|
13
|
+
```
|
14
|
+
|
15
|
+
As much as possible, the methods available in the client match those given in the [API docs](https://portal.inshosteddata.com/docs/api)
|
16
|
+
|
17
|
+
List of methods:
|
18
|
+
|
19
|
+
```ruby
|
20
|
+
client.latest_files # GET /api/account/(:accountId|self)/file/latest
|
21
|
+
client.dumps # GET /api/account/(:accountId|self)/dump
|
22
|
+
client.dump('b349ad95-4839-48f3-b763-ec555fc2f42f') # GET /api/account/(:accountId|self)/file/byDump/:dumpId
|
23
|
+
client.tables('course_dim') # GET /api/account/(:accountId|self)/file/byTable/:tableName
|
24
|
+
client.schemas # GET /api/schema
|
25
|
+
client.latest_schema # GET /api/schema/latest
|
26
|
+
client.schema('1.0.0') # GET /api/schema/:version
|
27
|
+
|
28
|
+
# Downloads a table file from the specified dump (or all files for a table should the dump have multiple)
|
29
|
+
# and writes them to a single CSV file. This also includes a CSV header row of the column names
|
30
|
+
# pulled from the schema definition
|
31
|
+
client.download_to_csv_file(dump_id: 'b349ad95-4839-48f3-b763-ec555fc2f42f', table: 'requests', path: '/path/to/output_file.csv')
|
32
|
+
|
33
|
+
# Same as #download_to_csv_file, but uses the latest dump available
|
34
|
+
client.download_latest_to_csv_file(table: 'requests', path: '/path/to/output_file.csv')
|
35
|
+
```
|
36
|
+
|
37
|
+
## Source
|
38
|
+
|
39
|
+
Originally from a partners GitHub at https://github.com/ben-y/canvas_data_client.
|
40
|
+
|
41
|
+
|
42
|
+
## License
|
43
|
+
|
44
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "canvas_data_client"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'canvas_data_client/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "canvas_data_client"
|
8
|
+
spec.version = CanvasDataClient::VERSION
|
9
|
+
spec.authors = ["Calvin Kern", "pseng"]
|
10
|
+
spec.email = ["ckern@instructure.com", "pseng@instructure.com"]
|
11
|
+
|
12
|
+
spec.summary = "Wraps the endpoints provided by Canvas Data."
|
13
|
+
spec.homepage = "http://instructure.com"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
|
+
spec.bindir = "exe"
|
18
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.10"
|
22
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
+
spec.add_development_dependency "rspec"
|
24
|
+
spec.add_development_dependency "pry"
|
25
|
+
|
26
|
+
spec.add_runtime_dependency "rest-client", "~> 1.8.0"
|
27
|
+
spec.add_runtime_dependency 'open_uri_redirections', '~> 0.2.1'
|
28
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'rest-client'
|
2
|
+
|
3
|
+
module CanvasDataClient
|
4
|
+
class Client
|
5
|
+
include CanvasDataClient::Helpers::HMACHelper
|
6
|
+
include CanvasDataClient::Helpers::CsvHelper
|
7
|
+
|
8
|
+
attr_accessor :key, :secret, :subdomain, :account, :logger
|
9
|
+
|
10
|
+
def initialize(key, secret, opts = {})
|
11
|
+
self.key = key
|
12
|
+
self.secret = secret
|
13
|
+
self.subdomain = opts[:subdomain] || 'portal'
|
14
|
+
self.account = opts[:account] || 'self'
|
15
|
+
self.logger = Logger.new(STDOUT)
|
16
|
+
end
|
17
|
+
|
18
|
+
def domain
|
19
|
+
"https://#{subdomain}.inshosteddata.com"
|
20
|
+
end
|
21
|
+
|
22
|
+
def latest_files
|
23
|
+
json_request "#{domain}/api/account/#{account}/file/latest"
|
24
|
+
end
|
25
|
+
alias_method :latest, :latest_files
|
26
|
+
|
27
|
+
def dumps
|
28
|
+
paginated_request "#{domain}/api/account/#{account}/dump?after=%s"
|
29
|
+
end
|
30
|
+
|
31
|
+
def dump(dump_id)
|
32
|
+
json_request "#{domain}/api/account/#{account}/file/byDump/#{dump_id}"
|
33
|
+
end
|
34
|
+
|
35
|
+
def tables(table)
|
36
|
+
paginated_request "#{domain}/api/account/#{account}/file/byTable/#{table}?after=%s"
|
37
|
+
end
|
38
|
+
|
39
|
+
def schemas
|
40
|
+
json_request "#{domain}/api/schema"
|
41
|
+
end
|
42
|
+
|
43
|
+
def latest_schema
|
44
|
+
json_request "#{domain}/api/schema/latest"
|
45
|
+
end
|
46
|
+
|
47
|
+
def schema(version)
|
48
|
+
json_request "#{domain}/api/schema/#{version}"
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
def json_request(path, method = 'get')
|
53
|
+
resp = RestClient.get path, headers(key, secret, { path: path, method: method })
|
54
|
+
JSON.parse resp
|
55
|
+
end
|
56
|
+
|
57
|
+
def paginated_request(path)
|
58
|
+
received = []
|
59
|
+
sequence = '0'
|
60
|
+
loop do
|
61
|
+
resp = json_request(path % sequence)
|
62
|
+
resp = resp['history'] if resp.is_a?(Hash)
|
63
|
+
resp.sort_by! { |h| h['sequence'] }
|
64
|
+
received += resp
|
65
|
+
break if resp.length < 50
|
66
|
+
sequence = resp.last['sequence']
|
67
|
+
end
|
68
|
+
received
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require 'open-uri'
|
3
|
+
require 'open_uri_redirections'
|
4
|
+
|
5
|
+
module CanvasDataClient::Helpers::CsvHelper
|
6
|
+
class TableNotPresentError < StandardError; end
|
7
|
+
|
8
|
+
def download_to_csv_file(dump_id:, table:, path:)
|
9
|
+
dump_definition = dump(dump_id)
|
10
|
+
schema_definition = schema(dump_definition['schemaVersion'])
|
11
|
+
raise TableNotPresentError.new("Table #{table} not present in dump #{dump_id}") unless dump_definition['artifactsByTable'][table]
|
12
|
+
|
13
|
+
csv = CSV.open(path, 'w')
|
14
|
+
columns = table_headers(schema_definition, table)
|
15
|
+
csv << columns
|
16
|
+
|
17
|
+
Dir.mktmpdir do |dir|
|
18
|
+
dump_definition['artifactsByTable'][table]['files'].each do |file_mapping|
|
19
|
+
renew_urls(dump_id, table, dump_definition['artifactsByTable'][table]['files']) if url_expired?(file_mapping['url'])
|
20
|
+
logger.info("Downloading table file: #{file_mapping['filename']}")
|
21
|
+
file_path = download_raw_file(file_mapping, dir)
|
22
|
+
logger.info("Processing table file: #{file_mapping['filename']}")
|
23
|
+
File.foreach(file_path) do |row|
|
24
|
+
split_row = row.gsub(/\n/, '').split(/\t/)
|
25
|
+
split_row.fill(nil, split_row.length...columns.length) if split_row.length < columns.length
|
26
|
+
csv << split_row.map { |col| col == '\\N' ? nil : col }
|
27
|
+
end
|
28
|
+
FileUtils.rm_f file_path
|
29
|
+
end
|
30
|
+
end
|
31
|
+
ensure
|
32
|
+
csv.close if csv
|
33
|
+
end
|
34
|
+
|
35
|
+
def download_latest_to_csv_file(table:, path:)
|
36
|
+
latest_dump = latest
|
37
|
+
download_to_csv_file dump_id: latest_dump['dumpId'], table: table, path: path
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
def table_headers(schema_definition, table)
|
42
|
+
schema_definition['schema'].find { |k, v| v['tableName'] == table }.last['columns'].map { |c| c['name'] }
|
43
|
+
end
|
44
|
+
|
45
|
+
def download_raw_file(file_mapping, dir)
|
46
|
+
resp = open(file_mapping['url'])
|
47
|
+
file_path = "#{dir}/#{File.basename(file_mapping['filename'], '.gz')}"
|
48
|
+
csv_file_path = "#{dir}/#{File.basename(file_mapping['filename'], '.gz')}.csv"
|
49
|
+
if resp.is_a?(StringIO)
|
50
|
+
File.open(file_path, 'wb') { |file| file.write(resp.read) }
|
51
|
+
else
|
52
|
+
FileUtils.cp resp, file_path
|
53
|
+
end
|
54
|
+
File.open(csv_file_path, 'wb') do |file|
|
55
|
+
Zlib::GzipReader.open(file_path) do |gz|
|
56
|
+
while !gz.eof?
|
57
|
+
file.write gz.readpartial(50_000)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
FileUtils.rm_f file_path
|
62
|
+
csv_file_path
|
63
|
+
end
|
64
|
+
|
65
|
+
def url_expired?(url)
|
66
|
+
uri = URI.parse(url)
|
67
|
+
params = CGI::parse(uri.query)
|
68
|
+
params['Expires'].first.to_i < (Time.now.to_i + 600)
|
69
|
+
end
|
70
|
+
|
71
|
+
def renew_urls(dump_id, table, mappings)
|
72
|
+
logger.info("Download URLs have expired. Pulling dump again to get a fresh set")
|
73
|
+
new_definition = dump(dump_id)
|
74
|
+
new_definition['artifactsByTable'][table]['files'].each_with_index do |new_mapping, idx|
|
75
|
+
mappings[idx]['url'] = new_mapping['url']
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'base64'
|
2
|
+
|
3
|
+
module CanvasDataClient::Helpers::HMACHelper
|
4
|
+
include ::CanvasDataClient::Helpers::TimeHelper
|
5
|
+
|
6
|
+
def compute_signature(secret, time, opts = {})
|
7
|
+
message = build_message(secret, rfc7231(time), opts)
|
8
|
+
digest = OpenSSL::Digest.new('sha256')
|
9
|
+
signature = OpenSSL::HMAC.digest(digest, secret, message)
|
10
|
+
Base64.encode64(signature).strip
|
11
|
+
end
|
12
|
+
|
13
|
+
def headers(key, secret, opts = {})
|
14
|
+
raise 'A url must be defined as :path in opts' unless opts[:path]
|
15
|
+
opts[:method] ||= 'get'
|
16
|
+
opts[:content_type] ||= 'application/json'
|
17
|
+
time = Time.now
|
18
|
+
signature = compute_signature(secret, time, opts)
|
19
|
+
{
|
20
|
+
'Authorization' => "HMACAuth #{key}:#{signature}",
|
21
|
+
'Date' => rfc7231(time),
|
22
|
+
'Content-Type' => opts[:content_type]
|
23
|
+
}
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
def build_message(secret, time_string, opts = {})
|
28
|
+
uri = URI(opts[:path])
|
29
|
+
sorted_params = uri.query ? uri.query.split(/&/).sort : []
|
30
|
+
parts = [
|
31
|
+
opts[:method].upcase,
|
32
|
+
uri.host,
|
33
|
+
opts[:content_type] || '',
|
34
|
+
opts[:content_md5] || '',
|
35
|
+
uri.path,
|
36
|
+
sorted_params.join('&'),
|
37
|
+
time_string,
|
38
|
+
secret
|
39
|
+
]
|
40
|
+
parts.join("\n")
|
41
|
+
end
|
42
|
+
end
|
metadata
ADDED
@@ -0,0 +1,147 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: canvas_data_client
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Calvin Kern
|
8
|
+
- pseng
|
9
|
+
autorequire:
|
10
|
+
bindir: exe
|
11
|
+
cert_chain: []
|
12
|
+
date: 2017-10-16 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '1.10'
|
21
|
+
type: :development
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '1.10'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: rake
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '10.0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '10.0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: rspec
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: pry
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: rest-client
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - "~>"
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: 1.8.0
|
77
|
+
type: :runtime
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - "~>"
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: 1.8.0
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: open_uri_redirections
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - "~>"
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: 0.2.1
|
91
|
+
type: :runtime
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - "~>"
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: 0.2.1
|
98
|
+
description:
|
99
|
+
email:
|
100
|
+
- ckern@instructure.com
|
101
|
+
- pseng@instructure.com
|
102
|
+
executables: []
|
103
|
+
extensions: []
|
104
|
+
extra_rdoc_files: []
|
105
|
+
files:
|
106
|
+
- ".gitignore"
|
107
|
+
- ".rspec"
|
108
|
+
- ".travis.yml"
|
109
|
+
- Gemfile
|
110
|
+
- LICENSE.txt
|
111
|
+
- README.md
|
112
|
+
- Rakefile
|
113
|
+
- bin/console
|
114
|
+
- bin/setup
|
115
|
+
- canvas_data_client.gemspec
|
116
|
+
- lib/canvas_data_client.rb
|
117
|
+
- lib/canvas_data_client/client.rb
|
118
|
+
- lib/canvas_data_client/helpers.rb
|
119
|
+
- lib/canvas_data_client/helpers/csv_helper.rb
|
120
|
+
- lib/canvas_data_client/helpers/hmac_helper.rb
|
121
|
+
- lib/canvas_data_client/helpers/time_helper.rb
|
122
|
+
- lib/canvas_data_client/version.rb
|
123
|
+
homepage: http://instructure.com
|
124
|
+
licenses:
|
125
|
+
- MIT
|
126
|
+
metadata: {}
|
127
|
+
post_install_message:
|
128
|
+
rdoc_options: []
|
129
|
+
require_paths:
|
130
|
+
- lib
|
131
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - ">="
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
137
|
+
requirements:
|
138
|
+
- - ">="
|
139
|
+
- !ruby/object:Gem::Version
|
140
|
+
version: '0'
|
141
|
+
requirements: []
|
142
|
+
rubyforge_project:
|
143
|
+
rubygems_version: 2.6.12
|
144
|
+
signing_key:
|
145
|
+
specification_version: 4
|
146
|
+
summary: Wraps the endpoints provided by Canvas Data.
|
147
|
+
test_files: []
|