canvas_data_client 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d190d87207c61e7aa4615e38b96418c598ff1cee
+  data.tar.gz: 77547c95bd694534b07ddc326434223091957412
+SHA512:
+  metadata.gz: 654c8b81bdb6a172068b5d98dd44da5ed3398566bab9363d82d104063d417f0317931124f24cbfe407aef3e187cfe78ce210b91c6e39e7bdc776134c5bbddad3
+  data.tar.gz: 8e210a9a12486f711c0b995d4d89f9f1aa54cb5a4a1c96b7549f51baf0f1e0bd1ca60e699bb53a0e97a3b14d0b7092ed691a36f9fccfc9785ac3ae5db2e9a735

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.byebug_history

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.5
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in canvas_data_client.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Ben Young
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,44 @@
+# CanvasDataClient
+
+This gem is meant to provide an easy-to-use ruby client wrapping the [Canvas](https://canvaslms.com) Hosted Data API.
+
+It calculates and attaches HMAC signatures for each request, and returns the parsed JSON response.
+
+## Usage
+
+The client can be initialized as such:
+
+```ruby
+client = CanvasDataClient::Client.new(api_key, api_secret)
+```
+
+As much as possible, the methods available in the client match those given in the [API docs](https://portal.inshosteddata.com/docs/api)
+
+List of methods:
+
+```ruby
+client.latest_files # GET /api/account/(:accountId|self)/file/latest
+client.dumps # GET /api/account/(:accountId|self)/dump
+client.dump('b349ad95-4839-48f3-b763-ec555fc2f42f') # GET /api/account/(:accountId|self)/file/byDump/:dumpId
+client.tables('course_dim') # GET /api/account/(:accountId|self)/file/byTable/:tableName
+client.schemas # GET /api/schema
+client.latest_schema # GET /api/schema/latest
+client.schema('1.0.0') # GET /api/schema/:version
+
+# Downloads a table file from the specified dump (or all files for a table should the dump have multiple)
+# and writes them to a single CSV file.  This also includes a CSV header row of the column names
+# pulled from the schema definition
+client.download_to_csv_file(dump_id: 'b349ad95-4839-48f3-b763-ec555fc2f42f', table: 'requests', path: '/path/to/output_file.csv')
+
+# Same as #download_to_csv_file, but uses the latest dump available
+client.download_latest_to_csv_file(table: 'requests', path: '/path/to/output_file.csv')
+```
+
+## Source
+
+Originally from a partners GitHub at https://github.com/ben-y/canvas_data_client.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "canvas_data_client"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/canvas_data_client.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'canvas_data_client/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "canvas_data_client"
+  spec.version       = CanvasDataClient::VERSION
+  spec.authors       = ["Calvin Kern", "pseng"]
+  spec.email         = ["ckern@instructure.com", "pseng@instructure.com"]
+
+  spec.summary       = "Wraps the endpoints provided by Canvas Data."
+  spec.homepage      = "http://instructure.com"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
+
+  spec.add_runtime_dependency "rest-client", "~> 1.8.0"
+  spec.add_runtime_dependency 'open_uri_redirections', '~> 0.2.1'
+end

data/lib/canvas_data_client.rb

@@ -0,0 +1,6 @@
+require "canvas_data_client/version"
+
+module CanvasDataClient
+  require 'canvas_data_client/helpers'
+  require 'canvas_data_client/client'
+end

data/lib/canvas_data_client/client.rb

@@ -0,0 +1,72 @@
+require 'rest-client'
+
+module CanvasDataClient
+  class Client
+    include CanvasDataClient::Helpers::HMACHelper
+    include CanvasDataClient::Helpers::CsvHelper
+
+    attr_accessor :key, :secret, :subdomain, :account, :logger
+
+    def initialize(key, secret, opts = {})
+      self.key = key
+      self.secret = secret
+      self.subdomain = opts[:subdomain] || 'portal'
+      self.account = opts[:account] || 'self'
+      self.logger = Logger.new(STDOUT)
+    end
+
+    def domain
+      "https://#{subdomain}.inshosteddata.com"
+    end
+
+    def latest_files
+      json_request "#{domain}/api/account/#{account}/file/latest"
+    end
+    alias_method :latest, :latest_files
+
+    def dumps
+      paginated_request "#{domain}/api/account/#{account}/dump?after=%s"
+    end
+
+    def dump(dump_id)
+      json_request "#{domain}/api/account/#{account}/file/byDump/#{dump_id}"
+    end
+
+    def tables(table)
+      paginated_request "#{domain}/api/account/#{account}/file/byTable/#{table}?after=%s"
+    end
+
+    def schemas
+      json_request "#{domain}/api/schema"
+    end
+
+    def latest_schema
+      json_request "#{domain}/api/schema/latest"
+    end
+
+    def schema(version)
+      json_request "#{domain}/api/schema/#{version}"
+    end
+
+    private
+    def json_request(path, method = 'get')
+      resp = RestClient.get path, headers(key, secret, { path: path, method: method })
+      JSON.parse resp
+    end
+
+    def paginated_request(path)
+      received = []
+      sequence = '0'
+      loop do
+        resp = json_request(path % sequence)
+        resp = resp['history'] if resp.is_a?(Hash)
+        resp.sort_by! { |h| h['sequence'] }
+        received += resp
+        break if resp.length < 50
+        sequence = resp.last['sequence']
+      end
+      received
+    end
+
+  end
+end

data/lib/canvas_data_client/helpers.rb

@@ -0,0 +1,5 @@
+module CanvasDataClient::Helpers
+  require 'canvas_data_client/helpers/time_helper'
+  require 'canvas_data_client/helpers/hmac_helper'
+  require 'canvas_data_client/helpers/csv_helper'
+end

data/lib/canvas_data_client/helpers/csv_helper.rb

@@ -0,0 +1,78 @@
+require 'csv'
+require 'open-uri'
+require 'open_uri_redirections'
+
+module CanvasDataClient::Helpers::CsvHelper
+  class TableNotPresentError < StandardError; end
+
+  def download_to_csv_file(dump_id:, table:, path:)
+    dump_definition = dump(dump_id)
+    schema_definition = schema(dump_definition['schemaVersion'])
+    raise TableNotPresentError.new("Table #{table} not present in dump #{dump_id}") unless dump_definition['artifactsByTable'][table]
+
+    csv = CSV.open(path, 'w')
+    columns = table_headers(schema_definition, table)
+    csv << columns
+
+    Dir.mktmpdir do |dir|
+      dump_definition['artifactsByTable'][table]['files'].each do |file_mapping|
+        renew_urls(dump_id, table, dump_definition['artifactsByTable'][table]['files']) if url_expired?(file_mapping['url'])
+        logger.info("Downloading table file: #{file_mapping['filename']}")
+        file_path = download_raw_file(file_mapping, dir)
+        logger.info("Processing table file: #{file_mapping['filename']}")
+        File.foreach(file_path) do |row|
+          split_row = row.gsub(/\n/, '').split(/\t/)
+          split_row.fill(nil, split_row.length...columns.length) if split_row.length < columns.length
+          csv << split_row.map { |col| col == '\\N' ? nil : col }
+        end
+        FileUtils.rm_f file_path
+      end
+    end
+  ensure
+    csv.close if csv
+  end
+
+  def download_latest_to_csv_file(table:, path:)
+    latest_dump = latest
+    download_to_csv_file dump_id: latest_dump['dumpId'], table: table, path: path
+  end
+
+  private
+  def table_headers(schema_definition, table)
+    schema_definition['schema'].find { |k, v| v['tableName'] == table }.last['columns'].map { |c| c['name'] }
+  end
+
+  def download_raw_file(file_mapping, dir)
+    resp = open(file_mapping['url'])
+    file_path = "#{dir}/#{File.basename(file_mapping['filename'], '.gz')}"
+    csv_file_path = "#{dir}/#{File.basename(file_mapping['filename'], '.gz')}.csv"
+    if resp.is_a?(StringIO)
+      File.open(file_path, 'wb') { |file| file.write(resp.read) }
+    else
+      FileUtils.cp resp, file_path
+    end
+    File.open(csv_file_path, 'wb') do |file|
+      Zlib::GzipReader.open(file_path) do |gz|
+        while !gz.eof?
+          file.write gz.readpartial(50_000)
+        end
+      end
+    end
+    FileUtils.rm_f file_path
+    csv_file_path
+  end
+
+  def url_expired?(url)
+    uri = URI.parse(url)
+    params = CGI::parse(uri.query)
+    params['Expires'].first.to_i < (Time.now.to_i + 600)
+  end
+
+  def renew_urls(dump_id, table, mappings)
+    logger.info("Download URLs have expired.  Pulling dump again to get a fresh set")
+    new_definition = dump(dump_id)
+    new_definition['artifactsByTable'][table]['files'].each_with_index do |new_mapping, idx|
+      mappings[idx]['url'] = new_mapping['url']
+    end
+  end
+end

data/lib/canvas_data_client/helpers/hmac_helper.rb

@@ -0,0 +1,42 @@
+require 'base64'
+
+module CanvasDataClient::Helpers::HMACHelper
+  include ::CanvasDataClient::Helpers::TimeHelper
+
+  def compute_signature(secret, time, opts = {})
+    message = build_message(secret, rfc7231(time), opts)
+    digest = OpenSSL::Digest.new('sha256')
+    signature = OpenSSL::HMAC.digest(digest, secret, message)
+    Base64.encode64(signature).strip
+  end
+
+  def headers(key, secret, opts = {})
+    raise 'A url must be defined as :path in opts' unless opts[:path]
+    opts[:method] ||= 'get'
+    opts[:content_type] ||= 'application/json'
+    time = Time.now
+    signature = compute_signature(secret, time, opts)
+    {
+      'Authorization' => "HMACAuth #{key}:#{signature}",
+      'Date' => rfc7231(time),
+      'Content-Type' => opts[:content_type]
+    }
+  end
+
+  private
+  def build_message(secret, time_string, opts = {})
+    uri = URI(opts[:path])
+    sorted_params = uri.query ? uri.query.split(/&/).sort : []
+    parts = [
+      opts[:method].upcase,
+      uri.host,
+      opts[:content_type] || '',
+      opts[:content_md5] || '',
+      uri.path,
+      sorted_params.join('&'),
+      time_string,
+      secret
+    ]
+    parts.join("\n")
+  end
+end

data/lib/canvas_data_client/helpers/time_helper.rb

@@ -0,0 +1,6 @@
+module CanvasDataClient::Helpers::TimeHelper
+
+  def rfc7231(time)
+    time.utc.strftime('%a, %d %b %Y %H:%M:%S GMT')
+  end
+end

data/lib/canvas_data_client/version.rb

@@ -0,0 +1,3 @@
+module CanvasDataClient
+  VERSION = "0.0.3"
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: canvas_data_client
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Calvin Kern
+- pseng
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-10-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: open_uri_redirections
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+description: 
+email:
+- ckern@instructure.com
+- pseng@instructure.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- canvas_data_client.gemspec
+- lib/canvas_data_client.rb
+- lib/canvas_data_client/client.rb
+- lib/canvas_data_client/helpers.rb
+- lib/canvas_data_client/helpers/csv_helper.rb
+- lib/canvas_data_client/helpers/hmac_helper.rb
+- lib/canvas_data_client/helpers/time_helper.rb
+- lib/canvas_data_client/version.rb
+homepage: http://instructure.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Wraps the endpoints provided by Canvas Data.
+test_files: []