holistics 0.3.5 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: cab913239d13210e0e3c091b01bc3f9b61cdc78d
-  data.tar.gz: 53b64ba89eb7ffb7134b7b80c15948d291b75efe
+SHA256:
+  metadata.gz: 32364a4f3b5aa6da19afdc05c34a6a9c3361721a200a6788bbc81bd4c7039b52
+  data.tar.gz: a3a993895ae584784d2abe4b3ba4e27a21688e8a4222c71be942cad483bfa809
 SHA512:
-  metadata.gz: 89b361bb3763a3c89c59ffe4845d5f3b1625da74292fd6e19164dfb19ee1814566e9c8f246f7d7a753fb56a4e926c7894738099006dad0063f2c4dc1e1753925
-  data.tar.gz: f115d45d5baa6972a7d30d2b9610032aa39b924d1cf97c1ff98101cfc20e9ca8e6b640b5830245c77f39b1a2915e3c424ebdc40d3959eb7d68d90b8646eefa1c
+  metadata.gz: 8d5cfeb9ffc0990025139117aa02c31a0bada08b86c267bfd37e1257f240267221f9acb0d8eb162e7f9f1ab5a60a00a152c1d070c30ff8bc60a18dafd70c3919
+  data.tar.gz: 441660cafeee9c3b74f6a8cd6f9984b1e2b4e13bf7cbfd33c58d0a765ab98c45b5da371a3a36c315ea236287f5110b2a012aa3501e6caa85ba0f2931d84a4ff5

data/CHANGELOG.md

@@ -2,6 +2,9 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## v0.4.0
+* Add `dbt upload` command
+
 ## v0.3.4
 * Update retry mechanism for `fetch_job_details`
 

data/README.md

@@ -80,6 +80,11 @@ Example:
     [job:738] Hot-swapping with Redshift table public.users ...
     [job:738] Done.
 
+## dbt commands
+### Upload dbt's manifest.json file to Holistics
+    $ holistics dbt upload --file-path /path/to/manifest.json --data-source demodb
+
+
 ### Custom table build transport
 
 See `samples/clicks_mysql_to_redshift.json` for details of transport configs.

data/holistics.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.0"
 
-  spec.add_dependency 'activesupport', '~> 4.2'
+  spec.add_dependency 'activesupport'
   spec.add_dependency 'httparty', '~> 0.13'
   spec.add_dependency 'thor', '~> 0.19'
   spec.add_dependency 'multipart-post', '~> 2.0'

data/lib/dbt.rb

@@ -0,0 +1,48 @@
+require 'thor'
+require 'holistics/helpers/http_request'
+
+module Holistics
+  class Dbt < Thor
+    method_option :filepath, aliases: '--file-path', type: :string, required: false, desc: 'Path to manifest.json file'
+    method_option :data_source_name, aliases: '--data-source', type: :string, required: false, desc: 'Name of data_source'
+    desc 'upload', 'Upload manifest.json file to Holistics'
+    def upload
+      data_source_name = options[:data_source_name]
+      manifest_file = get_manifest_file_from(options)
+
+      params = {
+        data_source_name: data_source_name
+      }
+
+      http_request.post_file('api/v2/data_sources/upload_dbt_manifest.json', params, manifest_file, 'application/json',
+                             'Error uploading manifest.json file')
+
+      puts 'Upload completed!'
+    end
+
+    private
+
+    def get_manifest_file_from(options)
+      local_filepath =
+        if options[:filepath].present?
+          options[:filepath]
+        else
+          Dir['./**/manifest.json'].first
+        end
+
+      begin
+        file = File.open(local_filepath)
+      rescue StandardError
+        warn "Could not open file at '#{local_filepath}'."
+        puts 'Invalid file path. Please check your file path.'
+        return
+      end
+
+      file
+    end
+
+    def http_request
+      @http_request ||= Helpers::HttpRequest.new
+    end
+  end
+end

data/lib/holistics/api_client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'tabular_formatter'
 require 'yaml'
 require 'json'
@@ -10,16 +12,16 @@ module Holistics
     class ImportError < StandardError
     end
 
-    def import_csv options
+    def import_csv(options)
       local_filepath = options[:filepath]
       dest_ds_id = options[:dest_ds_id]
       dest_fqname = options[:dest_table_name]
 
       begin
         file = File.open local_filepath
-      rescue
-        STDERR.puts "Could not open file at '#{local_filepath}'."
-        puts "Invalid file path. Please check your file path."
+      rescue StandardError
+        warn "Could not open file at '#{local_filepath}'."
+        puts 'Invalid file path. Please check your file path.'
 
         return
       end
@@ -31,7 +33,7 @@ module Holistics
         dest_ds_id: dest_ds_id
       }
 
-      json = http_request.post_csv 'data_imports/import_csv.json', params, file, 'Error uploading CSV file'
+      json = http_request.post_file 'data_imports/import_csv.json', params, file, 'text/csv', 'Error uploading CSV file'
 
       job_id = json['message']['job_id']
 
@@ -42,8 +44,8 @@ module Holistics
     def ds_list
       result = http_request.get 'data_sources.json', 'Error retrieving list of data sources'
 
-      table = [%w(ID Type Name)]
-      rows = result.map {|record| [record['id'], record['dbtype'], record['name']]}
+      table = [%w[ID Type Name]]
+      rows = result.map { |record| [record['id'], record['dbtype'], record['name']] }
       table.concat(rows)
 
       puts TabularFormatter.new(table).to_pretty_table
@@ -52,8 +54,8 @@ module Holistics
     def import_list
       result = http_request.get 'data_imports.json', 'Error retrieving list of data imports'
 
-      table = [%w(ID Name)]
-      rows = result.map {|record| [record['id'], record['title']]}
+      table = [%w[ID Name]]
+      rows = result.map { |record| [record['id'], record['title']] }
       table.concat(rows)
 
       puts TabularFormatter.new(table).to_pretty_table
@@ -62,8 +64,8 @@ module Holistics
     def transform_list
       result = http_request.get 'data_transforms.json', 'Error retrieving list of data transports'
 
-      table = [%w(ID Name)]
-      rows = result.map {|record| [record['id'], record['title']]}
+      table = [%w[ID Name]]
+      rows = result.map { |record| [record['id'], record['title']] }
       table.concat(rows)
 
       puts TabularFormatter.new(table).to_pretty_table
@@ -113,7 +115,7 @@ module Holistics
       job_manager.tail_logs job_id
       res = job_manager.fetch_job_results job_id
       unless res['status'] == 'success'
-        raise ImportError.new("Failed Import Job #{job_id}: #{res['error']}")
+        raise ImportError, "Failed Import Job #{job_id}: #{res['error']}"
       end
     end
 
@@ -123,16 +125,14 @@ module Holistics
     end
 
     def generate_configs(options)
-      unless Dir.exist?(options['output'])
-        raise 'Output location is invalid.'
-      end
+      raise 'Output location is invalid.' unless Dir.exist?(options['output'])
 
       puts "Generating transport JSON config files to #{options['output']} directory..."
 
       result = http_request.post_json 'transports/generate_configs.json', options, 'Error generating transport configs'
 
       result.each do |table_data|
-        File.open("#{options['output']}/#{table_data['filename']}", "w") do |f|
+        File.open("#{options['output']}/#{table_data['filename']}", 'w') do |f|
           f.write(JSON.pretty_generate(table_data['json_content']))
         end
 
@@ -145,24 +145,28 @@ module Holistics
 
     def build_submit_params(options)
       params = options.except(:config_path)
-      params[:configs] = parse_transport_config(options[:config_path]).to_json if options[:config_path]
+      if options[:config_path]
+        params[:configs] = parse_transport_config(options[:config_path]).to_json
+      end
       params
     end
 
     def parse_transport_config(filepath)
       file_ext = File.extname(filepath).downcase
       if file_ext == '.json'
-        return JSON.parse(File.read(filepath))
+        JSON.parse(File.read(filepath))
       elsif file_ext == '.yml'
-        return YAML.load(File.read(filepath))
+        YAML.safe_load(File.read(filepath))
       else
-        raise StandardError.new 'Invalid config file extension. Please use either JSON or YML'
+        raise StandardError, 'Invalid config file extension. Please use either JSON or YML'
       end
     rescue StandardError => e
-      STDERR.puts "Error parsing transport config file: #{e.message}"
+      warn "Error parsing transport config file: #{e.message}"
       exit 1
     end
 
+    def dbt_upload(filepath, ds_name); end
+
     private
 
     def file_manager
@@ -180,7 +184,5 @@ module Holistics
     def auth_info
       @auth_info ||= Helpers::AuthInfo.new
     end
-
-
   end
 end

data/lib/holistics/helpers/http_request.rb

@@ -34,7 +34,7 @@ module Holistics
         exit_if_error(msg_if_error, response)
         JSON.parse response.body
       end
-      
+
       def simple_get(url, token = nil)
         HTTParty
           .get url, headers: { API_KEY_HEADER => token || auth_helper.get_token_from_gconfig }
@@ -61,11 +61,11 @@ module Holistics
       #
       # see this link for setting it
       # https://github.com/nicksieger/multipart-post/issues/18#issuecomment-171479987
-      def post_csv url, params, file, msg_if_error = DEFAULT_ERROR_MSG
+      def post_file url, params, file, content_type, msg_if_error = DEFAULT_ERROR_MSG
         uri = URI.parse(endpoint_for(url))
-        csv = UploadIO.new file, "text/csv", File.basename(file.path)
-        params = params.merge file: csv
-        
+        upload_file = UploadIO.new file, content_type, File.basename(file.path)
+        params = params.merge file: upload_file
+
         post_data = Net::HTTP::Post::Multipart.new uri.path, params
         post_data.add_field(API_KEY_HEADER, auth_helper.get_token_from_gconfig)
 

data/lib/holistics/version.rb

@@ -1,4 +1,4 @@
 module Holistics
-  VERSION = '0.3.5'
-  DATE = '2021-04-19'
+  VERSION = '0.4.0'
+  DATE = '2021-10-15'
 end

data/lib/holistics.rb

@@ -12,6 +12,7 @@ require 'transport'
 require 'transform'
 require 'data_sources'
 require 'email_schedule'
+require 'dbt'
 
 module Holistics
   def self.root
@@ -50,6 +51,7 @@ module Holistics
     register(Holistics::Transform, 'transform', 'transform <command>', "Execute transform module's commands")
     register(Holistics::DataSources, 'data_sources', 'data_sources <command>', "Execute data_sources module's commands")
     register(Holistics::EmailSchedule, 'email_schedule', 'email_schedule <command>', "Execute email schedule's commands")
+    register(Holistics::Dbt, 'dbt', 'dbt <comnand>', 'dbt integration with Holistics')
 
     no_commands do
       def auth_api_client

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: holistics
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.4.0
 platform: ruby
 authors:
 - Thanh Dinh Khac
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-19 00:00:00.000000000 Z
+date: 2021-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -29,16 +29,16 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +108,7 @@ files:
 - bin/holistics
 - holistics.gemspec
 - lib/data_sources.rb
+- lib/dbt.rb
 - lib/email_schedule.rb
 - lib/holistics.rb
 - lib/holistics/api_client.rb
@@ -143,8 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.5.1
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: CLI interface for Holistics