export_to_gcloud 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b10b7cddc6441b96f235f33936eb9560e9662f86
+  data.tar.gz: 91f284abd485f6c6bbf8e7cd2ff99d49e3983157
+SHA512:
+  metadata.gz: ac65925a3d33b4b0d081e170a5a0f0a7d29d86e08a7490b9703b05e9163628514ef815db66c9c09422e50ab89d73376753b7e402935062b915a26e02eefb1129
+  data.tar.gz: 0a3b60ce3cfdc1c6523a9700701f502d0a6cf8e7329be4bc63bb72d4c41e373e7294ccf31fa2ccec7f9d3b3de5ecaee62cb6ad37b6ae5ed93422e54fdbb982bd

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Ondřej Želazko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/lib/export_to_gcloud/exporter/context.rb

@@ -0,0 +1,49 @@
+class ExportToGcloud::Exporter::Context
+
+  attr_reader :client
+
+  OPTIONS = %i[dump_path storage_prefix bucket dataset].freeze
+
+  def initialize client, **opts
+    @client = client
+    set opts
+  end
+
+  def set **opts
+    OPTIONS.each do |key|
+      value = opts[key]
+      send "set_#{key}", value if value
+    end
+    self
+  end
+
+  def set_dump_path path
+    @dump_path = Pathname.new path
+  end
+
+  def set_storage_prefix prefix
+    @storage_prefix = prefix
+  end
+
+  def set_bucket bucket
+    bucket = client.storage.bucket bucket if String === bucket
+    @bucket = bucket
+  end
+
+  def set_dataset dataset
+    dataset = client.bigquery.dataset dataset if String === dataset
+    @dataset = dataset
+  end
+
+  OPTIONS.each do |key|
+    define_method key do
+      value = instance_variable_get "@#{key}"
+      value || raise("Undefined value for #{key} in exporter options!")
+    end
+  end
+
+  def copy
+    self.class.new client, OPTIONS.inject({}){|h, k| h[k] = instance_variable_get "@#{k}"; h}
+  end
+
+end

data/lib/export_to_gcloud/exporter/definition.rb

@@ -0,0 +1,47 @@
+class ExportToGcloud::Exporter::Definition < OpenStruct
+
+  def initialize exporter_type, attrs
+    super attrs.merge!(type: exporter_type)
+  end
+
+  def validate!
+    (String === name && !name.empty?)   || raise('`name` must be defined!')
+    Proc === bq_schema                  || raise('`bq_schema` must be defined as a Proc!')
+    data                                || raise('`data` must be defined!')
+    type.validate_definition! self if type.respond_to? 'validate_definition!'
+  end
+
+  def get_data *args
+    Proc === data ? data.call(*args) : data
+  end
+
+  def get_bq_table_name
+    bq_table_name || name
+  end
+
+  def self.set_last_definition klass, attrs={}, &block
+    last_definition = new klass, attrs
+    block.call last_definition if block
+
+    last_definition.validate!
+    @last_definition = last_definition
+  end
+
+  def self.load_definition name, finder
+    file_path = finder.call name
+    load file_path
+    definition = @last_definition
+    @last_definition = nil
+
+    unless definition
+      raise("File #{file_path.to_s} must define exporter for '#{name}'!")
+    end
+
+    unless definition.name == name
+      raise "File #{file_path.to_s} defines '#{definition.name}' instead of '#{name}'"
+    end
+
+    definition
+  end
+
+end

data/lib/export_to_gcloud/exporters/csv_exporter.rb

@@ -0,0 +1,18 @@
+
+module ExportToGcloud
+
+  class CSVExporter < Exporter
+
+    def create_data_file! file, *part_data
+      data = @definition.get_data(*part_data)
+
+      csv_data = CSV.generate col_sep: ';', force_quotes: false do |csv|
+        data.each{|row| csv << row}
+      end
+
+      File.write file.to_path, csv_data
+    end
+
+  end
+
+end

data/lib/export_to_gcloud/exporters/exporter.rb

@@ -0,0 +1,112 @@
+
+module ExportToGcloud
+
+  class Exporter
+
+    def initialize definition, context
+      @definition = definition
+      @context = context
+
+      @parts = []
+      case definition.parts
+        when Array then definition.parts.each{|label, *part_args| add_data_part *part_args, label: label}
+        when Proc  then definition.parts.call self
+      end
+    end
+
+    def local_file_path label
+      @context.dump_path.join "#{@definition.name}#{prepend_underscore label}.csv"
+    end
+
+    def storage_file_path label
+      prefix = @definition.storage_prefix || @context.storage_prefix
+      "#{prefix}#{@definition.name}#{prepend_underscore label}.csv"
+    end
+
+    def add_data_part *args, label:nil
+      args.unshift(label ? label.to_s : (@parts.length+1).to_s)
+      @parts << args
+    end
+
+    def process_all_parts! recreate_table=true
+      add_data_part label: 'all' if @parts.empty?
+      recreate_bq_table! if recreate_table
+
+      @parts.map{|*args| process_part! *args}
+    end
+
+    def process_part! label, *part_args
+      file = local_file_path label
+      create_data_file! file, *part_args
+
+      storage_name = storage_file_path label
+      gcloud_file = upload_file! file, storage_name
+      start_load_job gcloud_file
+    end
+
+    def create_data_file! file, *part_data
+      File.write file.to_path, @definition.get_data(*part_data)
+    end
+
+    def upload_file!(file, storage_name)
+      file = compress_file! file
+      gcloud_file = @context.bucket.create_file file, storage_name, chunk_size: 2**21 # 2MB
+      file.delete
+      gcloud_file
+    end
+
+    def get_storage_files
+      @parts.map do |label, *_|
+        @context.bucket.file storage_file_path(label)
+      end.compact
+    end
+
+    def bq_table
+      unless defined? @bq_table
+        @bq_table = @context.dataset.table @definition.get_bq_table_name
+      end
+      @bq_table
+    end
+
+    def recreate_bq_table!
+      bq_table.delete if bq_table
+      @bq_table = @context.dataset.create_table @definition.get_bq_table_name, &@definition.bq_schema
+    end
+
+    def start_load_job gcloud_file, **_load_settings
+      load_settings = {
+          format: 'csv',
+          quote: '"',
+          delimiter: ';',
+          create: 'never',
+          write: 'append',
+          max_bad_records: 0
+      }
+      load_settings.merge! _load_settings unless _load_settings.empty?
+      bq_table.load gcloud_file, **load_settings
+    end
+
+    def self.define **kwargs, &block
+      ::ExportToGcloud::Exporter::Definition.set_last_definition self, kwargs, &block
+    end
+
+    private
+
+    def compress_file!(original_file)
+      err = %x(pigz -f9 #{original_file.to_path} 2>&1)
+      compressed_file = Pathname.new "#{original_file.to_path}.gz"
+      raise "Compression of #{original_file.to_path} failed: #{err}" unless compressed_file.exist?
+      original_file.delete if original_file.exist?
+      compressed_file
+    end
+
+    def prepend_underscore text
+      "_#{text}" if String === text && !text.empty?
+    end
+
+  end
+
+end
+
+require_relative '../exporter/definition'
+require_relative '../exporter/context'

data/lib/export_to_gcloud/exporters/pg_exporter.rb

@@ -0,0 +1,37 @@
+
+module ExportToGcloud
+
+  class PGExporter < Exporter
+
+    def create_data_file! file, *part_data
+      sql = @definition.get_data(*part_data)
+
+      schema = ::Gcloud::Bigquery::Table::Schema.new nil
+      @definition.bq_schema.call schema
+      string_fields = schema.fields.select{|f| f['type']=='STRING'}.map{|f| f['name']}
+
+      force_quote = if string_fields.empty?
+        ''
+      else
+        ", FORCE_QUOTE (#{string_fields.join ', '})"
+      end
+      sql = "COPY (#{sql}) TO '#{file.to_path}' WITH (FORMAT CSV, DELIMITER ';', QUOTE '\"'#{force_quote});"
+
+
+      executor = @definition.get_sql_executor || self.class.default_executor
+      executor.call sql
+    end
+    
+    def self.validate_definition! definition
+      definition.get_sql_executor || default_executor || raise('`sql_executor` needs to be defined!')
+    end
+
+    class << self
+
+      attr_accessor :default_executor
+
+    end
+
+  end
+
+end

data/lib/export_to_gcloud/library.rb

@@ -0,0 +1,67 @@
+require 'gcloud'
+require 'gcloud/bigquery'
+
+# large files uploading
+require 'httpclient'
+Faraday.default_adapter = :httpclient
+
+# monkeypatch :/ some issue in google-api
+# see http://googlecloudplatform.github.io/gcloud-ruby/docs/master/Gcloud/Storage.html
+#     -> A note about large uploads
+require 'google/api_client'
+Faraday::Response.register_middleware gzip: Faraday::Response::Middleware
+
+module ExportToGcloud
+
+  def self.definitions_resolver= proc
+    @definitions_resolver = proc
+  end
+
+  # waits for BigQuery jobs
+  # - send a block to do something with failed
+  def self.wait_for_load_jobs(jobs, &block)
+    jobs_left = jobs.dup
+    failed = []
+    sleeper = ->(_retries) {sleep 2 * _retries + 5}
+    retries = 0
+
+    until jobs_left.empty?
+      sleeper.call retries
+      retries += 1
+      jobs_left.each &:reload!
+      jobs_left.delete_if do |j|
+        if j.done?
+          failed << {id: j.job_id, error: j.error, sources: j.sources} if j.failed?
+          true
+        end
+      end
+    end
+
+    block.call failed unless failed.empty?
+  end
+
+  def self.get_exporter name, context
+    name = name.to_s
+
+    @definitions ||= {}
+    unless @definitions.has_key? name
+      @definitions[name] = ::ExportToGcloud::Exporter::Definition.load_definition name, @definitions_resolver
+    end
+
+    definition = @definitions[name]
+    definition.type.new definition, context
+  end
+
+  def self.create_context **opts
+    ::ExportToGcloud::Exporter::Context.new client, opts
+  end
+
+end
+
+require 'pathname'
+require 'ostruct'
+require 'csv'
+
+require_relative 'exporters/exporter'
+require_relative 'exporters/csv_exporter'
+require_relative 'exporters/pg_exporter'

data/lib/export_to_gcloud/version.rb

@@ -0,0 +1,5 @@
+module ExportToGcloud
+
+  VERSION = '0.9.0'
+
+end

data/lib/export_to_gcloud.rb

@@ -0,0 +1,16 @@
+require_relative 'export_to_gcloud/version'
+
+module ExportToGcloud
+
+  def self.setup project_name:, config_file:, definitions_resolver:nil
+    require_relative 'export_to_gcloud/library'
+
+    self.definitions_resolver = definitions_resolver if definitions_resolver
+    @client = ::Gcloud.new project_name, config_file
+  end
+
+  def self.client
+    @client || raise('Gcloud client not present. call ExportToGcloud#setup first.')
+  end
+
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: export_to_gcloud
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- Ondřej Želazko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-11-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: gcloud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+description: A simple helper to export data to BigQuery via Google Drive
+email: zelazk.o@email.cz
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- lib/export_to_gcloud.rb
+- lib/export_to_gcloud/exporter/context.rb
+- lib/export_to_gcloud/exporter/definition.rb
+- lib/export_to_gcloud/exporters/csv_exporter.rb
+- lib/export_to_gcloud/exporters/exporter.rb
+- lib/export_to_gcloud/exporters/pg_exporter.rb
+- lib/export_to_gcloud/library.rb
+- lib/export_to_gcloud/version.rb
+homepage: https://github.com/doooby/export_to_gcloud
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Exporter to BigQuery
+test_files: []