takenoko 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6c6846bb822d84f908f35ea7e6737e87b20f5c17
+  data.tar.gz: 9c1533aaa9e1d993f7beb2f5ae8265f59a4fb697
+SHA512:
+  metadata.gz: 9d2627e910529f9c04a3b57477e46e4e738115a3cf8224f7c7e92151d471ac84c790265fa90a3c1e7688ea7e705fcb03ccbfd1885287d332b174c94a3be0cf6b
+  data.tar.gz: a700a1f350e7214fb1a1cb4d688d87ac0494531c7f50f7ee60ccfd4b77cc23cc3e4209bca8f3a8ba2f0a1186c326493c54699220d5e2d35da4d556b6985dad32

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2016 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,32 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Takenoko'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/lib/generators/takenoko/config_generator.rb

@@ -0,0 +1,13 @@
+module Takenoko
+  module Generators
+    class ConfigGenerator < Rails::Generators::Base
+      source_root File.expand_path(File.join(File.dirname(__FILE__), 'templates'))
+
+      desc 'takenoko_config.rb'
+
+      def copy_config_file
+        template 'takenoko.rb', 'config/initializers/takenoko.rb'
+      end
+    end
+  end
+end

data/lib/generators/takenoko/templates/takenoko.rb

@@ -0,0 +1,4 @@
+Takenoko.config do |conf|
+  # conf.google_cridential_file = "path to google credential file.json"
+  # conf.mapping_file = "path to mapping file.yml"
+end

data/lib/google_drive/google_drive.rb

@@ -0,0 +1,63 @@
+require "google_drive"
+
+module GoogleDrive
+  class Worksheet
+    define_method "header" do
+      rows.first
+    end
+
+    define_method "append_row" do |offset=0|
+      row_num = self.num_rows + offset + 1
+      return GoogleDrive::Alias::Row.new(row_num, [], rows.first, self)
+    end
+
+    define_method "populated_rows" do
+      header = nil
+      populated = []
+
+      rows.each_with_index do |row,index|
+        unless header then
+          header = row
+          next
+        end
+
+        populated.push GoogleDrive::Alias::Row.new(index + 1, row, header, self)
+      end
+
+      populated
+    end
+  end
+end
+
+module GoogleDrive
+  module Alias
+    class Row
+      attr_reader :row_num
+      
+      def initialize(row_num, row, header, ws)
+        @row_num = row_num
+        @row     = row
+        @header  = header
+        @ws      = ws
+      end
+
+      def method_missing(name,value=nil)
+        test = name.to_s.gsub(/=$/,'')
+
+        if @header.include?(test) then
+          col_num = @header.find_index(test)
+          super unless col_num
+
+          col_num += 1
+          if name.match(/=$/) then
+            @ws[@row_num,col_num] = value
+          else
+            return @ws[@row_num,col_num]
+          end
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/takenoko.rb

@@ -0,0 +1,112 @@
+require 'rails'
+require 'google_drive'
+require 'google_drive/google_drive'
+
+module Takenoko 
+  extend self
+  mattr_accessor :google_cridential_file
+  @@google_cridential_file = nil
+
+  mattr_accessor :mapping_file
+  @@mapping_file = false
+
+  @@google_client = nil
+  @@mapping_config = nil
+
+  mattr_accessor :file_extension
+  @@file_extension = :csv
+  SUPPORTED_FILE_EXT = [:csv,:yaml,:json]
+
+  mattr_accessor :export_file_location
+  @@export_file_location = "db/spreadsheet"
+
+  mattr_accessor :truncate_all_data
+  @@truncate_all_data = false
+
+  mattr_accessor :allow_overwrite
+  @@allow_overwrite = true
+
+  require 'takenoko/exporter'
+  require 'takenoko/google_client'
+
+  def config
+    yield self
+  end
+
+  def import
+    check_config
+  end
+
+  def google_client
+    @@google_client ||= GoogleClient.new(@@google_cridential_file)
+    return @@google_client
+  end
+
+  def check_config
+    raise "google_cridential_file setting cannot be nil" unless @@google_cridential_file
+    raise "file not found:#{@@google_cridential_file}" unless ::File.exist?(@@google_cridential_file)
+    raise "mapping_file cannot be nil" unless @@mapping_file
+    raise "file not found:#{@@mapping_file}" unless ::File.exist?(@@mapping_file)
+    return true
+  end
+
+  def mapping_config
+    return @@mapping_config if @@mapping_config
+    conf = YAML.load_file(@@mapping_file).with_indifferent_access
+    raise "tables not exists" if conf[:tables].blank?
+    conf[:tables].each do |t, v|
+      table = conf[:tables][t]
+      raise "Table config cannot be nil" unless table
+      [:sheet_id, :worksheet_id, :columns_mapping].each do |f|
+        raise "#{f} cannot be blank" if table[f].blank?
+      end
+      table[:find_column] = table[:find_column] || :id
+      table_name = table[:table_name] = t if table[:table_name].blank?
+      table[:class_name] = table[:class_name] || (table_name && table_name.singularize.camelize) || table[:table_name].singularize.camelize
+      unless table[:columns_mapping].key?(table[:find_column])
+        if(table[:find_column] == :id)
+          table[:columns_mapping] = {id: nil}.merge!(table[:columns_mapping])
+        else
+          table[:columns_mapping][table[:find_column]] = nil 
+        end
+      end
+      table[:columns_mapping].each do |s_col,db_col|
+        table[:columns_mapping][s_col] = s_col unless db_col 
+      end
+
+      table[:header] = table[:columns_mapping].keys
+      [:allow_overwrite,:truncate_all_data, :file_extension, :export_file_location].each do |f|
+        table[f] = class_variable_get("@@" + f.to_s) unless table.key?(f)
+      end
+
+      raise "Not support file extension: #{table[:file_extension]}" unless SUPPORTED_FILE_EXT.include?(table[:file_extension])
+    end
+    @@mapping_config = conf
+  end
+
+  def table_config(table_name)
+    raise "#{table_name} config not exists" unless conf = mapping_config[:tables][table_name]
+    return conf
+  end
+
+  (SUPPORTED_FILE_EXT.clone << :db).each do |output|
+    define_method "table_to_#{output}" do | table_name |
+      data = google_client.get_table(table_name)
+      Exporter.public_send("table_to_#{output}",data)
+    end
+
+    define_method "all_to_#{output}" do
+      mapping_config[:tables].each do |table,conf|
+        Takenoko.public_send("table_to_#{output}",table)
+      end
+    end
+  end
+
+  class Railtie < Rails::Railtie
+    railtie_name :takenoko
+
+    rake_tasks do
+      load "tasks/takenoko.rake"
+    end
+  end
+end

data/lib/takenoko/exporter.rb

@@ -0,0 +1,45 @@
+require "takenoko"
+require "csv"
+module Takenoko
+  module Exporter
+    extend self
+    def table_to_db(table)
+      tb_class = Object.const_get("::"+table[:class_name])
+      raise "Class not found:#{table[:class_name]}" unless tb_class <= ActiveRecord::Base
+      tb_class.destroy_all if table[:truncate_all_data]
+      table[:rows].each do |r|
+        if db_r = tb_class.find_by(table[:find_column] => r[table[:find_column]])
+          db_r.update(r) if table[:allow_overwrite]
+        else
+          tb_class.new(r).save!
+        end
+      end
+    end
+
+    SUPPORTED_FILE_EXT.each do |fx|
+      define_method "table_to_#{fx}" do |table|
+          dir = table[:export_file_location]
+          FileUtils.mkdir(dir) unless File.directory?(dir)
+          File.open("#{dir}/#{table[:table_name]}.#{fx}","w") do |f|
+            f.write public_send("convert_to_#{fx}",table)
+          end
+      end
+    end
+
+    def convert_to_csv(table)
+      CSV.generate do |csv|
+        csv << table[:header]
+        table[:rows].each do |row|
+          csv << table[:header].map {|col| row[col]}
+        end
+      end
+    end
+
+    [:yaml,:json].each do |output|
+      define_method "convert_to_#{output}" do |table|
+        table[:rows].public_send("to_#{output}")
+      end
+    end
+  end
+
+end

data/lib/takenoko/google_client.rb

@@ -0,0 +1,49 @@
+module Takenoko
+  class GoogleClient
+    def initialize(cridential_file)
+      @cridential = JSON.parse(File.read(cridential_file)).with_indifferent_access
+    end
+
+    def get_table(table_name)
+      table = Takenoko.table_config(table_name)
+      raise "GoogleDrive: Sheet not found" unless sheet = session.spreadsheet_by_key(table['sheet_id'])
+      raise "GoogleDrive: Worksheet not found: worksheet_id #{table['worksheet_id']}" unless worksheet = sheet.worksheet_by_gid(table['worksheet_id'])
+      header = worksheet.header.select {|h| table[:columns_mapping].keys.include?(h)}
+      rows = worksheet.populated_rows.map do |r|
+        hash = HashWithIndifferentAccess.new
+        table['columns_mapping'].each do |key,val|
+          begin
+            hash[key] = r.public_send(val)
+          rescue Exception => e
+            if key == 'id'
+              hash[key] = r.row_num - 1
+            else
+              hash[key] = nil
+            end
+          end
+        end
+        hash
+      end
+      table[:rows] = rows
+      return table
+    end
+
+    def session
+    key = OpenSSL::PKey::RSA.new(@cridential['private_key'])
+    auth = Signet::OAuth2::Client.new(
+      token_credential_uri: @cridential['token_uri'],
+      audience: @cridential['token_uri'],
+      scope: %w(
+        https://www.googleapis.com/auth/drive
+        https://spreadsheets.google.com/feeds/
+      ),
+      issuer: @cridential['client_email'],
+      signing_key: key
+    )
+
+    auth.fetch_access_token!
+    session = GoogleDrive.login_with_oauth(auth.access_token)
+    return session
+    end
+  end
+end

data/lib/takenoko/version.rb

@@ -0,0 +1,3 @@
+module Takenoko
+  VERSION = "0.0.4"
+end

data/lib/tasks/takenoko.rake

@@ -0,0 +1,11 @@
+namespace :takenoko do
+  (Takenoko::SUPPORTED_FILE_EXT.clone << :db ).each do |output|
+    task "all_to_#{output}".to_sym => :environment do
+      Takenoko.public_send("all_to_#{output}")
+    end
+
+    task "table_to_#{output}".to_sym, [:table] => :environment do |t, args|
+      Takenoko.public_send("table_to_#{output}",args[:table])
+    end
+  end
+end

data/test/takenoko_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class TakenokoTest < ActiveSupport::TestCase
+  test "truth" do
+    assert_kind_of Module, Takenoko
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,15 @@
+# Configure Rails Environment
+ENV["RAILS_ENV"] = "test"
+
+require File.expand_path("../dummy/config/environment.rb",  __FILE__)
+require "rails/test_help"
+
+Rails.backtrace_cleaner.remove_silencers!
+
+# Load support files
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
+
+# Load fixtures from the engine
+if ActiveSupport::TestCase.method_defined?(:fixture_path=)
+  ActiveSupport::TestCase.fixture_path = File.expand_path("../fixtures", __FILE__)
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: takenoko
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- KhiemNS
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.13
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: google_drive
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Import data from google spreadsheet to database or files
+email:
+- khiemns54@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- Rakefile
+- lib/generators/takenoko/config_generator.rb
+- lib/generators/takenoko/templates/takenoko.rb
+- lib/google_drive/google_drive.rb
+- lib/takenoko.rb
+- lib/takenoko/exporter.rb
+- lib/takenoko/google_client.rb
+- lib/takenoko/version.rb
+- lib/tasks/takenoko.rake
+- test/takenoko_test.rb
+- test/test_helper.rb
+homepage: https://github.com/khiemns54/takenoko
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Import data from google spreadsheet to database or files
+test_files:
+- test/takenoko_test.rb
+- test/test_helper.rb