takenoko 0.0.4 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c6846bb822d84f908f35ea7e6737e87b20f5c17
-  data.tar.gz: 9c1533aaa9e1d993f7beb2f5ae8265f59a4fb697
+  metadata.gz: afd7651c59b5026c0129a6844a0d4fc84a9e581c
+  data.tar.gz: dfe614608f5718772f113611324231918c4b1bf3
 SHA512:
-  metadata.gz: 9d2627e910529f9c04a3b57477e46e4e738115a3cf8224f7c7e92151d471ac84c790265fa90a3c1e7688ea7e705fcb03ccbfd1885287d332b174c94a3be0cf6b
-  data.tar.gz: a700a1f350e7214fb1a1cb4d688d87ac0494531c7f50f7ee60ccfd4b77cc23cc3e4209bca8f3a8ba2f0a1186c326493c54699220d5e2d35da4d556b6985dad32
+  metadata.gz: da1b26dc90fc9409d533d03d70c41ec50824a575021ed9d7087fff3ed9f79eb37f68a6179472dc14ea5b2364c5e49af9926097402d9b0dede187593adf5b1572
+  data.tar.gz: 2b69692d6580d799e0e0d2f4571162e86cb064e5389db08936cc6bec88adad525e201819267ed98e06f0f37a67afe61c8679140a79f98c6cf32f8ef59d9b5fd2

data/lib/generators/takenoko/templates/takenoko.rb

@@ -1,4 +1,38 @@
 Takenoko.config do |conf|
+
+  # Path to service account cridential file when using account service permission
+  # If not set, Takenoko will use persional cridential auto matically and as you for permission at the first time via command line
   # conf.google_cridential_file = "path to google credential file.json"
+
+  # Path to mapping file, if not be set, Takenoko will fetch all worksheet and export base on worksheet
   # conf.mapping_file = "path to mapping file.yml"
+
+
+  # GLOBAL CONFIG, you can overwrite under config for each table in mapping_file.yml
+  # Global Spreadsheet id, optain via url of your spreadsheet
+  # conf.sheet_id = "SPREADSHEET_ID" ###IMPORTANT###, Must set this value here or in mapping file
+
+  # Export file format, default :csv, support: csv,yaml,json
+  # conf.file_extension = :csv
+
+  # Truncate all data before saving, default: false
+  # conf.truncate_all_data = false
+
+  # Export file location, default: db/spreadsheet
+  # conf.export_file_location = "db/spreadsheet"
+
+  # Allow overwrite duplicated row, default: true
+  # conf.allow_overwrite = true
+
+  # Enable post processing after get row from spreadsheet, default false
+  # Define three more method on your post process class to handle it
+  # spreadsheet_row_valid? : skip invalid rows, return bool (Optional)
+  # postprocess_spreadsheet_row : modify row, return processed row(Optional)
+  # postprocess_spreadsheet_table : modify whole talbe, return processed table(Optional)
+
+  # conf.enable_postprocess = false
+
+  # Class for post processing, nil for Class = table class name
+  # conf.postprocess_class = nil
+
 end

data/lib/takenoko/google_client.rb

@@ -1,14 +1,29 @@
 module Takenoko
   class GoogleClient
-    def initialize(cridential_file)
-      @cridential = JSON.parse(File.read(cridential_file)).with_indifferent_access
+    def initialize(cridential_file=nil)
+      if cridential_file && ::File.exist?(cridential_file)
+        @cridential = JSON.parse(File.read(cridential_file)).with_indifferent_access
+      end
     end
 
     def get_table(table_name)
       table = Takenoko.table_config(table_name)
       raise "GoogleDrive: Sheet not found" unless sheet = session.spreadsheet_by_key(table['sheet_id'])
-      raise "GoogleDrive: Worksheet not found: worksheet_id #{table['worksheet_id']}" unless worksheet = sheet.worksheet_by_gid(table['worksheet_id'])
-      header = worksheet.header.select {|h| table[:columns_mapping].keys.include?(h)}
+
+      if table[:worksheet_id].present?
+        worksheet = sheet.worksheet_by_gid(table[:worksheet_id])
+        table[:worksheet] = worksheet.title
+      elsif table[:worksheet].present?
+        raise "Worksheet #{table[:worksheet]} not found" unless worksheet = sheet.worksheet_by_title(table[:worksheet])
+        table[:worksheet_id] = worksheet.gid.to_i
+      elsif
+        raise "You must specify worksheet or worksheet_id if mapping_file.yml"
+      end
+
+      update_table_config(table,worksheet.header)
+      postprocess_class = Object.const_get(table[:postprocess_class]) if table[:enable_postprocess] 
+
+      Rails.logger.info "Getting table #{table_name}"
       rows = worksheet.populated_rows.map do |r|
         hash = HashWithIndifferentAccess.new
         table['columns_mapping'].each do |key,val|
@@ -23,27 +38,82 @@ module Takenoko
           end
         end
         hash
+      end.reject do |row|
+        begin
+          table[:enable_postprocess] && !postprocess_class.public_send("spreadsheet_row_valid?",row)
+        rescue NoMethodError => e
+          Rails.logger.warn e.message
+          false
+        end
+      end.map do |row|
+        begin
+          table[:enable_postprocess] ? postprocess_class.public_send("postprocess_spreadsheet_row",row) : row
+        rescue NoMethodError => e
+          Rails.logger.warn e.message
+          row
+        end
       end
+
       table[:rows] = rows
+      if table[:enable_postprocess]
+        begin
+          table = postprocess_class.public_send("postprocess_spreadsheet_table",table) 
+        rescue NoMethodError => e
+          Rails.logger.warn e.message
+        end
+      end
       return table
     end
 
     def session
-    key = OpenSSL::PKey::RSA.new(@cridential['private_key'])
-    auth = Signet::OAuth2::Client.new(
-      token_credential_uri: @cridential['token_uri'],
-      audience: @cridential['token_uri'],
-      scope: %w(
-        https://www.googleapis.com/auth/drive
-        https://spreadsheets.google.com/feeds/
-      ),
-      issuer: @cridential['client_email'],
-      signing_key: key
-    )
-
-    auth.fetch_access_token!
-    session = GoogleDrive.login_with_oauth(auth.access_token)
-    return session
+      Rails.logger.info "Init session"
+      unless @cridential
+        return GoogleDrive.saved_session(Takenoko.personal_cridential_file)
+      end
+
+      key = OpenSSL::PKey::RSA.new(@cridential['private_key'])
+      auth = Signet::OAuth2::Client.new(
+        token_credential_uri: @cridential['token_uri'],
+        audience: @cridential['token_uri'],
+        scope: %w(
+          https://www.googleapis.com/auth/drive
+          https://spreadsheets.google.com/feeds/
+        ),
+        issuer: @cridential['client_email'],
+        signing_key: key
+      )
+
+      auth.fetch_access_token!
+      session = GoogleDrive.login_with_oauth(auth.access_token)
+      return session
+    end
+
+    def spreadsheet(sheet_id=Takenoko.sheet_id)
+      session.spreadsheet_by_key(sheet_id)
+    end
+
+    private
+    def update_table_config(table,ws_header)
+      columns_mapping = HashWithIndifferentAccess.new
+      ws_header.select do |col|
+        col.present? && ! col.match(/\s*#.*/)
+      end.each do |col|
+        if(table[:columns_mapping].present?)
+          columns_mapping[col] = col && next unless table[:columns_mapping][col].present?
+          next if table[:columns_mapping][col] == false
+          columns_mapping[col] = table[:columns_mapping][col]
+        else
+          columns_mapping[col] = col
+        end
+      end
+
+      table[:columns_mapping] = columns_mapping
+      unless table[:columns_mapping].key?(table[:find_column])
+          table[:columns_mapping][table[:find_column]] = table[:find_column]
+      end
+
+      table[:header] = table[:columns_mapping].keys
+      return table
     end
   end
 end

data/lib/takenoko/version.rb

@@ -1,3 +1,3 @@
 module Takenoko
-  VERSION = "0.0.4"
+  VERSION = "0.1.2"
 end

data/lib/takenoko.rb

@@ -7,6 +7,9 @@ module Takenoko
   mattr_accessor :google_cridential_file
   @@google_cridential_file = nil
 
+  mattr_accessor :personal_cridential_file
+  @@personal_cridential_file = "config/my_cridential.json"
+
   mattr_accessor :mapping_file
   @@mapping_file = false
 
@@ -26,6 +29,15 @@ module Takenoko
   mattr_accessor :allow_overwrite
   @@allow_overwrite = true
 
+  mattr_accessor :sheet_id
+  @@sheet_id = nil
+
+  mattr_accessor :enable_postprocess
+  @@enable_postprocess = false
+
+  mattr_accessor :postprocess_class
+  @@postprocess_class = nil
+
   require 'takenoko/exporter'
   require 'takenoko/google_client'
 
@@ -33,50 +45,53 @@ module Takenoko
     yield self
   end
 
-  def import
-    check_config
-  end
-
   def google_client
     @@google_client ||= GoogleClient.new(@@google_cridential_file)
     return @@google_client
   end
 
   def check_config
-    raise "google_cridential_file setting cannot be nil" unless @@google_cridential_file
-    raise "file not found:#{@@google_cridential_file}" unless ::File.exist?(@@google_cridential_file)
-    raise "mapping_file cannot be nil" unless @@mapping_file
-    raise "file not found:#{@@mapping_file}" unless ::File.exist?(@@mapping_file)
+    raise "Must specify mapping_file or sheet_id" unless (@@mapping_file || @@sheet_id)
+    raise "file not found:#{@@mapping_file}" if @@mapping_file && !::File.exist?(@@mapping_file)
     return true
   end
 
   def mapping_config
-    return @@mapping_config if @@mapping_config
-    conf = YAML.load_file(@@mapping_file).with_indifferent_access
-    raise "tables not exists" if conf[:tables].blank?
+    return unless check_config
+    if @@mapping_file
+      conf = YAML.load_file(@@mapping_file).with_indifferent_access
+      raise "tables not exists" if conf[:tables].blank?
+    else
+      conf = HashWithIndifferentAccess.new({tables: {}})
+      google_client.spreadsheet.worksheets.each do |ws|
+        next if ws.title.match(/\s*#.*/)
+        conf[:tables][ws.title] = {
+          worksheet_id: ws.gid,
+          worksheet: ws.title
+        }
+      end
+    end
     conf[:tables].each do |t, v|
       table = conf[:tables][t]
       raise "Table config cannot be nil" unless table
-      [:sheet_id, :worksheet_id, :columns_mapping].each do |f|
-        raise "#{f} cannot be blank" if table[f].blank?
-      end
+      raise "#{f} cannot be blank" unless table[:sheet_id] ||= @@sheet_id
+      table[:worksheet] = t if table[:worksheet].blank? && table[:worksheet_id].blank?
       table[:find_column] = table[:find_column] || :id
-      table_name = table[:table_name] = t if table[:table_name].blank?
+      table_name = table[:table_name] = t.pluralize if table[:table_name].blank?
       table[:class_name] = table[:class_name] || (table_name && table_name.singularize.camelize) || table[:table_name].singularize.camelize
-      unless table[:columns_mapping].key?(table[:find_column])
-        if(table[:find_column] == :id)
-          table[:columns_mapping] = {id: nil}.merge!(table[:columns_mapping])
-        else
-          table[:columns_mapping][table[:find_column]] = nil 
-        end
-      end
-      table[:columns_mapping].each do |s_col,db_col|
-        table[:columns_mapping][s_col] = s_col unless db_col 
+      [
+        :allow_overwrite,
+        :truncate_all_data,
+        :file_extension,
+        :export_file_location,
+        :enable_postprocess,
+        :postprocess_class
+      ].each do |f|
+        table[f] = class_variable_get("@@" + f.to_s) unless table.key?(f)
       end
 
-      table[:header] = table[:columns_mapping].keys
-      [:allow_overwrite,:truncate_all_data, :file_extension, :export_file_location].each do |f|
-        table[f] = class_variable_get("@@" + f.to_s) unless table.key?(f)
+      if table[:enable_postprocess] && table[:postprocess_class].blank?
+        table[:postprocess_class] = table[:class_name]
       end
 
       raise "Not support file extension: #{table[:file_extension]}" unless SUPPORTED_FILE_EXT.include?(table[:file_extension])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: takenoko
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.2
 platform: ruby
 authors:
 - KhiemNS
@@ -66,7 +66,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Import data from google spreadsheet to database or files
+description: Import data from google spreadsheet to database or files for Rails
 email:
 - khiemns54@gmail.com
 executables: []
@@ -107,7 +107,7 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Import data from google spreadsheet to database or files
+summary: Import data from google spreadsheet to database or files for Rails
 test_files:
 - test/takenoko_test.rb
 - test/test_helper.rb