takenoko 0.0.4 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/generators/takenoko/templates/takenoko.rb +34 -0
- data/lib/takenoko/google_client.rb +89 -19
- data/lib/takenoko/version.rb +1 -1
- data/lib/takenoko.rb +42 -27
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: afd7651c59b5026c0129a6844a0d4fc84a9e581c
|
4
|
+
data.tar.gz: dfe614608f5718772f113611324231918c4b1bf3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: da1b26dc90fc9409d533d03d70c41ec50824a575021ed9d7087fff3ed9f79eb37f68a6179472dc14ea5b2364c5e49af9926097402d9b0dede187593adf5b1572
|
7
|
+
data.tar.gz: 2b69692d6580d799e0e0d2f4571162e86cb064e5389db08936cc6bec88adad525e201819267ed98e06f0f37a67afe61c8679140a79f98c6cf32f8ef59d9b5fd2
|
@@ -1,4 +1,38 @@
|
|
1
1
|
Takenoko.config do |conf|
|
2
|
+
|
3
|
+
# Path to service account cridential file when using account service permission
|
4
|
+
# If not set, Takenoko will use persional cridential auto matically and as you for permission at the first time via command line
|
2
5
|
# conf.google_cridential_file = "path to google credential file.json"
|
6
|
+
|
7
|
+
# Path to mapping file, if not be set, Takenoko will fetch all worksheet and export base on worksheet
|
3
8
|
# conf.mapping_file = "path to mapping file.yml"
|
9
|
+
|
10
|
+
|
11
|
+
# GLOBAL CONFIG, you can overwrite under config for each table in mapping_file.yml
|
12
|
+
# Global Spreadsheet id, optain via url of your spreadsheet
|
13
|
+
# conf.sheet_id = "SPREADSHEET_ID" ###IMPORTANT###, Must set this value here or in mapping file
|
14
|
+
|
15
|
+
# Export file format, default :csv, support: csv,yaml,json
|
16
|
+
# conf.file_extension = :csv
|
17
|
+
|
18
|
+
# Truncate all data before saving, default: false
|
19
|
+
# conf.truncate_all_data = false
|
20
|
+
|
21
|
+
# Export file location, default: db/spreadsheet
|
22
|
+
# conf.export_file_location = "db/spreadsheet"
|
23
|
+
|
24
|
+
# Allow overwrite duplicated row, default: true
|
25
|
+
# conf.allow_overwrite = true
|
26
|
+
|
27
|
+
# Enable post processing after get row from spreadsheet, default false
|
28
|
+
# Define three more method on your post process class to handle it
|
29
|
+
# spreadsheet_row_valid? : skip invalid rows, return bool (Optional)
|
30
|
+
# postprocess_spreadsheet_row : modify row, return processed row(Optional)
|
31
|
+
# postprocess_spreadsheet_table : modify whole talbe, return processed table(Optional)
|
32
|
+
|
33
|
+
# conf.enable_postprocess = false
|
34
|
+
|
35
|
+
# Class for post processing, nil for Class = table class name
|
36
|
+
# conf.postprocess_class = nil
|
37
|
+
|
4
38
|
end
|
@@ -1,14 +1,29 @@
|
|
1
1
|
module Takenoko
|
2
2
|
class GoogleClient
|
3
|
-
def initialize(cridential_file)
|
4
|
-
|
3
|
+
def initialize(cridential_file=nil)
|
4
|
+
if cridential_file && ::File.exist?(cridential_file)
|
5
|
+
@cridential = JSON.parse(File.read(cridential_file)).with_indifferent_access
|
6
|
+
end
|
5
7
|
end
|
6
8
|
|
7
9
|
def get_table(table_name)
|
8
10
|
table = Takenoko.table_config(table_name)
|
9
11
|
raise "GoogleDrive: Sheet not found" unless sheet = session.spreadsheet_by_key(table['sheet_id'])
|
10
|
-
|
11
|
-
|
12
|
+
|
13
|
+
if table[:worksheet_id].present?
|
14
|
+
worksheet = sheet.worksheet_by_gid(table[:worksheet_id])
|
15
|
+
table[:worksheet] = worksheet.title
|
16
|
+
elsif table[:worksheet].present?
|
17
|
+
raise "Worksheet #{table[:worksheet]} not found" unless worksheet = sheet.worksheet_by_title(table[:worksheet])
|
18
|
+
table[:worksheet_id] = worksheet.gid.to_i
|
19
|
+
elsif
|
20
|
+
raise "You must specify worksheet or worksheet_id if mapping_file.yml"
|
21
|
+
end
|
22
|
+
|
23
|
+
update_table_config(table,worksheet.header)
|
24
|
+
postprocess_class = Object.const_get(table[:postprocess_class]) if table[:enable_postprocess]
|
25
|
+
|
26
|
+
Rails.logger.info "Getting table #{table_name}"
|
12
27
|
rows = worksheet.populated_rows.map do |r|
|
13
28
|
hash = HashWithIndifferentAccess.new
|
14
29
|
table['columns_mapping'].each do |key,val|
|
@@ -23,27 +38,82 @@ module Takenoko
|
|
23
38
|
end
|
24
39
|
end
|
25
40
|
hash
|
41
|
+
end.reject do |row|
|
42
|
+
begin
|
43
|
+
table[:enable_postprocess] && !postprocess_class.public_send("spreadsheet_row_valid?",row)
|
44
|
+
rescue NoMethodError => e
|
45
|
+
Rails.logger.warn e.message
|
46
|
+
false
|
47
|
+
end
|
48
|
+
end.map do |row|
|
49
|
+
begin
|
50
|
+
table[:enable_postprocess] ? postprocess_class.public_send("postprocess_spreadsheet_row",row) : row
|
51
|
+
rescue NoMethodError => e
|
52
|
+
Rails.logger.warn e.message
|
53
|
+
row
|
54
|
+
end
|
26
55
|
end
|
56
|
+
|
27
57
|
table[:rows] = rows
|
58
|
+
if table[:enable_postprocess]
|
59
|
+
begin
|
60
|
+
table = postprocess_class.public_send("postprocess_spreadsheet_table",table)
|
61
|
+
rescue NoMethodError => e
|
62
|
+
Rails.logger.warn e.message
|
63
|
+
end
|
64
|
+
end
|
28
65
|
return table
|
29
66
|
end
|
30
67
|
|
31
68
|
def session
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
69
|
+
Rails.logger.info "Init session"
|
70
|
+
unless @cridential
|
71
|
+
return GoogleDrive.saved_session(Takenoko.personal_cridential_file)
|
72
|
+
end
|
73
|
+
|
74
|
+
key = OpenSSL::PKey::RSA.new(@cridential['private_key'])
|
75
|
+
auth = Signet::OAuth2::Client.new(
|
76
|
+
token_credential_uri: @cridential['token_uri'],
|
77
|
+
audience: @cridential['token_uri'],
|
78
|
+
scope: %w(
|
79
|
+
https://www.googleapis.com/auth/drive
|
80
|
+
https://spreadsheets.google.com/feeds/
|
81
|
+
),
|
82
|
+
issuer: @cridential['client_email'],
|
83
|
+
signing_key: key
|
84
|
+
)
|
85
|
+
|
86
|
+
auth.fetch_access_token!
|
87
|
+
session = GoogleDrive.login_with_oauth(auth.access_token)
|
88
|
+
return session
|
89
|
+
end
|
90
|
+
|
91
|
+
def spreadsheet(sheet_id=Takenoko.sheet_id)
|
92
|
+
session.spreadsheet_by_key(sheet_id)
|
93
|
+
end
|
94
|
+
|
95
|
+
private
|
96
|
+
def update_table_config(table,ws_header)
|
97
|
+
columns_mapping = HashWithIndifferentAccess.new
|
98
|
+
ws_header.select do |col|
|
99
|
+
col.present? && ! col.match(/\s*#.*/)
|
100
|
+
end.each do |col|
|
101
|
+
if(table[:columns_mapping].present?)
|
102
|
+
columns_mapping[col] = col && next unless table[:columns_mapping][col].present?
|
103
|
+
next if table[:columns_mapping][col] == false
|
104
|
+
columns_mapping[col] = table[:columns_mapping][col]
|
105
|
+
else
|
106
|
+
columns_mapping[col] = col
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
table[:columns_mapping] = columns_mapping
|
111
|
+
unless table[:columns_mapping].key?(table[:find_column])
|
112
|
+
table[:columns_mapping][table[:find_column]] = table[:find_column]
|
113
|
+
end
|
114
|
+
|
115
|
+
table[:header] = table[:columns_mapping].keys
|
116
|
+
return table
|
47
117
|
end
|
48
118
|
end
|
49
119
|
end
|
data/lib/takenoko/version.rb
CHANGED
data/lib/takenoko.rb
CHANGED
@@ -7,6 +7,9 @@ module Takenoko
|
|
7
7
|
mattr_accessor :google_cridential_file
|
8
8
|
@@google_cridential_file = nil
|
9
9
|
|
10
|
+
mattr_accessor :personal_cridential_file
|
11
|
+
@@personal_cridential_file = "config/my_cridential.json"
|
12
|
+
|
10
13
|
mattr_accessor :mapping_file
|
11
14
|
@@mapping_file = false
|
12
15
|
|
@@ -26,6 +29,15 @@ module Takenoko
|
|
26
29
|
mattr_accessor :allow_overwrite
|
27
30
|
@@allow_overwrite = true
|
28
31
|
|
32
|
+
mattr_accessor :sheet_id
|
33
|
+
@@sheet_id = nil
|
34
|
+
|
35
|
+
mattr_accessor :enable_postprocess
|
36
|
+
@@enable_postprocess = false
|
37
|
+
|
38
|
+
mattr_accessor :postprocess_class
|
39
|
+
@@postprocess_class = nil
|
40
|
+
|
29
41
|
require 'takenoko/exporter'
|
30
42
|
require 'takenoko/google_client'
|
31
43
|
|
@@ -33,50 +45,53 @@ module Takenoko
|
|
33
45
|
yield self
|
34
46
|
end
|
35
47
|
|
36
|
-
def import
|
37
|
-
check_config
|
38
|
-
end
|
39
|
-
|
40
48
|
def google_client
|
41
49
|
@@google_client ||= GoogleClient.new(@@google_cridential_file)
|
42
50
|
return @@google_client
|
43
51
|
end
|
44
52
|
|
45
53
|
def check_config
|
46
|
-
raise "
|
47
|
-
raise "file not found:#{@@
|
48
|
-
raise "mapping_file cannot be nil" unless @@mapping_file
|
49
|
-
raise "file not found:#{@@mapping_file}" unless ::File.exist?(@@mapping_file)
|
54
|
+
raise "Must specify mapping_file or sheet_id" unless (@@mapping_file || @@sheet_id)
|
55
|
+
raise "file not found:#{@@mapping_file}" if @@mapping_file && !::File.exist?(@@mapping_file)
|
50
56
|
return true
|
51
57
|
end
|
52
58
|
|
53
59
|
def mapping_config
|
54
|
-
return
|
55
|
-
|
56
|
-
|
60
|
+
return unless check_config
|
61
|
+
if @@mapping_file
|
62
|
+
conf = YAML.load_file(@@mapping_file).with_indifferent_access
|
63
|
+
raise "tables not exists" if conf[:tables].blank?
|
64
|
+
else
|
65
|
+
conf = HashWithIndifferentAccess.new({tables: {}})
|
66
|
+
google_client.spreadsheet.worksheets.each do |ws|
|
67
|
+
next if ws.title.match(/\s*#.*/)
|
68
|
+
conf[:tables][ws.title] = {
|
69
|
+
worksheet_id: ws.gid,
|
70
|
+
worksheet: ws.title
|
71
|
+
}
|
72
|
+
end
|
73
|
+
end
|
57
74
|
conf[:tables].each do |t, v|
|
58
75
|
table = conf[:tables][t]
|
59
76
|
raise "Table config cannot be nil" unless table
|
60
|
-
[:sheet_id
|
61
|
-
|
62
|
-
end
|
77
|
+
raise "#{f} cannot be blank" unless table[:sheet_id] ||= @@sheet_id
|
78
|
+
table[:worksheet] = t if table[:worksheet].blank? && table[:worksheet_id].blank?
|
63
79
|
table[:find_column] = table[:find_column] || :id
|
64
|
-
table_name = table[:table_name] = t if table[:table_name].blank?
|
80
|
+
table_name = table[:table_name] = t.pluralize if table[:table_name].blank?
|
65
81
|
table[:class_name] = table[:class_name] || (table_name && table_name.singularize.camelize) || table[:table_name].singularize.camelize
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
table[
|
82
|
+
[
|
83
|
+
:allow_overwrite,
|
84
|
+
:truncate_all_data,
|
85
|
+
:file_extension,
|
86
|
+
:export_file_location,
|
87
|
+
:enable_postprocess,
|
88
|
+
:postprocess_class
|
89
|
+
].each do |f|
|
90
|
+
table[f] = class_variable_get("@@" + f.to_s) unless table.key?(f)
|
75
91
|
end
|
76
92
|
|
77
|
-
table[:
|
78
|
-
|
79
|
-
table[f] = class_variable_get("@@" + f.to_s) unless table.key?(f)
|
93
|
+
if table[:enable_postprocess] && table[:postprocess_class].blank?
|
94
|
+
table[:postprocess_class] = table[:class_name]
|
80
95
|
end
|
81
96
|
|
82
97
|
raise "Not support file extension: #{table[:file_extension]}" unless SUPPORTED_FILE_EXT.include?(table[:file_extension])
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: takenoko
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- KhiemNS
|
@@ -66,7 +66,7 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
-
description: Import data from google spreadsheet to database or files
|
69
|
+
description: Import data from google spreadsheet to database or files for Rails
|
70
70
|
email:
|
71
71
|
- khiemns54@gmail.com
|
72
72
|
executables: []
|
@@ -107,7 +107,7 @@ rubyforge_project:
|
|
107
107
|
rubygems_version: 2.5.1
|
108
108
|
signing_key:
|
109
109
|
specification_version: 4
|
110
|
-
summary: Import data from google spreadsheet to database or files
|
110
|
+
summary: Import data from google spreadsheet to database or files for Rails
|
111
111
|
test_files:
|
112
112
|
- test/takenoko_test.rb
|
113
113
|
- test/test_helper.rb
|