recon_tools 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +5 -0
- data/LICENSE +21 -0
- data/README.md +53 -0
- data/Rakefile +18 -0
- data/lib/recon_services.rb +75 -0
- data/lib/recon_tools/google_sheets_connect.rb +135 -0
- data/lib/recon_tools/jira_connect.rb +105 -0
- data/lib/recon_tools/version.rb +3 -0
- data/lib/recon_tools.rb +146 -0
- data/recon_tools.gemspec +41 -0
- metadata +168 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 5569c3da1aed08b4c3aebecd24ffc7fc835d3ff15bf4834d0b4e1216b5d0d87b
|
4
|
+
data.tar.gz: 7a3df1fe4067f8e9a12b1c2f2cd8e1d90bae71e3c930330d05b90f30cfb840dd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 62be24d6d7835681cf131eca7795e3bd29b6850beebe40ff287266c955aa0b2b93d96dffe87297857c363b7e1155e77264ff119e7c5750abb743280af1d1fd84
|
7
|
+
data.tar.gz: 8cc24f10e651970b1d57fe93e2fd00d89d27a8be38f105fdf49ac5eeec365bc8cbf75170ba7d188358af1c4528247cea657815c78c6ad68c623db2dc21daa182
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2021 Chris Rowe
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
# recon_tools
|
2
|
+
recon_tools: Tools to support importing, exporting and reconciliation of data from JIRA, GoogleSheets and more..
|
3
|
+
|
4
|
+
## Functional Todos
|
5
|
+
|
6
|
+
|
7
|
+
## Project Todos
|
8
|
+
- Install rubocop
|
9
|
+
- Release gem
|
10
|
+
- Ensure correct relative paths etc for includes
|
11
|
+
|
12
|
+
## Accessing Google sheets
|
13
|
+
|
14
|
+
In order to access Google Sheets you need to provide credentials to the script.
|
15
|
+
|
16
|
+
#### Create credentials.json file and put in config directory
|
17
|
+
|
18
|
+
Follow instructions below and put the credentials.json file in the config directory.
|
19
|
+
|
20
|
+
Video Example: https://www.youtube.com/watch?v=VqoSUSy011I
|
21
|
+
Related Blog Post: https://www.twilio.com/blog/2017/03/google-spreadsheets-ruby.html (Slightly old UI so see instrucitons below)
|
22
|
+
|
23
|
+
Official Instructions: https://developers.google.com/workspace/guides/create-credentials
|
24
|
+
|
25
|
+
- To obtain credentials for your service account:
|
26
|
+
- Click on your newly-create service account.
|
27
|
+
- Click Keys.
|
28
|
+
- Click Add key > Create new key. The "Create private key" dialog appears.
|
29
|
+
- Select JSON.
|
30
|
+
- Click Create. ...
|
31
|
+
- Click Close.
|
32
|
+
|
33
|
+
#### Give Access To The Specific Sheet
|
34
|
+
|
35
|
+
Share the GoogleSheet you want to access with the email listed in the **<client_email>** section of the email
|
36
|
+
|
37
|
+
## Accessing JIRA sheets
|
38
|
+
|
39
|
+
Instructions: https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
|
40
|
+
|
41
|
+
- Log in to https://id.atlassian.com/manage/api-tokens.
|
42
|
+
- Click 'Create API token.'
|
43
|
+
- From the dialog that appears, enter a memorable and concise 'Label' for your token and click 'Create.'
|
44
|
+
- Use 'Copy to clipboard' and paste the token into the JIRA API token field on the JIRA account user page.
|
45
|
+
|
46
|
+
#### Set Environment Variables in shell
|
47
|
+
|
48
|
+
```
|
49
|
+
export RECON_TOOLS_JIRA_TOKEN=<TOKEN>
|
50
|
+
echo "$RECON_TOOLS_JIRA_TOKEN"
|
51
|
+
export RECON_TOOLS_JIRA_EMAIL=<EMAIL>
|
52
|
+
echo "$RECON_TOOLS_JIRA_EMAIL"
|
53
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'rake/testtask'
|
2
|
+
require './lib/recon_tools'
|
3
|
+
|
4
|
+
task :test do
|
5
|
+
Rake::TestTask.new do |t|
|
6
|
+
t.pattern = 'test/**/test_*.rb'
|
7
|
+
#t.libs << 'test'
|
8
|
+
t.verbose = true
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
desc "Run e2e Test Manually"
|
13
|
+
task :e2e2 do
|
14
|
+
ruby "./test/e2e_test_recon_tools.rb"
|
15
|
+
end
|
16
|
+
|
17
|
+
desc "Run tests"
|
18
|
+
task :default => :test
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'logging'
|
2
|
+
# https://dev.to/exampro/testunit-writing-test-code-in-ruby-part-1-of-3-44m2
|
3
|
+
|
4
|
+
class ReconServices
|
5
|
+
def initialize
|
6
|
+
@logger = Logging.logger(STDOUT)
|
7
|
+
@logger.level = :info
|
8
|
+
end
|
9
|
+
|
10
|
+
def get_jira_data(company_url_base = "leadtechie", project = "TEST", jira_col_start, jira_col_end)
|
11
|
+
token = ENV['RECON_TOOLS_JIRA_TOKEN']
|
12
|
+
email = ENV['RECON_TOOLS_JIRA_EMAIL']
|
13
|
+
#if token="" or email= ""
|
14
|
+
assert_not_equal nil, token, "Token not set as environment variable"
|
15
|
+
assert_not_equal nil, email, "email not set as environment variable"
|
16
|
+
jira_connect = JiraConnect.new(email, token)
|
17
|
+
|
18
|
+
|
19
|
+
components = jira_connect.get_jira_components company_url_base, project
|
20
|
+
|
21
|
+
components_from_jira = jira_connect.parseComponentsJSON(JSON.pretty_generate(components))
|
22
|
+
#components_from_jira = components_from_jira.each { |e| e.delete_at(0)}
|
23
|
+
components_from_jira = components_from_jira.map { |e| e[jira_col_start..jira_col_end]}
|
24
|
+
end
|
25
|
+
|
26
|
+
# Connects to JIRA and gets list of components_from_jira
|
27
|
+
# Connects to GoogleSheets and reads list from matched_records
|
28
|
+
# Does a reconciliation of the data
|
29
|
+
# Makes updates to the data on the sheet with the new data from JIRA
|
30
|
+
# Rereads data from the sheet and confirms the same as from JIRA
|
31
|
+
def jira_googlesheets_reconcile_and_update(sheet_name, tab_number, sheet_col_start, sheet_col_end, copy_flag=false, write_change_log=false, google_credentials_file,
|
32
|
+
company_url_base, project, jira_col_start, jira_col_end)
|
33
|
+
@logger.info "Starting e2e tests in jira_googlesheets_reconcile_and_update"
|
34
|
+
@logger.info "Connecting to JIRA and gets list of components_from_jira"
|
35
|
+
components_from_jira = get_jira_data company_url_base, project, jira_col_start, jira_col_end
|
36
|
+
#puts Dir.pwd
|
37
|
+
@logger.info "Connecting to GoogleSheets and reads list from matched_records"
|
38
|
+
googlesheets_connect = GoogleSheetsConnect.new(google_credentials_file)
|
39
|
+
sheet_data = googlesheets_connect.read_sheet_data sheet_name, 0, 0, sheet_col_end
|
40
|
+
sheet_data = sheet_data.map { |e| e[sheet_col_start..sheet_col_end]}
|
41
|
+
|
42
|
+
@logger.debug "sheet_data"
|
43
|
+
@logger.debug sheet_data
|
44
|
+
@logger.debug "sheet_data"
|
45
|
+
|
46
|
+
@logger.debug "components_from_jira"
|
47
|
+
@logger.debug components_from_jira
|
48
|
+
@logger.debug "components_from_jira"
|
49
|
+
|
50
|
+
@logger.info "Reconciling the data"
|
51
|
+
recon_tools = ReconTools.new(sheet_data, components_from_jira)
|
52
|
+
|
53
|
+
|
54
|
+
|
55
|
+
sheet_id = tab_number
|
56
|
+
|
57
|
+
if copy_flag
|
58
|
+
#Duplicate original data
|
59
|
+
sheet_id = googlesheets_connect.duplicate_worksheet(sheet_name, tab_number)
|
60
|
+
end
|
61
|
+
|
62
|
+
@logger.info "Updating he data on the sheet with the new data from JIRA"
|
63
|
+
googlesheets_connect.update_specific_cells(recon_tools.updates, sheet_name, sheet_id, sheet_col_start)
|
64
|
+
|
65
|
+
#sheet_data_new = googlesheets_connect.read_sheet_data sheet_name, sheet_id, 0, 5
|
66
|
+
#sheet_data_new.each { |e| e.delete_at(0)}
|
67
|
+
#assert_equal components_from_jira, sheet_data_new, "Compare updated data from sheet with JIRA"
|
68
|
+
|
69
|
+
if write_change_log
|
70
|
+
googlesheets_connect.write_column recon_tools.changelog, sheet_name, sheet_id, sheet_col_end
|
71
|
+
end
|
72
|
+
|
73
|
+
@logger.info "ending e2e tests in jira_googlesheets_reconcile_and_update"
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
# FileImporter
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
# Googlesheets
|
5
|
+
require "google_drive"
|
6
|
+
require "google/apis/sheets_v4"
|
7
|
+
require "google_docs"
|
8
|
+
|
9
|
+
class GoogleSheetsConnect
|
10
|
+
@session
|
11
|
+
def initialize(json_crednetials = "config/credentials.json")
|
12
|
+
#credentials = File.read("../config/credentials.json")
|
13
|
+
#puts credentials
|
14
|
+
@session = GoogleDrive::Session.from_service_account_key(json_crednetials)
|
15
|
+
end
|
16
|
+
|
17
|
+
def hello()
|
18
|
+
puts "Hello"
|
19
|
+
"Hello"
|
20
|
+
end
|
21
|
+
|
22
|
+
def duplicate_worksheet(sheet_name, tabNumber=0)
|
23
|
+
spreadsheet = @session.spreadsheet_by_title(sheet_name)
|
24
|
+
worksheet_source = spreadsheet.worksheets[tabNumber]
|
25
|
+
spreadsheet.add_worksheet("Tab #{spreadsheet.worksheets.length+1}")
|
26
|
+
data_to_add = unfreeze_array(worksheet_source.rows)
|
27
|
+
|
28
|
+
worksheet_target=spreadsheet.worksheets[spreadsheet.worksheets.length-1]
|
29
|
+
worksheet_target.insert_rows(1 , data_to_add)
|
30
|
+
worksheet_target.save
|
31
|
+
spreadsheet.worksheets.length-1
|
32
|
+
end
|
33
|
+
|
34
|
+
def insert_data(dataToInsert, sheetName)
|
35
|
+
spreadsheet = @session.spreadsheet_by_title(sheetName)
|
36
|
+
worksheet = spreadsheet.worksheets.first
|
37
|
+
# worksheet.rows.each { |row| puts row.first(6).join(" | ") }
|
38
|
+
|
39
|
+
worksheet.insert_rows(1 , dataToInsert)
|
40
|
+
worksheet.save
|
41
|
+
end
|
42
|
+
|
43
|
+
def update_data(data_to_insert, sheet_name, tabNumber=0)
|
44
|
+
spreadsheet = @session.spreadsheet_by_title(sheet_name)
|
45
|
+
worksheet = spreadsheet.worksheets[tabNumber]
|
46
|
+
|
47
|
+
data_to_insert.each_with_index do |row, rowIndex|
|
48
|
+
row.each_with_index do |value, colIndex|
|
49
|
+
worksheet[rowIndex+1, colIndex+1]= value
|
50
|
+
puts "#{rowIndex}, #{colIndex} = #{value}"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
worksheet.save
|
54
|
+
end
|
55
|
+
|
56
|
+
def update_specific_cells(data_to_insert, sheet_name, tabNumber=0, colOffset=0)
|
57
|
+
spreadsheet = @session.spreadsheet_by_title(sheet_name)
|
58
|
+
worksheet = spreadsheet.worksheets[tabNumber]
|
59
|
+
|
60
|
+
data_to_insert.each do |update|
|
61
|
+
worksheet[update[0]+1, update[1]+1+colOffset]= update[2]
|
62
|
+
#puts "#{row}, #{col} = #{value}"
|
63
|
+
end
|
64
|
+
worksheet.save
|
65
|
+
end
|
66
|
+
|
67
|
+
def write_column(data_to_insert, sheet_name, tabNumber=0, colOffset=0)
|
68
|
+
spreadsheet = @session.spreadsheet_by_title(sheet_name)
|
69
|
+
worksheet = spreadsheet.worksheets[tabNumber]
|
70
|
+
|
71
|
+
data_to_insert.each_with_index do |value, index|
|
72
|
+
worksheet[index+1, 1+colOffset]= "#{DateTime::now} #{value} \n"+ worksheet[index+1, 1+colOffset]
|
73
|
+
end
|
74
|
+
worksheet.save
|
75
|
+
end
|
76
|
+
|
77
|
+
def read_sheet_data(sheetName, tabNumber=0, skip_rows=0, columns=0)
|
78
|
+
spreadsheet = @session.spreadsheet_by_title(sheetName)
|
79
|
+
puts sheetName
|
80
|
+
worksheet = spreadsheet.worksheets[tabNumber]
|
81
|
+
puts worksheet
|
82
|
+
data = []
|
83
|
+
frozen_data = worksheet.rows
|
84
|
+
data = unfreeze_array(frozen_data)
|
85
|
+
if columns>0
|
86
|
+
data2 = []
|
87
|
+
data.each do |row|
|
88
|
+
data2 << row.first(columns)
|
89
|
+
end
|
90
|
+
else
|
91
|
+
data2 = data
|
92
|
+
end
|
93
|
+
data2
|
94
|
+
end
|
95
|
+
|
96
|
+
def unfreeze_array(arrayIn2d)
|
97
|
+
unfrozen_array = []
|
98
|
+
arrayIn2d.each do |row|
|
99
|
+
new_row = []
|
100
|
+
row.each do |element|
|
101
|
+
new_row.push(element)
|
102
|
+
end
|
103
|
+
unfrozen_array.push(new_row)
|
104
|
+
end
|
105
|
+
unfrozen_array
|
106
|
+
end
|
107
|
+
|
108
|
+
def update_test_sheet(sheetName, tabNumber=0)
|
109
|
+
#BEST Example: https://www.youtube.com/watch?v=VqoSUSy011I
|
110
|
+
#https://www.twilio.com/blog/2017/03/google-spreadsheets-ruby.html
|
111
|
+
|
112
|
+
# Authenticate a session with your Service Account
|
113
|
+
#session = GoogleDrive::Session.from_service_account_key("../../config/credentials.json")
|
114
|
+
#https://docs.google.com/spreadsheets/d/1_<ID>/edit#gid=152787366
|
115
|
+
|
116
|
+
# Get the spreadsheet by its title
|
117
|
+
spreadsheet = session.spreadsheet_by_title(sheetName)
|
118
|
+
# Get the first worksheet
|
119
|
+
worksheet = spreadsheet.worksheets[tabNumber]
|
120
|
+
# Print out the first 6 columns of each row
|
121
|
+
worksheet.rows.each { |row| puts row.first(6).join(" | ") }
|
122
|
+
|
123
|
+
worksheet.insert_rows(worksheet.num_rows + 1 ,
|
124
|
+
[
|
125
|
+
["col1", "col2"],
|
126
|
+
["col2", "col3"]
|
127
|
+
]
|
128
|
+
)
|
129
|
+
|
130
|
+
worksheet.save
|
131
|
+
worksheet["C5"] = "updated"
|
132
|
+
|
133
|
+
worksheet.save
|
134
|
+
end
|
135
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
#https://medium.com/building-rigup/wrangling-google-services-in-ruby-5493e906c84f
|
4
|
+
#https://spin.atomicobject.com/2016/07/16/google-sheets-api-ruby/
|
5
|
+
#https://www.botreetechnologies.com/blog/google-api-authorization-with-ruby-using-long-lasting-refresh-token
|
6
|
+
|
7
|
+
require 'net/http'
|
8
|
+
require 'net/https'
|
9
|
+
require 'uri'
|
10
|
+
require 'json'
|
11
|
+
require 'date'
|
12
|
+
|
13
|
+
class JiraConnect
|
14
|
+
@email = ""
|
15
|
+
@token = ""
|
16
|
+
|
17
|
+
def initialize(email, token)
|
18
|
+
@email = email
|
19
|
+
@token = token
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
def get_jira_components(company_url_base, project)
|
24
|
+
uriString="https://#{company_url_base}.atlassian.net/rest/api/3/project/#{project}/components"
|
25
|
+
data = get_jira_api(uriString)
|
26
|
+
#puts data
|
27
|
+
data
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
def get_jira_issues(company_url_base, project)
|
32
|
+
uriString="https://#{company_url_base}.atlassian.net/rest/api/2/search?jql=project=#{project}&maxResults=1000"
|
33
|
+
data = get_jira_api(uriString)
|
34
|
+
#puts data
|
35
|
+
data
|
36
|
+
end
|
37
|
+
|
38
|
+
def parseComponentsJSON(data)
|
39
|
+
results = []
|
40
|
+
jsondata=JSON.parse(data)
|
41
|
+
jsondata.each do |hash|
|
42
|
+
results.push( [ DateTime::now,
|
43
|
+
hash["id"],
|
44
|
+
hash["name"],
|
45
|
+
hash['lead'] ? hash['lead']['displayName'] : "<No Owner>",
|
46
|
+
hash['description']?hash['description']:'<No Description>'
|
47
|
+
])
|
48
|
+
end
|
49
|
+
results
|
50
|
+
end
|
51
|
+
|
52
|
+
def save_jira_components(data, fileName)
|
53
|
+
File.open("sample_data/cache/"+fileName, 'w') {|f| f.write(JSON.pretty_generate(data)) }
|
54
|
+
end
|
55
|
+
|
56
|
+
def get_cached_json(fileName)
|
57
|
+
data = File.read("test/sample_data/cache/"+fileName)
|
58
|
+
data
|
59
|
+
end
|
60
|
+
|
61
|
+
def get_sample_json(fileName)
|
62
|
+
#puts "Current Directory"
|
63
|
+
#puts Dir.pwd
|
64
|
+
data = File.read("test/sample_data/"+fileName)
|
65
|
+
data
|
66
|
+
end
|
67
|
+
|
68
|
+
def get_jira_api(uriIn)
|
69
|
+
uri = URI(uriIn)
|
70
|
+
|
71
|
+
Net::HTTP.start(uri.host, uri.port,
|
72
|
+
:use_ssl => uri.scheme == 'https',
|
73
|
+
:verify_mode => OpenSSL::SSL::VERIFY_NONE) do |http|
|
74
|
+
|
75
|
+
request = Net::HTTP::Get.new uri.request_uri
|
76
|
+
request.basic_auth @email, @token
|
77
|
+
response = http.request request # Net::HTTPResponse object
|
78
|
+
|
79
|
+
#puts response.body
|
80
|
+
|
81
|
+
data = JSON.parse(response.body)
|
82
|
+
data
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def get_jira_api2(dataIn)
|
87
|
+
dataOut = []
|
88
|
+
data.each do |component|
|
89
|
+
dataOut << [component['name'], component['displayName']]
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# def get_jira_api(dataIn)
|
94
|
+
# data.each do |child|
|
95
|
+
# puts child['id'] + " " + child['name'] + " " + ( child['lead'] ? child['lead']['displayName'] : "(No Owner)") + " " + (child['description']?child['description']:'No Description')
|
96
|
+
# end
|
97
|
+
# end
|
98
|
+
|
99
|
+
def parse_component_list()
|
100
|
+
data.each do |component|
|
101
|
+
puts component['name']
|
102
|
+
puts component['displayName']
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
data/lib/recon_tools.rb
ADDED
@@ -0,0 +1,146 @@
|
|
1
|
+
# https://dev.to/exampro/testunit-writing-test-code-in-ruby-part-1-of-3-44m2
|
2
|
+
|
3
|
+
class ReconTools
|
4
|
+
attr_reader :array_data_in1, :array_data_in2,
|
5
|
+
:new_records, :deleted_records, :updated_records, :matched_records, :duplicate_data1_records, :duplicate_data2_records,
|
6
|
+
:updated_array, :updates, :changelog
|
7
|
+
|
8
|
+
def hello_world_test()
|
9
|
+
puts "Hello World"
|
10
|
+
end
|
11
|
+
|
12
|
+
def initialize(array_data_in1, array_data_in2)
|
13
|
+
@data_in1 = {}
|
14
|
+
@data_in2 = {}
|
15
|
+
@matched_records = {}
|
16
|
+
@updated_records = {}
|
17
|
+
@new_records = {}
|
18
|
+
@deleted_records = {}
|
19
|
+
@duplicate_data1_records = []
|
20
|
+
@duplicate_data2_records = []
|
21
|
+
@updated_array = []
|
22
|
+
@updates = []
|
23
|
+
|
24
|
+
@array_data_in1 = array_data_in1
|
25
|
+
@array_data_in2 = array_data_in2
|
26
|
+
@data_in1, @duplicate_data1_records = importArray(array_data_in1)
|
27
|
+
@data_in2, @duplicate_data2_records = importArray(array_data_in2)
|
28
|
+
|
29
|
+
@matched_records, @updated_records = get_matching_records(@data_in1, @data_in2)
|
30
|
+
@new_records = get_only_in_datain1_records(@data_in1, @data_in2)
|
31
|
+
@deleted_records = get_only_in_datain1_records(@data_in2, @data_in1)
|
32
|
+
@updated_array, @changelog = update_data(@array_data_in1)
|
33
|
+
@updates = generate_updates(@updated_array)
|
34
|
+
#@deleted_records = get_deleted_records(@data_in1, @data_in2)
|
35
|
+
end
|
36
|
+
|
37
|
+
# Remove duplicates and collect list of duplicates
|
38
|
+
def importArray(arrayIn)
|
39
|
+
data_hash = {}
|
40
|
+
duplicates = []
|
41
|
+
arrayIn.each{ |record|
|
42
|
+
if data_hash.key?(record[0])
|
43
|
+
duplicates << record
|
44
|
+
else
|
45
|
+
data_hash[record[0]]=record
|
46
|
+
end
|
47
|
+
}
|
48
|
+
return data_hash, duplicates
|
49
|
+
end
|
50
|
+
|
51
|
+
# Update data1 by updating records, removing deleted ones and adding new ones
|
52
|
+
def update_data(arrayIn)
|
53
|
+
changelog = []
|
54
|
+
updated_array = []
|
55
|
+
arrayIn.each{ |record|
|
56
|
+
if @data_in2.key?(record[0])
|
57
|
+
#puts "updated data"
|
58
|
+
#update changed records
|
59
|
+
updated_array << @data_in2[record[0]]
|
60
|
+
if @data_in2[record[0]] == record
|
61
|
+
changelog << "No change #{record}"
|
62
|
+
else
|
63
|
+
changelog << "Updated from #{record}"
|
64
|
+
end
|
65
|
+
elsif @deleted_records.key?(record[0])
|
66
|
+
#puts "deleted data"
|
67
|
+
#puts Array.new record.size, ""
|
68
|
+
#puts record
|
69
|
+
updated_array << (Array.new record.size, "")
|
70
|
+
changelog << "Deleted #{record}"
|
71
|
+
end
|
72
|
+
}
|
73
|
+
@new_records.each { |key,value|
|
74
|
+
changelog << "New #{value}"
|
75
|
+
updated_array << value
|
76
|
+
}
|
77
|
+
|
78
|
+
return updated_array, changelog
|
79
|
+
end
|
80
|
+
|
81
|
+
def generate_updates(updated_array)
|
82
|
+
updates = []
|
83
|
+
#updates_with_diff = []
|
84
|
+
updated_array.each_with_index { |val, index|
|
85
|
+
#puts " updated_array #{val}"
|
86
|
+
val.each_with_index { |val2, index2|
|
87
|
+
#puts " val2 #{val2}"
|
88
|
+
if @array_data_in1.length > index and @array_data_in1[index][index2] == val2
|
89
|
+
#puts "matches "
|
90
|
+
#updates << [index, index2, val2]
|
91
|
+
else
|
92
|
+
#puts "doesn't matche"
|
93
|
+
updates << [index, index2, updated_array[index][index2]]
|
94
|
+
#updates_with_diff << [index, index2, "Updated #{@array_data_in1[index][index2]} #{updated_array[index][index2]]}"]
|
95
|
+
end
|
96
|
+
}
|
97
|
+
}
|
98
|
+
#puts "Updates #{updates}"
|
99
|
+
#puts "end generate_updates"
|
100
|
+
updates #, updates_with_diff
|
101
|
+
end
|
102
|
+
|
103
|
+
def get_matching_records(data_in1, data_in2)
|
104
|
+
matches = data_in1.keys.filter { |key| key if data_in2.has_key?(key) }
|
105
|
+
matches.each { |key|
|
106
|
+
if data_in1[key] == data_in2[key]
|
107
|
+
matched_records[key]=data_in1[key]
|
108
|
+
else
|
109
|
+
updated_records[key] = data_in2[key]
|
110
|
+
end
|
111
|
+
}
|
112
|
+
return matched_records, updated_records
|
113
|
+
end
|
114
|
+
|
115
|
+
def get_only_in_datain1_records(data_in1, data_in2)
|
116
|
+
keys = data_in2.keys - data_in1.keys
|
117
|
+
data1_records = {}
|
118
|
+
keys.each { |key|
|
119
|
+
data1_records[key]=data_in2[key]
|
120
|
+
}
|
121
|
+
#puts new_records
|
122
|
+
data1_records
|
123
|
+
end
|
124
|
+
|
125
|
+
|
126
|
+
def get_updated_data(array1, array2)
|
127
|
+
array1.each_with_index do |element, index|
|
128
|
+
puts "element: #{element}, array2: #{array2[index]}, index: #{index}\n"
|
129
|
+
puts element if element==array2[index]
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
def debug(hash_in)
|
134
|
+
hash_in.each_with_index do |(key, value), index|
|
135
|
+
puts "key: #{key}, value: #{value}, index: #{index}\n"
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
|
141
|
+
|
142
|
+
|
143
|
+
|
144
|
+
|
145
|
+
|
146
|
+
#assert_equal dataset2, dataset1, 'Datasets should match'
|
data/recon_tools.gemspec
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require File.expand_path('lib/recon_tools/version', __dir__)
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = 'recon_tools'
|
7
|
+
spec.version = ReconTool::VERSION
|
8
|
+
spec.authors = ['Chris Rowe']
|
9
|
+
spec.email = ['coder-chris-github@gmail.com']
|
10
|
+
spec.summary = 'recon_tools: Tools to support importing, exporting and reconciliation of data'
|
11
|
+
spec.description = 'recon_tools: Tools to support importing, exporting and reconciliation of data from JIRA, GoogleSheets and more..'
|
12
|
+
spec.homepage = 'https://github.com/coder-chris/recon_recontool'
|
13
|
+
spec.license = 'MIT'
|
14
|
+
spec.platform = Gem::Platform::RUBY
|
15
|
+
spec.required_ruby_version = '>= 2.5.0'
|
16
|
+
|
17
|
+
spec.files = Dir['README.md', 'LICENSE',
|
18
|
+
'CHANGELOG.md', 'lib/**/*.rb',
|
19
|
+
'lib/**/*.rake',
|
20
|
+
'recon_tools.gemspec', '.github/*.md',
|
21
|
+
'Gemfile', 'Rakefile']
|
22
|
+
spec.test_files = Dir['spec/**/*.rb']
|
23
|
+
spec.extra_rdoc_files = ['README.md']
|
24
|
+
spec.require_paths = ['lib']
|
25
|
+
|
26
|
+
#spec.add_dependency 'rubyzip', '~> 2.3'
|
27
|
+
|
28
|
+
#spec.add_development_dependency 'dotenv', '~> 2.5'
|
29
|
+
if ENV['TEST_RAILS_VERSION'].nil?
|
30
|
+
spec.add_development_dependency 'rails', '~> 6.1.4'
|
31
|
+
else
|
32
|
+
spec.add_development_dependency 'rails', ENV['TEST_RAILS_VERSION'].to_s
|
33
|
+
end
|
34
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
35
|
+
spec.add_development_dependency 'rspec', '~> 3.6'
|
36
|
+
spec.add_development_dependency 'rubocop', '~> 1.0'
|
37
|
+
spec.add_development_dependency 'rubocop-performance', '~> 1.5'
|
38
|
+
spec.add_development_dependency 'rubocop-rspec', '~> 2.4.0'
|
39
|
+
spec.add_development_dependency 'simplecov', '~> 0.16'
|
40
|
+
spec.add_development_dependency 'vcr', '~> 6.0'
|
41
|
+
end
|
metadata
ADDED
@@ -0,0 +1,168 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: recon_tools
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Chris Rowe
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2021-11-07 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rails
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 6.1.4
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 6.1.4
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '13.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '13.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.6'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.6'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rubocop
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rubocop-performance
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.5'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.5'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rubocop-rspec
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 2.4.0
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 2.4.0
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: simplecov
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0.16'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0.16'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: vcr
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '6.0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '6.0'
|
125
|
+
description: 'recon_tools: Tools to support importing, exporting and reconciliation
|
126
|
+
of data from JIRA, GoogleSheets and more..'
|
127
|
+
email:
|
128
|
+
- coder-chris-github@gmail.com
|
129
|
+
executables: []
|
130
|
+
extensions: []
|
131
|
+
extra_rdoc_files:
|
132
|
+
- README.md
|
133
|
+
files:
|
134
|
+
- Gemfile
|
135
|
+
- LICENSE
|
136
|
+
- README.md
|
137
|
+
- Rakefile
|
138
|
+
- lib/recon_services.rb
|
139
|
+
- lib/recon_tools.rb
|
140
|
+
- lib/recon_tools/google_sheets_connect.rb
|
141
|
+
- lib/recon_tools/jira_connect.rb
|
142
|
+
- lib/recon_tools/version.rb
|
143
|
+
- recon_tools.gemspec
|
144
|
+
homepage: https://github.com/coder-chris/recon_recontool
|
145
|
+
licenses:
|
146
|
+
- MIT
|
147
|
+
metadata: {}
|
148
|
+
post_install_message:
|
149
|
+
rdoc_options: []
|
150
|
+
require_paths:
|
151
|
+
- lib
|
152
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
153
|
+
requirements:
|
154
|
+
- - ">="
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: 2.5.0
|
157
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
158
|
+
requirements:
|
159
|
+
- - ">="
|
160
|
+
- !ruby/object:Gem::Version
|
161
|
+
version: '0'
|
162
|
+
requirements: []
|
163
|
+
rubygems_version: 3.2.22
|
164
|
+
signing_key:
|
165
|
+
specification_version: 4
|
166
|
+
summary: 'recon_tools: Tools to support importing, exporting and reconciliation of
|
167
|
+
data'
|
168
|
+
test_files: []
|