dataset_explorer 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: d8d37acf8dbd69347ee5d469c4c6ee4a70b3c8a86a74ec30e7d92518a2b2943f
4
+ data.tar.gz: 291900362c27de4d230b492e58983a9f8ac3c44b0a52cdcabd28636ee5536416
5
+ SHA512:
6
+ metadata.gz: 9aff0c19a1a1fdaae24faa517a488e7611c26f445d80db30da997f3aa60a1ee583d30f4215cb3b48076b5bc8c51d166f01724cc322c77e2cedcc0ef5adec32be
7
+ data.tar.gz: 007b2822c83146392d54429f63b43038ce5cf09e3d41b9e51d8bc81592fc617022c4b6af5e8e29617b928328badb4bbb205397f41bcd08452e8efff29aee2711
@@ -0,0 +1,12 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+
10
+ # rspec failure tracking
11
+ .rspec_status
12
+ /Gemfile.lock
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --color
@@ -0,0 +1,66 @@
1
+ require:
2
+ - rubocop-performance
3
+ - rubocop-rspec
4
+
5
+ Metrics/BlockLength:
6
+ Exclude:
7
+ - 'spec/**/*'
8
+
9
+ Metrics/MethodLength:
10
+ Exclude:
11
+ - 'db/migrate/**.rb'
12
+
13
+ Layout/LineLength:
14
+ Max: 100
15
+ Exclude:
16
+ - 'db/seeds.rb'
17
+
18
+ Style/Documentation:
19
+ Enabled: false
20
+
21
+ Style/GuardClause:
22
+ Enabled: false
23
+
24
+ Style/IfUnlessModifier:
25
+ Enabled: false
26
+
27
+ Layout/MultilineMethodCallIndentation:
28
+ EnforcedStyle: indented
29
+
30
+ Layout/MultilineOperationIndentation:
31
+ Enabled: true
32
+ EnforcedStyle: indented
33
+
34
+ Layout/FirstArrayElementIndentation:
35
+ EnforcedStyle: consistent
36
+
37
+ Naming/RescuedExceptionsVariableName:
38
+ PreferredName: 'exception'
39
+
40
+ Style/FormatStringToken:
41
+ EnforcedStyle: template
42
+
43
+ RSpec:
44
+ Exclude:
45
+ - 'spec/factories/**/*'
46
+
47
+ RSpec/ExampleLength:
48
+ Max: 10
49
+
50
+ RSpec/MultipleExpectations:
51
+ Enabled: false
52
+
53
+ AllCops:
54
+ Exclude:
55
+ - 'node_modules/**/*'
56
+ - 'script/**/*'
57
+ - 'log/**/*'
58
+ - 'docker/**/*'
59
+ - 'doc/**/*'
60
+ - 'bin/**/*'
61
+ - 'Rakefile'
62
+ - 'config.ru'
63
+ - 'config/**/*'
64
+ - 'db/schema.rb'
65
+ - 'node_modules/**/*'
66
+ - 'vendor/**/*'
@@ -0,0 +1,11 @@
1
+ ---
2
+ language: ruby
3
+ cache: bundler
4
+ rvm:
5
+ - 2.6.5
6
+ before_install: gem install bundler -v 2.1.4
7
+
8
+ script:
9
+ - RAILS_ENV=test bundle exec rspec
10
+ - bundle exec rubocop --version
11
+ - bundle exec rubocop -a
@@ -0,0 +1,74 @@
1
+ # Contributor Covenant Code of Conduct
2
+
3
+ ## Our Pledge
4
+
5
+ In the interest of fostering an open and welcoming environment, we as
6
+ contributors and maintainers pledge to making participation in our project and
7
+ our community a harassment-free experience for everyone, regardless of age, body
8
+ size, disability, ethnicity, gender identity and expression, level of experience,
9
+ nationality, personal appearance, race, religion, or sexual identity and
10
+ orientation.
11
+
12
+ ## Our Standards
13
+
14
+ Examples of behavior that contributes to creating a positive environment
15
+ include:
16
+
17
+ * Using welcoming and inclusive language
18
+ * Being respectful of differing viewpoints and experiences
19
+ * Gracefully accepting constructive criticism
20
+ * Focusing on what is best for the community
21
+ * Showing empathy towards other community members
22
+
23
+ Examples of unacceptable behavior by participants include:
24
+
25
+ * The use of sexualized language or imagery and unwelcome sexual attention or
26
+ advances
27
+ * Trolling, insulting/derogatory comments, and personal or political attacks
28
+ * Public or private harassment
29
+ * Publishing others' private information, such as a physical or electronic
30
+ address, without explicit permission
31
+ * Other conduct which could reasonably be considered inappropriate in a
32
+ professional setting
33
+
34
+ ## Our Responsibilities
35
+
36
+ Project maintainers are responsible for clarifying the standards of acceptable
37
+ behavior and are expected to take appropriate and fair corrective action in
38
+ response to any instances of unacceptable behavior.
39
+
40
+ Project maintainers have the right and responsibility to remove, edit, or
41
+ reject comments, commits, code, wiki edits, issues, and other contributions
42
+ that are not aligned to this Code of Conduct, or to ban temporarily or
43
+ permanently any contributor for other behaviors that they deem inappropriate,
44
+ threatening, offensive, or harmful.
45
+
46
+ ## Scope
47
+
48
+ This Code of Conduct applies both within project spaces and in public spaces
49
+ when an individual is representing the project or its community. Examples of
50
+ representing a project or community include using an official project e-mail
51
+ address, posting via an official social media account, or acting as an appointed
52
+ representative at an online or offline event. Representation of a project may be
53
+ further defined and clarified by project maintainers.
54
+
55
+ ## Enforcement
56
+
57
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
58
+ reported by contacting the project team at marcelo.jacobus@gmail.com. All
59
+ complaints will be reviewed and investigated and will result in a response that
60
+ is deemed necessary and appropriate to the circumstances. The project team is
61
+ obligated to maintain confidentiality with regard to the reporter of an incident.
62
+ Further details of specific enforcement policies may be posted separately.
63
+
64
+ Project maintainers who do not follow or enforce the Code of Conduct in good
65
+ faith may face temporary or permanent repercussions as determined by other
66
+ members of the project's leadership.
67
+
68
+ ## Attribution
69
+
70
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
71
+ available at [https://contributor-covenant.org/version/1/4][version]
72
+
73
+ [homepage]: https://contributor-covenant.org
74
+ [version]: https://contributor-covenant.org/version/1/4/
data/Gemfile ADDED
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ source 'https://rubygems.org'
4
+
5
+ # Specify your gem's dependencies in dataset_explorer.gemspec
6
+ gemspec
7
+
8
+ gem 'awesome_print'
9
+ gem 'coveralls', '~> 0.8.23'
10
+ gem 'rake', '~> 12.0'
11
+ gem 'reek'
12
+ gem 'rspec', '~> 3.0'
13
+ gem 'rubocop', '0.79.0'
14
+ gem 'rubocop-performance'
15
+ gem 'rubocop-rspec'
16
+ gem 'simplecov', '~> 0.16.1'
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2020 Marcelo Jacobus
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
@@ -0,0 +1,185 @@
1
+ # DatasetExplorer
2
+
3
+ [![Build Status](https://travis-ci.org/mjacobus/dataset_explorer.svg?branch=master)](https://travis-ci.org/mjacobus/dataset_explorer)
4
+ [![Coverage Status](https://coveralls.io/repos/github/mjacobus/dataset_explorer/badge.svg?branch=master)](https://coveralls.io/github/mjacobus/dataset_explorer?branch=master)
5
+ [![Maintainability](https://api.codeclimate.com/v1/badges/ae41e3facbadaabaa463/maintainability)](https://codeclimate.com/github/mjacobus/dataset_explorer/maintainability)
6
+
7
+ Explores a data set and keep track of all the fields ever returned.
8
+
9
+ The idea is to collect all the fields from a dataset, including the ones nested from an API that will omit some fields when they are `null`, or even to make sure that the fields listed in the documentation (or not listed at all) can be explained by the output.
10
+
11
+ ## Installation
12
+
13
+ Add this line to your application's Gemfile:
14
+
15
+ ```ruby
16
+ gem 'dataset_explorer'
17
+ ```
18
+
19
+ And then execute:
20
+
21
+ $ bundle install
22
+
23
+ Or install it yourself as:
24
+
25
+ $ gem install dataset_explorer
26
+
27
+ ## Usage
28
+
29
+ ```ruby
30
+ def some_method_that_pulls_data(args)
31
+ response = client.get('url', args)
32
+ DatasetExplorer::Collector.instance.collect('type-of-request', result)
33
+ end
34
+
35
+ # somewhere
36
+
37
+ p DatasetExplorer::Collector.explain('type-of-request')
38
+ p DatasetExplorer::Collector.explain_all
39
+ ```
40
+
41
+
42
+ ```ruby
43
+ require 'bundler/setup'
44
+ require 'open-uri'
45
+ require 'json'
46
+ require 'dataset_explorer'
47
+ require 'awesome_print'
48
+
49
+ urls = {
50
+ github: 'https://pages.github.com/versions.json',
51
+ github_repo_contents: 'https://api.github.com/repos/mjacobus/rubyjobsbrazil/contents',
52
+ }
53
+
54
+ COLLECTOR = DatasetExplorer::Collector.instance
55
+
56
+ urls.each do |provider, url|
57
+ data = [JSON.parse(open(url).read)].flatten
58
+
59
+ data.each do |item|
60
+ COLLECTOR.collect(provider, item)
61
+ end
62
+ end
63
+
64
+ explanation = COLLECTOR.explain_all
65
+ ```
66
+
67
+ Explanation will be:
68
+
69
+ ```ruby
70
+ {
71
+ github: %w[
72
+ jekyll
73
+ jekyll-sass-converter
74
+ kramdown
75
+ jekyll-commonmark-ghpages
76
+ liquid
77
+ rouge
78
+ github-pages-health-check
79
+ jekyll-redirect-from
80
+ jekyll-sitemap
81
+ jekyll-feed
82
+ jekyll-gist
83
+ jekyll-paginate
84
+ jekyll-coffeescript
85
+ jekyll-seo-tag
86
+ jekyll-github-metadata
87
+ jekyll-avatar
88
+ jekyll-remote-theme
89
+ jemoji
90
+ jekyll-mentions
91
+ jekyll-relative-links
92
+ jekyll-optional-front-matter
93
+ jekyll-readme-index
94
+ jekyll-default-layout
95
+ jekyll-titles-from-headings
96
+ jekyll-swiss
97
+ minima
98
+ jekyll-theme-primer
99
+ jekyll-theme-architect
100
+ jekyll-theme-cayman
101
+ jekyll-theme-dinky
102
+ jekyll-theme-hacker
103
+ jekyll-theme-leap-day
104
+ jekyll-theme-merlot
105
+ jekyll-theme-midnight
106
+ jekyll-theme-minimal
107
+ jekyll-theme-modernist
108
+ jekyll-theme-slate
109
+ jekyll-theme-tactile
110
+ jekyll-theme-time-machine
111
+ ruby
112
+ github-pages
113
+ html-pipeline
114
+ sass
115
+ safe_yaml
116
+ nokogiri
117
+ ],
118
+ github_repo: [
119
+ 'name',
120
+ 'path',
121
+ 'sha',
122
+ 'size',
123
+ 'url',
124
+ 'html_url',
125
+ 'git_url',
126
+ 'download_url',
127
+ 'type',
128
+ '_links.self',
129
+ '_links.git',
130
+ '_links.html'
131
+ ]
132
+ }
133
+ ```
134
+
135
+ ```ruby
136
+ data = {
137
+ user: {
138
+ first_name: 'John',
139
+ last_name: 'Doe',
140
+ achievements: [
141
+ {
142
+ headline: '10 nights with no sleep',
143
+ tag: 'amazing'
144
+ },
145
+ {
146
+ headline: 'read 2 books in 6 hours',
147
+ tag: 'nerd'
148
+ }
149
+ ]
150
+ }
151
+ }
152
+
153
+ DatasetExplorer::Collector.instance.collect(:achievements, data)
154
+ result = DatasetExplorer::Collector.explain_all(:achievements)
155
+
156
+ # result is:
157
+
158
+ [
159
+ "user.first_name",
160
+ "user.last_name",
161
+ "user.achievements.[].headline",
162
+ "user.achievements.[].tag"
163
+ ]
164
+ ```
165
+
166
+ Or if you try like this
167
+
168
+ ## Development
169
+
170
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
171
+
172
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
173
+
174
+ ## Contributing
175
+
176
+ Bug reports and pull requests are welcome on GitHub at https://github.com/mjacobus/dataset_explorer. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/mjacobus/dataset_explorer/blob/master/CODE_OF_CONDUCT.md).
177
+
178
+
179
+ ## License
180
+
181
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
182
+
183
+ ## Code of Conduct
184
+
185
+ Everyone interacting in the DatasetExplorer project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/mjacobus/dataset_explorer/blob/master/CODE_OF_CONDUCT.md).
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "dataset_explorer"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'lib/dataset_explorer/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = 'dataset_explorer'
7
+ spec.version = DatasetExplorer::VERSION
8
+ spec.authors = ['Marcelo Jacobus']
9
+ spec.email = ['marcelo.jacobus@gmail.com']
10
+
11
+ spec.summary = 'Explore and maps fields from a dataset - I.E. Api response'
12
+ spec.description = 'Explore and maps fields from a dataset - I.E. Api response'
13
+ spec.homepage = 'https://github.com/mjacobus/dataset_explorer'
14
+ spec.license = 'MIT'
15
+ spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
16
+
17
+ spec.metadata['allowed_push_host'] = 'https://rubygems.org'
18
+
19
+ spec.metadata['homepage_uri'] = spec.homepage
20
+ spec.metadata['source_code_uri'] = 'https://github.com/mjacobus/dataset_explorer'
21
+ spec.metadata['changelog_uri'] = 'https://github.com/mjacobus/dataset_explorer'
22
+
23
+ # Specify which files should be added to the gem when it is released.
24
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
25
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
26
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
27
+ end
28
+ spec.bindir = 'exe'
29
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
30
+ spec.require_paths = ['lib']
31
+ end
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'dataset_explorer/version'
4
+ require 'dataset_explorer/item_collector'
5
+ require 'dataset_explorer/collector'
6
+
7
+ module DatasetExplorer
8
+ class Error < StandardError; end
9
+ end
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DatasetExplorer
4
+ class Collector
5
+ def self.instance
6
+ @instance ||= new
7
+ end
8
+
9
+ def initialize
10
+ @collectors = {}
11
+ end
12
+
13
+ def collect(type, item)
14
+ collector_for(type).add(item)
15
+ end
16
+
17
+ def explain_all
18
+ @collectors.keys.map do |key|
19
+ [key, explain(key)]
20
+ end.to_h
21
+ end
22
+
23
+ def explain(type)
24
+ @collectors.fetch(type).values
25
+ end
26
+
27
+ private
28
+
29
+ def collector_for(type)
30
+ @collectors[type] ||= ItemCollector.new
31
+ end
32
+
33
+ private_class_method :new
34
+ end
35
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DatasetExplorer
4
+ class ItemCollector
5
+ def initialize
6
+ @values = []
7
+ end
8
+
9
+ attr_reader :values
10
+
11
+ def add(item)
12
+ new_values = item.keys.map do |key|
13
+ map_keys(item[key], key)
14
+ end.flatten.map(&:to_s).uniq
15
+
16
+ @values << new_values
17
+ @values = @values.flatten.uniq
18
+ end
19
+
20
+ private
21
+
22
+ # rubocop:disable Metrics/MethodLength
23
+ def map_keys(value, key = nil, prefix = nil)
24
+ prefix = [prefix, key].compact.join('.')
25
+
26
+ unless mappable?(value)
27
+ return prefix
28
+ end
29
+
30
+ if behaves_like_hash?(value)
31
+ return value.keys.map do |sub_key|
32
+ map_keys(value[sub_key], sub_key, prefix)
33
+ end
34
+ end
35
+
36
+ if value.respond_to?(:each)
37
+ prefix = "#{prefix}.[]"
38
+ return [].tap do |values|
39
+ value.each do |item_value|
40
+ values << map_keys(item_value, nil, prefix)
41
+ end
42
+ end
43
+ end
44
+
45
+ raise Error, 'Unforseen scenario'
46
+ end
47
+ # rubocop:enable Metrics/MethodLength
48
+
49
+ def mappable?(value)
50
+ if behaves_like_hash?(value)
51
+ return true
52
+ end
53
+
54
+ if behaves_like_array?(value)
55
+ return true
56
+ end
57
+
58
+ false
59
+ end
60
+
61
+ def behaves_like_hash?(value)
62
+ value.respond_to?(:keys)
63
+ end
64
+
65
+ def behaves_like_array?(value)
66
+ value.respond_to?(:each)
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DatasetExplorer
4
+ VERSION = '0.1.0'
5
+ end
metadata ADDED
@@ -0,0 +1,63 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: dataset_explorer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Marcelo Jacobus
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-04-23 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: Explore and maps fields from a dataset - I.E. Api response
14
+ email:
15
+ - marcelo.jacobus@gmail.com
16
+ executables: []
17
+ extensions: []
18
+ extra_rdoc_files: []
19
+ files:
20
+ - ".gitignore"
21
+ - ".rspec"
22
+ - ".rubocop.yml"
23
+ - ".travis.yml"
24
+ - CODE_OF_CONDUCT.md
25
+ - Gemfile
26
+ - LICENSE.txt
27
+ - README.md
28
+ - Rakefile
29
+ - bin/console
30
+ - bin/setup
31
+ - dataset_explorer.gemspec
32
+ - lib/dataset_explorer.rb
33
+ - lib/dataset_explorer/collector.rb
34
+ - lib/dataset_explorer/item_collector.rb
35
+ - lib/dataset_explorer/version.rb
36
+ homepage: https://github.com/mjacobus/dataset_explorer
37
+ licenses:
38
+ - MIT
39
+ metadata:
40
+ allowed_push_host: https://rubygems.org
41
+ homepage_uri: https://github.com/mjacobus/dataset_explorer
42
+ source_code_uri: https://github.com/mjacobus/dataset_explorer
43
+ changelog_uri: https://github.com/mjacobus/dataset_explorer
44
+ post_install_message:
45
+ rdoc_options: []
46
+ require_paths:
47
+ - lib
48
+ required_ruby_version: !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 2.3.0
53
+ required_rubygems_version: !ruby/object:Gem::Requirement
54
+ requirements:
55
+ - - ">="
56
+ - !ruby/object:Gem::Version
57
+ version: '0'
58
+ requirements: []
59
+ rubygems_version: 3.0.3
60
+ signing_key:
61
+ specification_version: 4
62
+ summary: Explore and maps fields from a dataset - I.E. Api response
63
+ test_files: []