bq_guess 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +72 -24
- data/lib/bq_guess/cli.rb +22 -5
- data/lib/bq_guess/guessers/base.rb +19 -0
- data/lib/bq_guess/guessers/json_lines.rb +5 -12
- data/lib/bq_guess/guessers/ltsv.rb +23 -0
- data/lib/bq_guess/version.rb +1 -1
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fd985006bea596c4f995de9cf43f7ffb25543962
|
4
|
+
data.tar.gz: 1e4b4cb7fd12fd1f5109f864b0cb390e34d29f6a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d787616533f2507d7c732ee075fd856fd17f523903b0e30985c6588a1432b20dbc3a52552135175a6676c326e429b49f33b7d1c3eae38170d1cbdc48aad44a54
|
7
|
+
data.tar.gz: d11e1b6652ef4065a854547ebef91bd370c72804554f6c991f80801574927a6dafccdea764cd75b7b810aee173b2f77ad46e9eae12ef7ba118be84d6f705fda9
|
data/README.md
CHANGED
@@ -1,41 +1,89 @@
|
|
1
1
|
# BqGuess
|
2
|
-
|
3
|
-
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/bq_guess`. To experiment with that code, run `bin/console` for an interactive prompt.
|
4
|
-
|
5
|
-
TODO: Delete this and the text above, and describe your gem
|
2
|
+
BqGuess guesses BigQuery schema from existing logs.
|
6
3
|
|
7
4
|
## Installation
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
```ruby
|
12
|
-
gem 'bq_guess'
|
5
|
+
```bash
|
6
|
+
$ gem install bq_guess
|
13
7
|
```
|
14
8
|
|
15
|
-
And then execute:
|
16
|
-
|
17
|
-
$ bundle
|
18
|
-
|
19
|
-
Or install it yourself as:
|
20
|
-
|
21
|
-
$ gem install bq_guess
|
22
|
-
|
23
9
|
## Usage
|
10
|
+
```bash
|
11
|
+
$ bq_guess existing_log.jsonl
|
12
|
+
```
|
13
|
+
|
14
|
+
For example:
|
15
|
+
|
16
|
+
```bash
|
17
|
+
$ cat existing_log.jsonl
|
18
|
+
{"required":123,"optional":true,"nested":{"required":1234,"optional":"yes"},"array":[0,1,2,3,4]}
|
19
|
+
{"required":456,"optional":false,"nested":{"required":1234,"optional":"yes","nested":{"prop":1}},"array":[5,6,7,8,9]}
|
20
|
+
{"required":789,"nested":{"required":1234,"optional":"yes","additional":"added"},"array":[]}
|
21
|
+
|
22
|
+
$ bq_guess existing_log.jsonl
|
23
|
+
[
|
24
|
+
{
|
25
|
+
"name": "required",
|
26
|
+
"type": "INTEGER",
|
27
|
+
"mode": "REQUIRED"
|
28
|
+
},
|
29
|
+
{
|
30
|
+
"name": "optional",
|
31
|
+
"type": "BOOLEAN",
|
32
|
+
"mode": "NULLABLE"
|
33
|
+
},
|
34
|
+
{
|
35
|
+
"name": "nested",
|
36
|
+
"type": "RECORD",
|
37
|
+
"mode": "REQUIRED",
|
38
|
+
"fields": [
|
39
|
+
{
|
40
|
+
"name": "required",
|
41
|
+
"type": "INTEGER",
|
42
|
+
"mode": "REQUIRED"
|
43
|
+
},
|
44
|
+
{
|
45
|
+
"name": "optional",
|
46
|
+
"type": "STRING",
|
47
|
+
"mode": "REQUIRED"
|
48
|
+
},
|
49
|
+
{
|
50
|
+
"name": "nested",
|
51
|
+
"type": "RECORD",
|
52
|
+
"mode": "NULLABLE",
|
53
|
+
"fields": [
|
54
|
+
{
|
55
|
+
"name": "prop",
|
56
|
+
"type": "INTEGER",
|
57
|
+
"mode": "REQUIRED"
|
58
|
+
}
|
59
|
+
]
|
60
|
+
},
|
61
|
+
{
|
62
|
+
"name": "additional",
|
63
|
+
"type": "STRING",
|
64
|
+
"mode": "NULLABLE"
|
65
|
+
}
|
66
|
+
]
|
67
|
+
},
|
68
|
+
{
|
69
|
+
"name": "array",
|
70
|
+
"type": "INTEGER",
|
71
|
+
"mode": "REPEATED"
|
72
|
+
}
|
73
|
+
]
|
74
|
+
```
|
24
75
|
|
25
|
-
|
76
|
+
## Supported Formats
|
77
|
+
* JSON Lines
|
78
|
+
* LTSV
|
26
79
|
|
27
80
|
## Development
|
28
|
-
|
29
81
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
30
82
|
|
31
83
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
32
84
|
|
33
85
|
## Contributing
|
34
|
-
|
35
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/bq_guess.
|
36
|
-
|
86
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/nownabe/bq_guess.
|
37
87
|
|
38
88
|
## License
|
39
|
-
|
40
89
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
41
|
-
|
data/lib/bq_guess/cli.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require "json"
|
4
4
|
require "bq_guess/guessers/json_lines"
|
5
|
+
require "bq_guess/guessers/ltsv"
|
5
6
|
|
6
7
|
module BqGuess
|
7
8
|
class Cli
|
@@ -12,15 +13,31 @@ module BqGuess
|
|
12
13
|
end
|
13
14
|
|
14
15
|
def execute
|
15
|
-
|
16
|
-
Guessers::JsonLines.new(
|
17
|
-
File.read(File.expand_path(options[:input_path]))
|
18
|
-
).guess.as_schema
|
19
|
-
puts JSON.pretty_generate(result)
|
16
|
+
puts JSON.pretty_generate(guesser.guess.as_schema)
|
20
17
|
end
|
21
18
|
|
22
19
|
private
|
23
20
|
|
21
|
+
def format
|
22
|
+
JSON.parse(input_content.lines.first)
|
23
|
+
:json_lines
|
24
|
+
rescue
|
25
|
+
:ltsv
|
26
|
+
end
|
27
|
+
|
28
|
+
def guesser
|
29
|
+
case format
|
30
|
+
when :json_lines
|
31
|
+
Guessers::JsonLines.new(input_content)
|
32
|
+
when :ltsv
|
33
|
+
Guessers::Ltsv.new(input_content)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def input_content
|
38
|
+
@input_content ||= File.read(File.expand_path(options[:input_path]))
|
39
|
+
end
|
40
|
+
|
24
41
|
# TODO: ignore error line
|
25
42
|
# TODO: default nullable instead of required
|
26
43
|
def parse_option(args)
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "bq_guess/guesser"
|
4
|
+
|
5
|
+
module BqGuess
|
6
|
+
module Guessers
|
7
|
+
class Base
|
8
|
+
attr_reader :file_content
|
9
|
+
|
10
|
+
def initialize(file_content)
|
11
|
+
@file_content = file_content
|
12
|
+
end
|
13
|
+
|
14
|
+
def guess
|
15
|
+
BqGuess::Guesser.guess_records(records)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -1,22 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "json"
|
4
|
-
require "bq_guess/
|
5
|
-
require "bq_guess/schema"
|
4
|
+
require "bq_guess/guessers/base"
|
6
5
|
|
7
6
|
module BqGuess
|
8
7
|
module Guessers
|
9
|
-
class JsonLines
|
10
|
-
|
8
|
+
class JsonLines < Base
|
9
|
+
private
|
11
10
|
|
12
|
-
def
|
13
|
-
|
14
|
-
end
|
15
|
-
|
16
|
-
def guess
|
17
|
-
BqGuess::Guesser.guess_records(
|
18
|
-
json_lines.map { |l| JSON.parse(l) }
|
19
|
-
)
|
11
|
+
def records
|
12
|
+
file_content.lines.map { |l| JSON.parse(l) }
|
20
13
|
end
|
21
14
|
end
|
22
15
|
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "bq_guess/guessers/base"
|
4
|
+
|
5
|
+
module BqGuess
|
6
|
+
module Guessers
|
7
|
+
class Ltsv < Base
|
8
|
+
private
|
9
|
+
|
10
|
+
def parse_line(line)
|
11
|
+
line.split("\t").each_with_object({}) do |kv, h|
|
12
|
+
next unless kv.include?(":")
|
13
|
+
key, value = kv.split(":", 2)
|
14
|
+
h[key] = value.empty? ? nil : value
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def records
|
19
|
+
file_content.lines.map { |l| parse_line(l) }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
data/lib/bq_guess/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bq_guess
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- nownabe
|
@@ -97,7 +97,9 @@ files:
|
|
97
97
|
- lib/bq_guess/fields/record.rb
|
98
98
|
- lib/bq_guess/fields/string.rb
|
99
99
|
- lib/bq_guess/guesser.rb
|
100
|
+
- lib/bq_guess/guessers/base.rb
|
100
101
|
- lib/bq_guess/guessers/json_lines.rb
|
102
|
+
- lib/bq_guess/guessers/ltsv.rb
|
101
103
|
- lib/bq_guess/schema.rb
|
102
104
|
- lib/bq_guess/version.rb
|
103
105
|
homepage: https://github.com/nownabe/bq_guess
|