bq_guess 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 86c281546032275e8328f83d05b3ac0921cf6617
-  data.tar.gz: f5b00c5f7bc2b2d1ad8d85d93ec4b241a169cacd
+  metadata.gz: fd985006bea596c4f995de9cf43f7ffb25543962
+  data.tar.gz: 1e4b4cb7fd12fd1f5109f864b0cb390e34d29f6a
 SHA512:
-  metadata.gz: 8b8330c80e83918c30fb87572ac632a7d549ad295dfd938782817b97b4bb2f6ea89c1961e3e8cc1f355286d7573561088941fb4b45b17a23b22e92d456956830
-  data.tar.gz: 1836e2970824b2bd0e25a08010f71167dad01e14ec2e8b98ba1c1e05d0fec0e0a98b4750136726f3801f1451ba1fa80465ed2b74e55ba6ec707411547f58496a
+  metadata.gz: d787616533f2507d7c732ee075fd856fd17f523903b0e30985c6588a1432b20dbc3a52552135175a6676c326e429b49f33b7d1c3eae38170d1cbdc48aad44a54
+  data.tar.gz: d11e1b6652ef4065a854547ebef91bd370c72804554f6c991f80801574927a6dafccdea764cd75b7b810aee173b2f77ad46e9eae12ef7ba118be84d6f705fda9

data/README.md

@@ -1,41 +1,89 @@
 # BqGuess
-
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/bq_guess`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+BqGuess guesses BigQuery schema from existing logs.
 
 ## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'bq_guess'
+```bash
+$ gem install bq_guess
 ```
 
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install bq_guess
-
 ## Usage
+```bash
+$ bq_guess existing_log.jsonl
+```
+
+For example:
+
+```bash
+$ cat existing_log.jsonl
+{"required":123,"optional":true,"nested":{"required":1234,"optional":"yes"},"array":[0,1,2,3,4]}
+{"required":456,"optional":false,"nested":{"required":1234,"optional":"yes","nested":{"prop":1}},"array":[5,6,7,8,9]}
+{"required":789,"nested":{"required":1234,"optional":"yes","additional":"added"},"array":[]}
+
+$ bq_guess existing_log.jsonl
+[
+  {
+    "name": "required",
+    "type": "INTEGER",
+    "mode": "REQUIRED"
+  },
+  {
+    "name": "optional",
+    "type": "BOOLEAN",
+    "mode": "NULLABLE"
+  },
+  {
+    "name": "nested",
+    "type": "RECORD",
+    "mode": "REQUIRED",
+    "fields": [
+      {
+        "name": "required",
+        "type": "INTEGER",
+        "mode": "REQUIRED"
+      },
+      {
+        "name": "optional",
+        "type": "STRING",
+        "mode": "REQUIRED"
+      },
+      {
+        "name": "nested",
+        "type": "RECORD",
+        "mode": "NULLABLE",
+        "fields": [
+          {
+            "name": "prop",
+            "type": "INTEGER",
+            "mode": "REQUIRED"
+          }
+        ]
+      },
+      {
+        "name": "additional",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      }
+    ]
+  },
+  {
+    "name": "array",
+    "type": "INTEGER",
+    "mode": "REPEATED"
+  }
+]
+```
 
-TODO: Write usage instructions here
+## Supported Formats
+* JSON Lines
+* LTSV
 
 ## Development
-
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
-
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/bq_guess.
-
+Bug reports and pull requests are welcome on GitHub at https://github.com/nownabe/bq_guess.
 
 ## License
-
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/lib/bq_guess/cli.rb

@@ -2,6 +2,7 @@
 
 require "json"
 require "bq_guess/guessers/json_lines"
+require "bq_guess/guessers/ltsv"
 
 module BqGuess
   class Cli
@@ -12,15 +13,31 @@ module BqGuess
     end
 
     def execute
-      result =
-        Guessers::JsonLines.new(
-          File.read(File.expand_path(options[:input_path]))
-        ).guess.as_schema
-      puts JSON.pretty_generate(result)
+      puts JSON.pretty_generate(guesser.guess.as_schema)
     end
 
     private
 
+    def format
+      JSON.parse(input_content.lines.first)
+      :json_lines
+    rescue
+      :ltsv
+    end
+
+    def guesser
+      case format
+      when :json_lines
+        Guessers::JsonLines.new(input_content)
+      when :ltsv
+        Guessers::Ltsv.new(input_content)
+      end
+    end
+
+    def input_content
+      @input_content ||= File.read(File.expand_path(options[:input_path]))
+    end
+
     # TODO: ignore error line
     # TODO: default nullable instead of required
     def parse_option(args)

data/lib/bq_guess/guessers/base.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "bq_guess/guesser"
+
+module BqGuess
+  module Guessers
+    class Base
+      attr_reader :file_content
+
+      def initialize(file_content)
+        @file_content = file_content
+      end
+
+      def guess
+        BqGuess::Guesser.guess_records(records)
+      end
+    end
+  end
+end

data/lib/bq_guess/guessers/json_lines.rb

@@ -1,22 +1,15 @@
 # frozen_string_literal: true
 
 require "json"
-require "bq_guess/guesser"
-require "bq_guess/schema"
+require "bq_guess/guessers/base"
 
 module BqGuess
   module Guessers
-    class JsonLines
-      attr_reader :json_lines
+    class JsonLines < Base
+      private
 
-      def initialize(json_lines)
-        @json_lines = json_lines.lines
-      end
-
-      def guess
-        BqGuess::Guesser.guess_records(
-          json_lines.map { |l| JSON.parse(l) }
-        )
+      def records
+        file_content.lines.map { |l| JSON.parse(l) }
       end
     end
   end

data/lib/bq_guess/guessers/ltsv.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "bq_guess/guessers/base"
+
+module BqGuess
+  module Guessers
+    class Ltsv < Base
+      private
+
+      def parse_line(line)
+        line.split("\t").each_with_object({}) do |kv, h|
+          next unless kv.include?(":")
+          key, value = kv.split(":", 2)
+          h[key] = value.empty? ? nil : value
+        end
+      end
+
+      def records
+        file_content.lines.map { |l| parse_line(l) }
+      end
+    end
+  end
+end

data/lib/bq_guess/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module BqGuess
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bq_guess
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - nownabe
@@ -97,7 +97,9 @@ files:
 - lib/bq_guess/fields/record.rb
 - lib/bq_guess/fields/string.rb
 - lib/bq_guess/guesser.rb
+- lib/bq_guess/guessers/base.rb
 - lib/bq_guess/guessers/json_lines.rb
+- lib/bq_guess/guessers/ltsv.rb
 - lib/bq_guess/schema.rb
 - lib/bq_guess/version.rb
 homepage: https://github.com/nownabe/bq_guess