bq_coaster 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 93e8bfd289547d0ead44a5d5f70558fbf5bfee9a
+  data.tar.gz: 6fb3ce3da647d07e3de0dd2cb989914aecd0e31f
+SHA512:
+  metadata.gz: 21a70ff50d50f2b5c4f9de5aaad4d4619df2cc01875df40101191d49d0e1b72757c328074b6f1ecdf976e9046e0df7a6d19c6030dd2609384a02dcedd2d5b143
+  data.tar.gz: 9c84a9d70a5505bc39281700b17c5565b8edfe9d7bf2ca16437720ec9ca5687625a480690f59e0c677487c55426f73a81d9315ca422f8b5b64f808fd7181c5ec

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.0
+before_install: gem install bundler -v 1.14.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bq_coaster.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# BqCoaster
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/bq_coaster`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'bq_coaster'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bq_coaster
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/bq_coaster.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "bq_coaster"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bq_coaster.gemspec

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bq_coaster/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "bq_coaster"
+  spec.version       = BqCoaster::VERSION
+  spec.authors       = ["nownabe"]
+  spec.email         = ["nownabe@gmail.com"]
+
+  spec.summary       = %q{Automation Tools for BigQuery / Dataflow}
+  spec.homepage      = "https://github.com/nownabe/bq_coaster"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "erubis"
+  spec.add_dependency "thor"
+  spec.add_dependency "hashie"
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/exe/coaster

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
+
+require "bq_coaster/cli"
+
+BqCoaster::Cli.start(ARGV)

data/lib/bq_coaster/cli.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "thor"
+require "bq_coaster/ext/string"
+
+module BqCoaster
+  class Cli < Thor
+    desc "schema <definition_file>", "Generate BigQuery schema JSON"
+    def schema(definition_file)
+      require "bq_coaster/generators/schema"
+      puts Generators::Schema.new(definition_file, options).generate
+    end
+
+    desc "java_class <definition_file>", "Generate Java class"
+    option :dataflow, type: :boolean, desc: "with Dataflow coder"
+    option :class_name, type: :string, desc: "Java class name"
+    option :package, type: :string, desc: "Java package name"
+    def java_class(definition_file)
+      require "bq_coaster/generators/java_class"
+      puts Generators::JavaClass.new(definition_file, options).generate
+    end
+
+    desc "dataflow_converter <definition_file>",
+      "Generate Java class that converts POJO generated by BqCoaster to TableRow class for Dataflow"
+    option :class_name, type: :string, desc: "Java class name"
+    option :source_class, type: :string, desc: "POJO class name"
+    option :package, type: :string, desc: "Java package name"
+    def dataflow_converter(definition_file)
+      require "bq_coaster/generators/dataflow_converter"
+      puts Generators::DataflowConverter.new(definition_file, options).generate
+    end
+  end
+end

data/lib/bq_coaster/ext/string.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class String
+  def camelize
+    gsub(/^(.)|[-_](.)/) { ($1 || $2).upcase }
+  end
+end

data/lib/bq_coaster/generators/base.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "hashie/mash"
+
+module BqCoaster
+  module Generators
+    class Base
+      attr_reader :definition_file, :options
+
+      def initialize(definition_file, options)
+        @definition_file = definition_file
+        @options = options
+      end
+
+      private
+
+      def definitions
+        @definitions ||=
+          Hashie::Mash.new(YAML.load_file(definition_file))
+      end
+    end
+  end
+end

data/lib/bq_coaster/generators/dataflow_converter/class.erb

@@ -0,0 +1,18 @@
+<%- if package %>
+package <%= package %>;
+
+<%- end %>
+import com.google.api.services.bigquery.model.TableRow;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+
+public class <%= class_name %>
+    extends SimpleFunction<<%= source_class %>, TableRow> {
+
+    @Override
+    public TableRow apply(<%= source_class %> src) {
+        TableRow row = new TableRow();
+
+<%= converter %>
+        return row;
+    }
+}

data/lib/bq_coaster/generators/dataflow_converter/context.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "bq_coaster"
+require "bq_coaster/generators/dataflow_converter/converter_renderer"
+
+module BqCoaster
+  module Generators
+    class DataflowConverter < Base
+      class Context
+        attr_reader :renderer
+
+        def initialize(renderer)
+          @renderer = renderer
+        end
+
+        def definitions
+          renderer.definitions
+        end
+
+        def field_name(name)
+          "#{prefix}.#{name}".sub(/src\./, "").gsub(/\./, "_")
+        end
+
+        def nested?(definition)
+          !(
+            definition.nil? || BqCoaster.edge_properties.any? { |c| definition.include?(c) }
+          )
+        end
+
+        def prefix
+          renderer.prefix
+        end
+
+        def render_converter(name, definition)
+          ConverterRenderer.new(
+            definition,
+            prefix: "#{prefix}.#{name}",
+            indent: 1
+          ).render
+        end
+
+        def value(name, definition)
+          if definition&.key?(:_convert)
+            definition._convert % ["#{prefix}.#{name}"]
+          else
+            "#{prefix}.#{name}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bq_coaster/generators/dataflow_converter/converter.erb

@@ -0,0 +1,9 @@
+<%- definitions.each do |name, definition| %>
+if (<%= prefix %>.<%= name %> != null) {
+    <%- if nested?(definition) %>
+<%= render_converter(name, definition) %>
+    <%- else %>
+    row.set("<%= field_name(name) %>", <%= value(name, definition) %>);
+    <%- end %>
+}
+<%- end %>

data/lib/bq_coaster/generators/dataflow_converter/converter_renderer.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "bq_coaster/generators/dataflow_converter/context"
+
+module BqCoaster
+  module Generators
+    class DataflowConverter < Base
+      class ConverterRenderer
+        include Renderer
+
+        attr_reader :definitions, :prefix, :indent
+
+        def initialize(definitions, prefix: "", indent: 0)
+          @definitions = definitions
+          @prefix      = prefix
+          @indent      = indent
+        end
+
+        def render
+          evaluate.gsub(/^/, " " * 4 * indent)
+        end
+
+        private
+
+        def context
+          Context.new(self)
+        end
+
+        def template_path
+          File.expand_path("../converter.erb", __FILE__)
+        end
+      end
+    end
+  end
+end

data/lib/bq_coaster/generators/dataflow_converter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "bq_coaster/renderer"
+require "bq_coaster/generators/base"
+require "bq_coaster/generators/dataflow_converter/converter_renderer"
+
+module BqCoaster
+  module Generators
+    class DataflowConverter < Base
+      include Renderer
+
+      def generate
+        result
+      end
+
+      private
+
+      def attributes
+        {
+          class_name:   options[:class_name],
+          converter:    converter,
+          source_class: options[:source_class],
+          package:      options[:package]
+        }
+      end
+
+      def converter
+        ConverterRenderer.new(
+          definitions,
+          prefix: "src",
+          indent: 2
+        ).render.gsub(/^\s*\n/, "")
+      end
+
+      def template_path
+        File.expand_path("../dataflow_converter/class.erb", __FILE__)
+      end
+    end
+  end
+end

data/lib/bq_coaster/generators/java_class/class.erb

@@ -0,0 +1,21 @@
+
+<% if dataflow? %>@DefaultCoder(AvroCoder.class)<% end %>
+<%= static? ? "static" : "public" %> class <%= class_name %> {
+    <%- definitions.each do |name, definition| %>
+    <%= dataflow? ? "@Nullable " : "" %>public <%= nested?(definition) ? name.camelize : type(definition) %> <%= name %>;
+        <%- if nested?(definition) %>
+        <%= render_class(name.camelize, definition) %>
+        <%- end %>
+    <%- end %>
+
+    public <%= class_name %>(
+<%= definitions.map do |name, definition|
+      type = nested?(definition) ? name.camelize : type(definition)
+      "        #{type} #{name}"
+    end.join(",\n") %>
+    ) {
+        <%- definitions.each do |name, definition| %>
+        this.<%= name %> = <%= name %>;
+        <%- end %>
+    }
+}

data/lib/bq_coaster/generators/java_class/class_context.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "bq_coaster"
+require "bq_coaster/generators/java_class/class_renderer"
+
+module BqCoaster
+  module Generators
+    class JavaClass < Base
+      class ClassContext
+        attr_reader :renderer
+
+        def initialize(renderer)
+          @renderer = renderer
+        end
+
+        def class_name
+          renderer.class_name
+        end
+
+        def definitions
+          renderer.definitions
+        end
+
+        def dataflow?
+          !!renderer.dataflow
+        end
+
+        def nested?(definition)
+          !(
+            definition.nil? || BqCoaster.edge_properties.any? { |c| definition.key?(c) }
+          )
+        end
+
+        def render_class(class_name, definitions)
+          ClassRenderer.new(
+            class_name,
+            definitions,
+            dataflow: renderer.dataflow,
+            static: true,
+            indent: 1
+          ).render
+        end
+
+        def static?
+          !!renderer.static
+        end
+
+        def type(definition)
+          return "String" unless definition
+          return definition[:_original_type] if definition[:_original_type]
+          case definition[:_type]
+          when "integer"
+            "Long"
+          when "float"
+            "Double"
+          when "timestamp"
+            "Long"
+          else
+            "String"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bq_coaster/generators/java_class/class_renderer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "bq_coaster/renderer"
+require "bq_coaster/generators/java_class/class_context"
+
+module BqCoaster
+  module Generators
+    class JavaClass < Base
+      class ClassRenderer
+        include Renderer
+
+        attr_reader :class_name, :definitions, :dataflow, :static, :indent
+
+        def initialize(class_name, definitions, dataflow: true, static: false, indent: 0)
+          @class_name  = class_name
+          @definitions = definitions
+          @dataflow    = dataflow
+          @static      = static
+          @indent      = indent
+        end
+
+        def render
+          evaluate.gsub(/^/, " " * 4 * indent).gsub(/^\s*$/, "").gsub(/\n{3,}/, "\n\n")
+        end
+
+        private
+
+        def context
+          ClassContext.new(self)
+        end
+
+        def template_path
+          File.expand_path("../class.erb", __FILE__)
+        end
+      end
+    end
+  end
+end

data/lib/bq_coaster/generators/java_class/java_class.erb

@@ -0,0 +1,10 @@
+<%- if package %>
+package <%= package %>;
+
+<%- end %>
+<%- if dataflow %>
+import org.apache.avro.reflect.Nullable;
+import org.apache.beam.sdk.coders.DefaultCoder;
+import org.apache.beam.sdk.coders.AvroCoder;
+<%- end %>
+<%= java_class %>

data/lib/bq_coaster/generators/java_class.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "bq_coaster/renderer"
+require "bq_coaster/generators/base"
+require "bq_coaster/generators/java_class/class_renderer"
+
+module BqCoaster
+  module Generators
+    class JavaClass < Base
+      include Renderer
+
+      def generate
+        result
+      end
+
+      private
+
+      def attributes
+        {
+          dataflow:   options[:dataflow],
+          java_class: java_class,
+          package:    options[:package]
+        }
+      end
+
+      def class_name
+        options[:class_name] || default_class_name
+      end
+
+      def default_class_name
+        File.basename(definition_file, ".*").camelize
+      end
+
+      def java_class
+        ClassRenderer.new(
+          class_name,
+          definitions,
+          dataflow: options[:dataflow]
+        ).render
+      end
+
+      def template_path
+        File.expand_path("../java_class/java_class.erb", __FILE__)
+      end
+    end
+  end
+end

data/lib/bq_coaster/generators/schema.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "json"
+require "bq_coaster"
+require "bq_coaster/generators/base"
+
+module BqCoaster
+  module Generators
+    class Schema < Base
+      def generate
+        JSON.pretty_generate(
+          definitions.map { |n, d| parse(n, d) }.flatten
+        )
+      end
+
+      private
+
+      def nested?(definition)
+        !(
+          definition.nil? || BqCoaster.edge_properties.any? { |c| definition.key?(c) }
+        )
+      end
+
+      def parse(name, definition, prefix = "")
+        if nested?(definition)
+          definition.map { |n, d| parse(n, d, "#{prefix}#{name}_") }
+        else
+          {
+            name: "#{prefix}#{name}",
+            type: (definition&._type || "string").upcase,
+            mode: (definition&._mode || "nullable").upcase
+          }
+        end
+      end
+    end
+  end
+end

data/lib/bq_coaster/renderer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "erubis"
+
+module BqCoaster
+  module Renderer
+    private
+
+    def evaluate
+      renderer.evaluate(context)
+    end
+
+    def renderer
+      Erubis::Eruby.new(template)
+    end
+
+    def result
+      renderer.result(attributes)
+    end
+
+    def template
+      File.read(template_path)
+    end
+  end
+end

data/lib/bq_coaster/version.rb

@@ -0,0 +1,3 @@
+module BqCoaster
+  VERSION = "0.2.0"
+end

data/lib/bq_coaster.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "bq_coaster/version"
+
+module BqCoaster
+  EDGE_PROPERTIES = %i(
+    _type
+    _mode
+    _original_type
+    _convert
+  ).freeze
+
+  def self.edge_properties
+    EDGE_PROPERTIES
+  end
+end

metadata

@@ -0,0 +1,155 @@
+--- !ruby/object:Gem::Specification
+name: bq_coaster
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- nownabe
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-03-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: erubis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: 
+email:
+- nownabe@gmail.com
+executables:
+- coaster
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- bq_coaster.gemspec
+- exe/coaster
+- lib/bq_coaster.rb
+- lib/bq_coaster/cli.rb
+- lib/bq_coaster/ext/string.rb
+- lib/bq_coaster/generators/base.rb
+- lib/bq_coaster/generators/dataflow_converter.rb
+- lib/bq_coaster/generators/dataflow_converter/class.erb
+- lib/bq_coaster/generators/dataflow_converter/context.rb
+- lib/bq_coaster/generators/dataflow_converter/converter.erb
+- lib/bq_coaster/generators/dataflow_converter/converter_renderer.rb
+- lib/bq_coaster/generators/java_class.rb
+- lib/bq_coaster/generators/java_class/class.erb
+- lib/bq_coaster/generators/java_class/class_context.rb
+- lib/bq_coaster/generators/java_class/class_renderer.rb
+- lib/bq_coaster/generators/java_class/java_class.erb
+- lib/bq_coaster/generators/schema.rb
+- lib/bq_coaster/renderer.rb
+- lib/bq_coaster/version.rb
+homepage: https://github.com/nownabe/bq_coaster
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Automation Tools for BigQuery / Dataflow
+test_files: []