avro_utils 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 138c585168f97b5d9786dfcfda80f7b5f25caa61
+  data.tar.gz: 1011ffd72b61545233da0a573608a2692d8d194c
+SHA512:
+  metadata.gz: ccd802a2f71f5a48a723dd457ed210662e6dae1a80a44d1500593cb909e5481981f2b40a6da95707f528fbadb24c4e668e80e4d0e753a5e2292da7424db30989
+  data.tar.gz: a5c4af7e7f56a3694fce9f78b0cdbd8780b726ce81f837ad0d6c3b256c6bd6a5d355f8c36c48786fab532ad0819b82850a84cf17a39ec1eadf3c5410571c59a3

data/avro_utils.gemspec

@@ -0,0 +1,20 @@
+Gem::Specification.new do |s|
+  s.name        = 'avro_utils'
+  s.version     = '0.1.0'
+  s.summary     = "A gem to convert different data format to avro"
+  s.description = "A gem to convert different data format to avro"
+  s.authors     = ['Linh Chau']
+  s.email       = 'chauhonglinh@gmail.com'
+  s.files       = [
+                    './avro_utils.gemspec', 'lib/avro_utils.rb',
+                    'lib/utils/avro_utils.rb', 'lib/exceptions/invalid_data_exception.rb'
+                  ]
+  s.homepage    = 'https://github.com/linhchauatl/avro_utils'
+  s.license     = 'MIT'
+  s.add_runtime_dependency 'json'
+  s.add_runtime_dependency 'json-schema'
+  s.add_runtime_dependency 'avro'
+  s.add_runtime_dependency 'logging'
+  s.add_runtime_dependency 'activesupport'
+  s.add_development_dependency 'rspec', '~> 3.3'
+end

data/lib/avro_utils.rb

@@ -0,0 +1,29 @@
+require 'active_support/all'
+require 'json'
+require 'json-schema'
+require 'avro'
+
+# avro types
+# string: unicode character sequence
+# bytes: sequence of 8-bit unsigned bytes
+# int: 32-bit signed integer
+# long: 64-bit signed integer
+# float: single precision (32-bit) IEEE 754 floating-point number
+# double: double precision (64-bit) IEEE 754 floating-point number
+# boolean: a binary value
+# null: no value
+
+RUBY_AVRO_TYPE_MAPPING = {
+  String: :string,
+  Symbol: :string,
+  Fixnum: :int,
+  Bignum: :long,
+  Float: :double,
+  TrueClass: :boolean,
+  FalseClass: :boolean,
+  NilClass: :null
+
+}
+
+Dir.glob('./lib/exceptions/*.rb').sort.each { |file| puts file; require file }
+Dir.glob('./lib/utils/*.rb').sort.each { |file| puts file; require file }

data/lib/exceptions/invalid_data_exception.rb

@@ -0,0 +1,2 @@
+class InvalidDataException < RuntimeError
+end

data/lib/utils/avro_utils.rb

@@ -0,0 +1,54 @@
+class AvroUtils
+
+  class << self
+    def avro_schema_hash(hash_data, record_type_name)
+      result = { 
+        type: 'record',
+        name: record_type_name,
+        fields: []
+      }
+
+      hash_data.each do |key, value|
+        if value.is_a? Hash 
+          result[:fields] << { name: key, type: avro_schema_hash(value, key.to_s.singularize.camelize) }
+        elsif value.is_a? Array
+          result[:fields] << { name: key, type: { type: 'array', items: RUBY_AVRO_TYPE_MAPPING[value[0].class.name.to_sym] } }
+        else
+          result[:fields] << { name: key, type: RUBY_AVRO_TYPE_MAPPING[value.class.name.to_sym] }
+        end
+      end
+      
+      result
+    rescue StandardError => error
+      puts("Error: #{error.message}\n#{error.backtrace.join("\n")}")
+      raise InvalidDataException.new(error.message)
+    end
+
+    def avro_schema(hash_data, record_type_name)
+      Avro::Schema.parse(avro_schema_hash(hash_data, record_type_name).to_json)
+    end
+
+    def json_to_avro(json_data, record_type_name)
+      buffer = StringIO.new
+
+      # Avro assumes that all the hashes use strings for keys. It does not accept Indifferent Hash.
+      # So the conversion bellow ensures that the hash data uses strings for keys.
+      json_data = json_data.to_json if json_data.is_a? Hash
+      hash_data = JSON.parse(json_data)
+
+      schema = avro_schema(hash_data, record_type_name)
+
+      datum_writer = Avro::IO::DatumWriter.new(schema)
+      file_writer  = Avro::DataFile::Writer.new(buffer, datum_writer, schema)
+      file_writer << hash_data
+      file_writer.flush
+
+      buffer.rewind
+      buffer
+    rescue StandardError => error
+      puts("Error: #{error.message}\n#{error.backtrace.join("\n")}")
+      raise InvalidDataException.new(error.message)
+    end
+  
+  end
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: avro_utils
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Linh Chau
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json-schema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: avro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+description: A gem to convert different data format to avro
+email: chauhonglinh@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- "./avro_utils.gemspec"
+- lib/avro_utils.rb
+- lib/exceptions/invalid_data_exception.rb
+- lib/utils/avro_utils.rb
+homepage: https://github.com/linhchauatl/avro_utils
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: A gem to convert different data format to avro
+test_files: []