harmonized_tariff 0.0.4

data/harmonized_tariff.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'harmonized_tariff/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "harmonized_tariff"
+  spec.version       = HarmonizedTariff::VERSION
+  spec.authors       = ["Todd Treece"]
+  spec.email         = ["toddtreece@gmail.com"]
+  spec.description   = %q{A utility for converting the Harmonized Tariff Schedule from hts.usitc.gov to JSON, SQL, and XML.}
+  spec.summary       = %q{Harmonized Tariff Schedule Converter}
+  spec.homepage      = "http://github.com/toddtreece/harmonized_tariff"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = ["hts"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "builder"
+  spec.add_dependency "heredoc_unindent"
+
+end

data/lib/harmonized_tariff.rb

@@ -0,0 +1,6 @@
+require 'harmonized_tariff/convert'
+require 'harmonized_tariff/cli'
+
+module HarmonizedTariff
+
+end

data/lib/harmonized_tariff/cli.rb

@@ -0,0 +1,105 @@
+require 'optparse'
+require 'optparse/time'
+require 'zlib'
+
+module HarmonizedTariff
+
+  class CLI
+
+    @@type = 'json'
+    @@verbose = false
+    @@destination = '~/'
+
+    def self.parse(args)
+
+      hts = HarmonizedTariff::Convert.new
+
+      opt_parser = OptionParser.new do |opts|
+
+        opts.banner = "Usage: bundle exec bin/hts [options]"
+
+        opts.separator ""
+        opts.separator "Specific options:"
+
+        opts.on("-s", "--source [PATH]", "Set source file") do |source|
+          hts.source = source || hts.source
+        end
+
+        opts.on("-o", "--output [PATH]", "Set destination folder path") do |path|
+          @@destination = path || destination
+        end
+
+        opts.on("--type [TYPE]", "Select output format (json, xml, sql, gz (gzipped sql))") do |t|
+          @@type = t.downcase
+        end
+
+        opts.on("-v", "--verbose", "Print output to screen") do |d|
+          @@verbose = d
+        end
+
+        opts.separator ""
+        opts.separator "Common options:"
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+
+        opts.on_tail("--version", "Show version") do
+          puts HarmonizedTariff::VERSION
+          exit
+        end
+
+      end
+
+      opt_parser.parse!(args)
+
+      # parse the file
+      hts.load
+
+      if @@type == 'json'
+        self.output hts.toJSON
+      elsif @@type == 'xml'
+        self.output hts.toXML
+      elsif @@type == 'sql'
+        self.output hts.toSQL
+      elsif @@type == 'gz'
+        self.gzip hts.toSQL
+      end
+
+    end
+
+    def self.output(data)
+
+      path = File.absolute_path(@@destination) + File::SEPARATOR + 'hts.' + @@type
+
+      puts 'Outputting converted ' + @@type.upcase + ' to: ' + path + "\n"
+
+      target = File.new(path, 'w')
+      target.write data
+      target.close
+
+      if @@verbose
+        puts data
+      end
+
+    end
+
+    def self.gzip(data)
+
+      path = File.expand_path(@@destination) + File::SEPARATOR + 'hts.' + @@type
+
+      puts 'Outputting converted gzipped SQL to: ' + path + "\n"
+
+      File.open(path, 'w') do |f|
+        gz = Zlib::GzipWriter.new(f)
+        gz.write data
+        gz.close
+      end
+
+    end
+
+  end
+
+end
+

data/lib/harmonized_tariff/convert.rb

@@ -0,0 +1,184 @@
+require "csv"
+require "json"
+require "active_support/core_ext"
+require "heredoc_unindent"
+
+module HarmonizedTariff
+
+  class Convert
+
+    attr_accessor :source, :encoding, :delimeter
+
+    def initialize
+
+      spec = Gem::Specification.find_by_name("harmonized_tariff")
+      gem_root = spec.gem_dir
+
+      @source = gem_root + '/data/1300_HTS_delimited.txt'
+      @encoding = 'ISO8859-1'
+      @delimeter = '|'
+      @raw = Array.new
+
+    end
+
+    def toJSON
+      return @data.to_json
+    end
+
+    def toXML
+      return @data.to_xml(:root => 'tariffs')
+    end
+
+    def toSQL
+
+      sql = <<-SQL.unindent
+              -- create the table
+              CREATE TABLE `harmonized_tariffs` (
+                `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+                `code` varchar(255) NOT NULL DEFAULT '',
+                `suffix` varchar(255) DEFAULT NULL,
+                `description` text,
+                `unit` varchar(255) DEFAULT NULL,
+                `rate_1` varchar(255) DEFAULT NULL,
+                `special_rate` varchar(255) DEFAULT '',
+                `rate_2` varchar(255) DEFAULT NULL,
+                `notes` text,
+                PRIMARY KEY (`id`),
+                KEY `tariff` (`code`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+              -- insert the flattened data
+            SQL
+
+      @data.each do |child|
+        sql << self.buildSQL(child)
+      end
+
+      return sql
+
+    end
+
+    def buildSQL(parent)
+
+      sql = ''
+
+      unless parent[:number].nil?
+        sql << self.sqlRow(parent)
+      end
+
+      if parent[:children].nil?
+        return sql
+      end
+
+      parent[:children].each do |child|
+
+        child[:description] = parent[:description] + ' ' + child[:description]
+
+        sql << self.buildSQL(child)
+
+      end
+
+      return sql
+
+    end
+
+    def sqlRow(row)
+
+      clean = Hash.new
+
+      row.each do |k,v|
+
+        if k == :children
+          next
+        end
+
+        if v.nil?
+          clean[k] = 'NULL'
+        else
+          clean[k] = "'" + self.escape(v) + "'"
+        end
+
+      end
+
+      sql = <<-SQL.unindent
+              INSERT INTO `harmonized_tariffs` (`id`, `code`, `suffix`, `description`, `unit`, `rate_1`, `special_rate`, `rate_2`, `notes`)
+              VALUES (NULL, #{clean[:number]}, #{clean[:suffix]}, #{clean[:description]}, #{clean[:unit]}, #{clean[:col1_rate]}, #{clean[:special_rate]}, #{clean[:col2_rate]}, #{clean[:note]});
+            SQL
+
+
+      return sql
+
+    end
+
+    def escape(text)
+
+      text = text.gsub(/'/) {|s| "\\'"}
+      text = text.gsub(/"/) {|s| "\""}
+      text = text.gsub(/\\/) {|s| "\\"}
+
+      return text
+
+    end
+
+    def load
+
+      parent = nil
+      previous = nil
+
+      @raw = CSV.read(@source, col_sep: @delimeter, encoding: @encoding, quote_char: "\x00")
+
+      root = self.build_heirarchy(Hash.new, 0, 1)
+
+      @data = root[:children]
+
+      @raw = nil
+      root = nil
+
+    end
+
+    def build_heirarchy(parent, depth, index)
+
+      siblings = Array.new
+
+      while index < @raw.length
+
+        current = Hash.new
+
+        current[:number], current[:suffix], level, current[:description], current[:unit], current[:col1_rate], current[:special_rate], current[:col2_rate], current[:note] = @raw[index]
+
+        if level.nil?
+          index += 1
+          next
+        end
+
+        if level.to_i == depth
+          index += 1
+          siblings.push self.build_heirarchy(current, depth + 1, index)
+          next
+        end
+
+        if level.to_i < depth
+
+          if siblings.length > 0
+            parent[:children] = siblings
+          end
+
+          return parent
+
+        end
+
+        index += 1
+
+      end
+
+      if siblings.length > 0
+        parent[:children] = siblings
+      end
+
+      return parent
+
+    end
+
+  end
+
+end
+

data/lib/harmonized_tariff/version.rb

@@ -0,0 +1,3 @@
+module HarmonizedTariff
+  VERSION = "0.0.4"
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: harmonized_tariff
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+  prerelease: 
+platform: ruby
+authors:
+- Todd Treece
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: builder
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: heredoc_unindent
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A utility for converting the Harmonized Tariff Schedule from hts.usitc.gov
+  to JSON, SQL, and XML.
+email:
+- toddtreece@gmail.com
+executables:
+- hts
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/hts
+- bin/rake
+- data/1300_HTS_delimited.txt
+- data/hts.gz
+- data/hts.json
+- data/hts.sql
+- data/hts.xml
+- harmonized_tariff.gemspec
+- lib/harmonized_tariff.rb
+- lib/harmonized_tariff/cli.rb
+- lib/harmonized_tariff/convert.rb
+- lib/harmonized_tariff/version.rb
+homepage: http://github.com/toddtreece/harmonized_tariff
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Harmonized Tariff Schedule Converter
+test_files: []