gorgeous 0.1.0

data/README.md

@@ -0,0 +1,22 @@
+Usage:
+
+    gorgeous [-i] [-F <in-format>] [-T <out-format>] [-q <query>] [-o <destination>] FILE ...
+
+This utility converts data between different formats.
+Formats are one of:  json, xml, yaml, ruby, email, url
+
+Input can be read from STDIN as well as files given on the command-line.
+
+Options:
+
+    -i    Convert file in-place
+    -F    Input format
+    -T    Output format
+    -q    Query selector in format similar to XPath (see below)
+    -o    Write output to file instead of STDOUT
+
+Query format:
+
+    "/items/comments[1]/text"    --  2nd comment body of each item
+    "/items[-1]/user/full_name"  --  name of user for last item
+    "//user/username"            --  usernames of all users anywhere in the document

data/bin/gorgeous

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+
+help = <<-MSG
+Usage: gorgeous [-i] [-F <in-format>] [-T <out-format>] [-q <query>] [-o <destination>] FILE ...
+
+This utility converts data between different formats.
+Formats are one of:  json, xml, yaml, ruby, email, url
+
+Input can be read from STDIN as well as files given on the command-line.
+
+Options:
+  -i    Convert file in-place
+  -F    Input format
+  -T    Output format
+  -q    Query selector in format similar to XPath (see below)
+  -o    Write output to file instead of STDOUT
+
+Query format:
+  "/items/comments[1]/text"    --  2nd comment body of each item
+  "/items[-1]/user/full_name"  --  name of user for last item
+  "//user/username"            --  usernames of all users anywhere in the document
+
+MSG
+
+if ARGV.index('-h') or ARGV.index('--help')
+  puts help
+  exit
+end
+
+require 'gorgeous'
+
+help_usage = help.split("\n", 2).first
+
+options = help_usage.scan(/[^\w-](-[a-z])(?: <(.+?)>)?/i).inject({}) do |opts, (flag, value)|
+  if value.nil?
+    opts[flag] = true if ARGV.delete flag
+  elsif idx = ARGV.index(flag)
+    opts[flag] = ARGV[idx + 1]
+    2.times { ARGV.delete_at idx }
+  end
+  opts
+end
+
+in_place = options['-i']
+query_path = options['-q']
+input_format = options['-F'] && options['-F'].to_sym
+output_format ||= options['-T'] && options['-T'].to_sym
+destination = options['-o']
+from_stdin = !$stdin.tty?
+
+if from_stdin
+  if in_place
+    warn "can't edit stdin in place"
+    exit 1
+  end
+elsif in_place and destination
+  warn "can't combine -i and -o flags"
+  exit 1
+elsif ARGV.empty?
+  warn help_usage
+  exit 1
+end
+
+# clobber the input file
+destination = ARGV.first if in_place
+
+input = Gorgeous.new(ARGF, :filename => ARGV.first, :format => input_format, :query => query_path)
+
+unless input.format
+  warn "unrecognized input format; specify explicit format with `-F <format>`"
+  exit 1
+end
+
+output_format ||= if destination
+  Gorgeous.filename_to_format(destination)
+else
+  input.format
+end
+
+if output_format == :ruby
+  require 'pp'
+  output = input.data.pretty_inspect
+elsif output_format == :json
+  require 'yajl/json_gem'
+  output = JSON.pretty_generate input.data
+elsif output_format == :yaml
+  require 'yaml'
+  output = YAML.dump input.data
+elsif input_format == :email and output_format == :xml
+  require 'nokogiri'
+  email = input.to_mail
+  part = email.html_part
+  html = Gorgeous.convert_utf8(part.decode_body, part.charset)
+  ugly = Nokogiri::HTML html
+  output = Gorgeous.pretty_xml(ugly)
+elsif input_format == :email and output_format == :txt and not input.filtered?
+  email = input.to_mail
+  part = email.text_part
+  output = Gorgeous.convert_utf8(part.decode_body.strip, part.charset)
+elsif output_format == :xml
+  output = Gorgeous.pretty_xml(input.to_xml)
+elsif output_format == :url
+  require 'rack/utils'
+  output = Rack::Utils.build_nested_query(input.data)
+elsif output_format == :txt
+  data = input.data
+  output = Array === data ? data.join("\n") : data.to_s
+else
+  warn "don't know how to convert #{input_format} to #{output_format}"
+  exit 1
+end
+
+if destination
+  begin
+    File.open(destination, 'w') { |f| f << output }
+  rescue Errno::ENOENT
+    warn "error opening #{destination} for writing"
+    exit 1
+  end
+else
+  begin
+    puts output
+  rescue Errno::EPIPE
+    # ignore "broken pipe" exception
+  end
+end

data/lib/gorgeous.rb

@@ -0,0 +1,241 @@
+class Gorgeous
+  def self.filename_to_format(filename)
+    case File.extname(filename)
+    when '.json' then :json
+    when '.xml', '.html' then :xml
+    when '.rb' then :ruby
+    when '.yml', '.yaml' then :yaml
+    when '.mail', '.email' then :email
+    end
+  end
+
+  def self.pretty_xml(ugly)
+    tidy = Nokogiri::XSLT DATA.read
+    tidy.transform(ugly).to_s
+  end
+
+  def self.convert_utf8(string, from_charset)
+    if from_charset.nil? or from_charset.downcase.tr('-', '') == 'utf8'
+      string
+    else
+      require 'iconv'
+      Iconv.conv 'utf-8', from_charset, string
+    end
+  end
+
+  def self.headers_from_mail(email)
+    require 'active_support/ordered_hash'
+    require 'active_support/core_ext/object/blank'
+
+    address_field = lambda { |name|
+      if field = email.header[name]
+        values = field.addrs.map { |a|
+          Mail::Encodings.unquote_and_convert_to(a.format, 'utf-8')
+        }
+        values.size < 2 ? values.first : values
+      end
+    }
+    header_value = lambda { |name|
+      field = email.header[name] and field.value.to_s
+    }
+    decoded_value = lambda { |name|
+      field = email.header[name]
+      Mail::Encodings.unquote_and_convert_to(field.value, 'utf-8') if field
+    }
+
+    data = ActiveSupport::OrderedHash.new
+    data[:subject] = decoded_value['subject']
+    data[:from] = address_field['from']
+    data[:to] = address_field['to']
+    data[:cc] = address_field['cc']
+    data[:bcc] = address_field['bcc']
+    data[:reply_to] = address_field['reply-to']
+    data[:return_path] = email.return_path
+
+    data[:message_id] = email.message_id
+    data[:in_reply_to] = email.in_reply_to
+    data[:references] = email.references
+
+    data[:date] = email.date
+    data[:sender] = address_field['sender']
+    data[:delivered_to] = header_value['delivered-to']
+    data[:original_sender] = header_value['x-original-sender']
+    data[:content_type] = email.content_type.to_s.split(';', 2).first.presence
+    data[:precedence] = header_value['precedence']
+
+    data.tap { |hash| hash.reject! { |k,v| v.nil? } }
+  end
+
+  # adapted from webmock
+  def self.http_from_string(raw_response)
+    if raw_response.is_a?(IO)
+      string = raw_response.read
+      raw_response.close
+      raw_response = string
+    end
+    socket = ::Net::BufferedIO.new(raw_response)
+    response = ::Net::HTTPResponse.read_new(socket)
+    transfer_encoding = response.delete('transfer-encoding') # chunks were already read
+    response.reading_body(socket, true) {}
+
+    options = {}
+    options[:headers] = {}
+    response.each_header { |name, value| options[:headers][name] = value }
+    options[:headers]['transfer-encoding'] = transfer_encoding if transfer_encoding
+    options[:body] = response.read_body
+    options[:status] = [response.code.to_i, response.message]
+    options
+  end
+
+  def initialize(input, options = {})
+    @input = input
+    @format = options[:format]
+    @options = options
+  end
+
+  def filtered?
+    !!@options[:query]
+  end
+
+  def to_s
+    @str ||= @input.respond_to?(:read) ? @input.read : @input
+  end
+
+  def format
+    @format ||= begin
+      if @options[:filename]
+        self.class.filename_to_format(@options[:filename])
+      else
+        guess_format
+      end
+    end
+  end
+
+  # typically hash or array
+  def data
+    apply_query case format
+    when :xml
+      require 'active_support/core_ext/hash/conversions'
+      Hash.from_xml(to_s)
+    when :json
+      require 'yajl/json_gem'
+      JSON.parse to_s
+    when :yaml
+      require 'yaml'
+      YAML.load to_s
+    when :email
+      self.class.headers_from_mail to_mail
+    when :ruby
+      eval to_s # TODO: sandbox
+    when :url
+      require 'rack/utils'
+      Rack::Utils.parse_nested_query(to_s.strip)
+    when :http
+      require 'net/http'
+      self.class.http_from_string(to_s)[:headers]
+    else
+      raise ArgumentError, "don't know how to decode #{format}"
+    end
+  end
+
+  def to_xml
+    require 'nokogiri'
+    Nokogiri to_s
+  end
+
+  def to_mail
+    require 'mail'
+    raw = to_s.lstrip
+    raw << "\n" unless raw[-1, 1] == "\n"
+    Mail.new raw
+  end
+
+  private
+
+  def guess_format
+    case to_s
+    when /\A\s*[\[\{]/ then :json
+    when /\A\s*</ then :xml
+    when /\A---\s/ then :yaml
+    when /\A\S+=\S+\Z/ then :url
+    end
+  end
+
+  def apply_query(obj)
+    if filtered?
+      require 'active_support/core_ext/object/blank'
+      query = @options[:query].dup
+      while query.sub!(%r{(//?)(\w+)(?:\[(-?\d+)\])?}, '')
+        obj = filter_resultset(obj, $2, $3, $1.length == 2)
+        break if obj.nil?
+      end
+    end
+    obj
+  end
+
+  def filter_resultset(obj, key, idx = nil, deep = false)
+    if Array === obj
+      obj.map { |o| filter_resultset(o, key, idx, deep) }.flatten.compact.presence
+    elsif Hash === obj
+      if deep
+        result = obj.map do |k, value|
+          if k.to_s == key.to_s then value
+          else filter_resultset(value, key, nil, deep)
+          end
+        end.flatten.compact.presence
+      else
+        result = obj[key.to_s] || obj[key.to_sym]
+      end
+      result = result[idx.to_i] if result and idx
+      result
+    end
+  end
+end
+
+__END__
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+  <xsl:output method="xml" encoding="UTF-8"/>
+  <xsl:param name="indent-increment" select="'   '"/>
+
+  <xsl:template name="newline">
+    <xsl:text disable-output-escaping="yes">
+</xsl:text>
+  </xsl:template>
+
+  <xsl:template match="comment() | processing-instruction()">
+    <xsl:param name="indent" select="''"/>
+    <xsl:call-template name="newline"/>
+    <xsl:value-of select="$indent"/>
+    <xsl:copy />
+  </xsl:template>
+
+  <xsl:template match="text()">
+    <xsl:param name="indent" select="''"/>
+    <xsl:call-template name="newline"/>
+    <xsl:value-of select="$indent"/>
+    <xsl:value-of select="normalize-space(.)"/>
+  </xsl:template>
+
+  <xsl:template match="text()[normalize-space(.)='']"/>
+
+  <xsl:template match="*">
+    <xsl:param name="indent" select="''"/>
+    <xsl:call-template name="newline"/>
+    <xsl:value-of select="$indent"/>
+      <xsl:choose>
+       <xsl:when test="count(child::*) > 0">
+        <xsl:copy>
+         <xsl:copy-of select="@*"/>
+         <xsl:apply-templates select="*|text()">
+           <xsl:with-param name="indent" select="concat ($indent, $indent-increment)"/>
+         </xsl:apply-templates>
+         <xsl:call-template name="newline"/>
+         <xsl:value-of select="$indent"/>
+        </xsl:copy>
+       </xsl:when>
+       <xsl:otherwise>
+        <xsl:copy-of select="."/>
+       </xsl:otherwise>
+     </xsl:choose>
+  </xsl:template>
+</xsl:stylesheet>

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification 
+name: gorgeous
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- "Mislav Marohni\xC4\x87"
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-02-10 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: 
+email: mislav.marohnic@gmail.com
+executables: 
+- gorgeous
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/gorgeous
+- lib/gorgeous.rb
+- README.md
+has_rdoc: false
+homepage: http://github.com/mislav/gorgeous
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.0
+signing_key: 
+specification_version: 3
+summary: Convert between different data formats
+test_files: []
+