ad_hoc_template 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac72ef7ef92a4a6aa9be31762b2cc29177cecd44
-  data.tar.gz: 6fd32a26232fbb869f73632610c301e470e3178b
+  metadata.gz: 476f29030bdd0bf5050ef7e961affd45a4a64667
+  data.tar.gz: 04901a83516ee55be4a9f887d4c050f12cabbd4b
 SHA512:
-  metadata.gz: e6935f638dd7c1cb8979c86bbfbd653ec75a7a4aca0d7579391026b47e60840b55a4444696cef0f2203bf6a5460b95822324b7322754cd9a215fb92e357a76a7
-  data.tar.gz: 42d8b17110f59b24fe529029da629fa4150ce4c531b15721d47e7783da12edee4acf56086a6f3ec86b03d84197bdcf71967a9c989d4a0a784972682e6c27fe13
+  metadata.gz: 04719a11683310dc6ff684782ea384f1c2a55056ec99d1ab845660f763c7a90ca000e7fa75670dc9537bfbfd047563e41ad24a9ba709de0b438d8562da83288d
+  data.tar.gz: bd458ab6de4d391f4a4ea7fafce0412841f46f449c4b88e8a37b08c53c08ed38b48ea9fb099bdb14a1f12323d7595843c04c222a977480246d83973361681bf5

data/.gitignore

@@ -15,3 +15,5 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+*~
+.ruby-version

data/Gemfile

@@ -2,7 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in ad_hoc_template.gemspec
 
-gem 'pseudohikiparser', :git => 'https://github.com/nico-hn/PseudoHikiParser.git', :tag => '0.0.2'
+gem 'pseudohikiparser', '0.0.5.develop'
 
 group :development do
   gem "bundler", "~> 1.3"

data/README.md

@@ -63,7 +63,7 @@ the second paragraph in block
 2. Execute the following at the command line:
 
 ```
-ad_hoc_template template.txt sample_data.txt
+$ ad_hoc_template template.txt sample_data.txt
 ```
 
 Then you will get the following result:
@@ -77,7 +77,6 @@ the value of sub_key2 is value1-2
 the value of sub_key1 is value2-1
 the value of sub_key2 is value2-2
 
-
 the first line of block
 the second line of block
 

data/ad_hoc_template.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
-  spec.add_runtime_dependency "pseudohikiparser", "0.0.2"
+  spec.add_runtime_dependency "pseudohikiparser", "0.0.5.develop"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake", "~> 10.1"

data/lib/ad_hoc_template/command_line_interface.rb

@@ -5,11 +5,36 @@ require 'optparse'
 
 module AdHocTemplate
   class CommandLineInterface
-    attr_accessor :output_filename, :template_data, :record_data
+    attr_accessor :output_filename, :template_data, :record_data, :tag_type, :data_format
+
+    TAG_RE_TO_TYPE = {
+      /\Ad(efault)?/i => :default,
+      /\Ac(urly_brackets)?/i => :curly_brackets,
+      /\As(quare_brackets)?/i => :square_brackets,
+      /\Axml_like1/i => :xml_like1,
+      /\Axml_like2/i => :xml_like2,
+    }
+
+    FORMAT_RE_TO_FORMAT = {
+      /\Ad(efault)?/i => :default,
+      /\Ay(a?ml)?/i => :yaml,
+      /\Aj(son)?/i => :json,
+      /\Ac(sv)?/i => :csv,
+      /\At(sv)?/i => :tsv,
+    }
+
+    FILE_EXTENTIONS = {
+      /\.ya?ml\Z/i => :yaml,
+      /\.json\Z/i => :json,
+      /\.csv\Z/i => :csv,
+      /\.tsv\Z/i => :tsv,
+    }
 
     def initialize
-      @formatter = AdHocTemplate::DefaultTagFormatter.new
+      @tag_formatter = AdHocTemplate::DefaultTagFormatter.new
       @output_filename = nil
+      @tag_type = :default
+      @data_format = nil
     end
 
     def set_encoding(given_opt)
@@ -19,7 +44,7 @@ module AdHocTemplate
     end
 
     def parse_command_line_options
-      OptionParser.new do |opt|
+      OptionParser.new("USAGE: #{File.basename($0)} [OPTION]... TEMPLATE_FILE DATA_FILE") do |opt|
         opt.on("-E [ex[:in]]", "--encoding [=ex[:in]]",
                "Specify the default external and internal character encodings (same as the option of MRI") do |given_opt|
           self.set_encoding(given_opt)
@@ -30,7 +55,27 @@ module AdHocTemplate
           @output_filename = File.expand_path(output_file)
         end
 
-       opt.parse!
+        opt.on("-t [tag_type]", "--tag-type [=tag_type]",
+               "Choose a template tag type: default, curly_brackets or square_brackets") do |given_type|
+          choose_tag_type(given_type)
+        end
+
+        opt.on("-d [data_format]", "--data-format [=data_format]",
+               "Specify the format of input data: default, yaml, json, csv or tsv") do |data_format|
+          choose_data_format(data_format)
+        end
+
+        opt.on("-u [tag_config.yaml]","--user-defined-tag [=tag_config.yaml]",
+               "Configure a user-defined tag. The configuration file is in YAML format.") do |tag_config_yaml|
+          register_user_defined_tag_type(tag_config_yaml)
+        end
+
+        opt.parse!
+      end
+
+      unless @data_format
+        guessed_format = ARGV.length < 2 ? :default : guess_file_format(ARGV[1])
+        @data_format =  guessed_format || :default
       end
     end
 
@@ -46,7 +91,8 @@ module AdHocTemplate
     end
 
     def convert
-      AdHocTemplate::Converter.convert(@record_data, @template_data, @formatter)
+      AdHocTemplate.convert(@record_data, @template_data, @tag_type,
+                            @data_format, @tag_formatter)
     end
 
     def open_output
@@ -62,9 +108,50 @@ module AdHocTemplate
     def execute
       parse_command_line_options
       read_input_files
-      open_output do |out|
-        out.print convert
+      open_output {|out| out.print convert }
+    end
+
+    private
+
+    def choose_tag_type(given_type)
+      if_any_regex_match(TAG_RE_TO_TYPE, given_type,
+                         "The given type is not found. The default tag is chosen.") do |re, tag_type|
+        @tag_type = tag_type
+      end
+    end
+
+    def choose_data_format(data_format)
+      if_any_regex_match(FORMAT_RE_TO_FORMAT, data_format,
+                         "The given format is not found. The default format is chosen.") do |re, format|
+        @data_format = [:csv, :tsv].include?(format) ? make_csv_option(data_format, format) : format
+      end
+    end
+
+    def register_user_defined_tag_type(tag_config_yaml)
+      config = File.read(File.expand_path(tag_config_yaml))
+      @tag_type = Parser.register_user_defined_tag_type(config)
+    end
+
+    def make_csv_option(data_format, format)
+      iteration_label = data_format.sub(/\A(csv|tsv):?/, "")
+      iteration_label.empty? ? format : { format => iteration_label }
+    end
+
+    def guess_file_format(filename)
+      if_any_regex_match(FILE_EXTENTIONS, filename) do |ext_re, format|
+        return format
+      end
+    end
+
+    def if_any_regex_match(regex_table, target, failure_message=nil)
+      regex_table.each do |re, paired_value|
+        if re =~ target
+          yield re, paired_value
+          return
+        end
       end
+      STDERR.puts failure_message if failure_message
+      nil
     end
   end
 end

data/lib/ad_hoc_template/parser.rb

@@ -0,0 +1,160 @@
+#!/usr/bin/env ruby
+
+require "pseudohiki/inlineparser"
+require "htmlelement"
+
+module AdHocTemplate
+  class Parser < TreeStack
+    class TagNode < Parser::Node
+      attr_reader :type
+
+      def push(node=TreeStack::Node.new)
+        node[0] = assign_type(node[0]) if self.empty?
+        super
+      end
+
+      def assign_type(first_leaf)
+        if not first_leaf.kind_of? String or /\A\s/ =~ first_leaf
+          return first_leaf.sub(/\A(?:\r?\n|\r)/, "")
+        end
+        @type, first_leaf_content = split_by_newline_or_spaces(first_leaf)
+        @type = '#'.freeze + @type if kind_of? IterationTagNode
+        first_leaf_content||""
+      end
+
+      def split_by_newline_or_spaces(first_leaf)
+        sep = /\A\S*(?:\r?\n|\r)/ =~ first_leaf ? /(?:\r?\n|\r)/ : /\s+/
+        first_leaf.split(sep, 2)
+      end
+      private :assign_type, :split_by_newline_or_spaces
+
+      def contains_any_value_assigned_tag_node?(record)
+        self.select {|n| n.kind_of?(TagNode) }.each do |node|
+          if node.kind_of? IterationTagNode
+            return true if any_value_assigned_to_iteration_tag?(node, record)
+          else
+            val = record[node.join.strip]
+            return true if val and not val.empty?
+          end
+        end
+        false
+      end
+
+      private
+
+      def empty_sub_records?(record, node)
+        sub_records = record[node.type]
+        return true if sub_records.nil? or sub_records.empty?
+        sub_records.each do |rec|
+          return false if rec.values.find {|val| val and not val.empty? }
+        end
+      end
+
+      def any_value_assigned_to_iteration_tag?(tag_node, record)
+        if tag_node.type
+          not empty_sub_records?(record, tag_node)
+        else
+          tag_node.contains_any_value_assigned_tag_node?(record)
+        end
+      end
+    end
+
+    class IterationTagNode < TagNode; end
+    class Leaf < Parser::Leaf; end
+
+    class TagType
+      attr_reader :head, :tail, :token_pat, :remove_iteration_indent
+      attr_reader :iteration_start, :iteration_end
+      @types = {}
+
+      def self.[](tag_name)
+        @types[tag_name]
+      end
+
+      def self.register(tag_name=:default, tag=["<%", "%>"], iteration_tag=["<%#", "#%>"],
+                        remove_iteration_indent=false)
+        @types[tag_name] = new(tag, iteration_tag, remove_iteration_indent)
+      end
+
+      def initialize(tag, iteration_tag, remove_iteration_indent)
+        assign_type(tag, iteration_tag)
+        @token_pat = PseudoHiki.compile_token_pat(@head.keys, @tail.keys)
+        @remove_iteration_indent = remove_iteration_indent
+      end
+
+      def assign_type(tag, iteration_tag)
+        @iteration_start, @iteration_end = iteration_tag
+        @head, @tail = {}, {}
+        [
+          [TagNode, tag],
+          [IterationTagNode, iteration_tag]
+        ].each do |node_type, head_tail|
+          head, tail = head_tail
+          @head[head] = node_type
+          @tail[tail] = node_type
+        end
+      end
+
+      register
+      register(:square_brackets, ["[[", "]]"], ["[[#", "#]]"])
+      register(:curly_brackets, ["{{", "}}"], ["{{#", "#}}"])
+      register(:xml_like1, ["<!--%", "%-->"], ["<iterate>", "</iterate>"], true)
+      register(:xml_like2, ["<fill>", "</fill>"], ["<iterate>", "</iterate>"], true)
+      register(:xml_comment_like, ["<!--%", "%-->"], ["<!--%iterate%-->", "<!--%/iterate%-->"], true)
+    end
+
+    class UserDefinedTagTypeConfigError < StandardError; end
+
+    def self.parse(str, tag_name=:default)
+      if TagType[tag_name].remove_iteration_indent
+        str = remove_indent_before_iteration_tags(str, TagType[tag_name])
+      end
+      new(str, TagType[tag_name]).parse.tree
+    end
+
+    def self.remove_indent_before_iteration_tags(template_source, tag_type)
+      [
+        tag_type.iteration_start,
+        tag_type.iteration_end
+      ].inject(template_source) do |s, tag|
+        s.gsub(/^([ \t]+#{Regexp.escape(tag)}(?:\r?\n|\r))/) { $1.lstrip }
+      end
+    end
+
+    def self.register_user_defined_tag_type(config_source)
+      config = YAML.load(config_source)
+      %w(tag_name tag iteration_tag).each do |item|
+        config[item] || raise(UserDefinedTagTypeConfigError,
+                              "\"#{item}\" should be defined.")
+      end
+      TagType.register(registered_tag_name = config["tag_name"].to_sym,
+                       config["tag"],
+                       config["iteration_tag"],
+                       config["remove_indent"] || false)
+      registered_tag_name
+    end
+
+    def initialize(str, tag)
+      @tag = tag
+      str = remove_trailing_newline_of_iteration_end_tag(str, @tag.iteration_end)
+      @tokens = PseudoHiki.split_into_tokens(str, @tag.token_pat)
+      super()
+    end
+
+    def parse
+      while token = @tokens.shift
+        next if @tag.tail[token] == current_node.class and self.pop
+        next if @tag.head[token] and self.push @tag.head[token].new
+        self.push Leaf.create(token)
+      end
+
+      self
+    end
+
+    private
+
+    def remove_trailing_newline_of_iteration_end_tag(str, iteration_end_tag)
+      str.gsub(/#{Regexp.escape(iteration_end_tag)}(?:\r?\n|\r)/, iteration_end_tag)
+    end
+  end
+end

data/lib/ad_hoc_template/record_reader.rb

@@ -0,0 +1,281 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'json'
+require 'csv'
+
+module AdHocTemplate
+  module RecordReader
+    module YAMLReader
+      def self.read_record(yaml_data)
+        YAML.load(yaml_data)
+      end
+
+      def self.to_yaml(config_data)
+        data = RecordReader.read_record(config_data)
+        YAML.dump(data)
+      end
+    end
+
+    module JSONReader
+      def self.read_record(json_data)
+        JSON.parse(json_data)
+      end
+
+      def self.to_json(config_data)
+        data = RecordReader.read_record(config_data)
+        JSON.dump(data)
+      end
+    end
+
+    module CSVReader
+      def self.read_record(csv_data, config={ csv: nil })
+        label, sep  = parse_config(config)
+        header, *data = CSV.new(csv_data, col_sep: sep).to_a
+        records = data.map {|row| convert_to_hash(header, row) }
+        if label
+          { '#' + label => records }
+        elsif records.length == 1
+          records[0]
+        else
+          records
+        end
+      end
+
+      def self.convert_to_hash(header, row_array)
+        {}.tap do |record|
+          header.zip(row_array).each do |key, value|
+            record[key] = value
+          end
+        end
+        # if RUBY_VERSION >= 2.1.0: header.zip(row_array).to_h
+      end
+
+      def self.parse_config(config)
+        case config
+        when Symbol
+          format, label = config, nil
+        when String
+          format, label = :csv, config
+        when Hash
+          format, label = config.to_a[0]
+        end
+        field_sep = format == :tsv ? "\t" : CSV::DEFAULT_OPTIONS[:col_sep]
+        return label, field_sep
+      end
+
+      private_class_method :convert_to_hash
+    end
+
+    SEPARATOR = /:\s*/o
+    BLOCK_HEAD = /\A\/\/@/o
+    ITERATION_HEAD = /\A\/\/@#/o
+    EMPTY_LINE = /\A(?:\r?\n|\r)\Z/o
+    ITERATION_MARK = /\A#/o
+    READERS_RE = {
+      key_value: SEPARATOR,
+      iteration: ITERATION_HEAD,
+      block: BLOCK_HEAD,
+      empty_line: EMPTY_LINE,
+    }
+
+    class ReaderState
+      attr_accessor :current_block_label
+
+      def initialize(config={}, stack=[])
+        @stack = stack
+        @configs = [config]
+        setup_reader
+      end
+
+      def push(reader)
+        @stack.push reader
+      end
+
+      def pop
+        @stack.pop unless @stack.length == 1
+      end
+
+      def setup_stack(line)
+        @stack[-1].setup_stack(line)
+      end
+
+      def current_reader
+        @stack[-1]
+      end
+
+      def read(line)
+        @stack[-1].read(line)
+      end
+
+      def push_new_record
+        new_record = {}
+        @configs.push new_record
+        new_record
+      end
+
+      def pop_current_record
+        @configs.pop
+      end
+
+      def current_record
+        @configs[-1]
+      end
+
+      def parsed_record
+        @configs[0]
+      end
+
+      def read_record(lines)
+        lines = lines.each_line.to_a if lines.kind_of? String
+        lines.each do |line|
+          setup_stack(line)
+          read(line)
+        end
+        remove_trailing_empty_lines_from_last_block!
+        parsed_record
+      end
+
+      def last_block_value
+        current_record[current_block_label]
+      end
+
+      def remove_trailing_empty_lines_from_last_block!
+        if current_reader.kind_of? BlockReader
+          last_block_value.sub!(/(#{$/})+\Z/, $/)
+        end
+      end
+
+      private
+
+      def setup_reader
+        Reader.setup_reader(self)
+      end
+    end
+
+    class Reader
+      def self.setup_reader(stack)
+        readers = {}
+        {
+          base: BaseReader,
+          key_value: KeyValueReader,
+          block: BlockReader,
+          iteration: IterationReader,
+        }.each do |k, v|
+          readers[k] = v.new(stack, readers)
+        end
+        stack.push readers[:base]
+        readers
+      end
+
+      def initialize(stack, readers)
+        @stack = stack
+        @readers = readers
+      end
+
+      def pop_stack
+        @stack.pop
+      end
+
+      def read(line)
+      end
+
+      private
+
+      def push_reader_if_match(line, readers)
+        readers.each do |reader|
+          return @stack.push(@readers[reader]) if READERS_RE[reader] === line
+        end
+      end
+
+      def setup_new_block(line, initial_value)
+        label = line.sub(BLOCK_HEAD, "").chomp
+        @stack.current_record[label] ||= initial_value
+        @stack.current_block_label = label
+      end
+    end
+
+
+    class BaseReader < Reader
+      def setup_stack(line)
+        push_reader_if_match(line, [:iteration, :block, :key_value])
+      end
+    end
+
+    class KeyValueReader < Reader
+      def setup_stack(line)
+        case line
+        when EMPTY_LINE, ITERATION_HEAD, BLOCK_HEAD
+          pop_stack
+        end
+        push_reader_if_match(line, [:iteration, :block])
+      end
+
+      def read(line)
+        key, value = line.split(SEPARATOR, 2)
+        @stack.current_record[key] = value.chomp
+      end
+    end
+
+    class BlockReader < Reader
+      def setup_stack(line)
+        case line
+        when ITERATION_HEAD, BLOCK_HEAD
+          @stack.remove_trailing_empty_lines_from_last_block!
+          pop_stack
+        end
+        push_reader_if_match(line, [:iteration, :block])
+      end
+
+      def read(line)
+        block_value = @stack.last_block_value
+        case line
+        when BLOCK_HEAD
+          setup_new_block(line, String.new)
+        when EMPTY_LINE
+          block_value << line unless block_value.empty?
+        else
+          block_value << line
+        end
+      end
+    end
+
+    class IterationReader < Reader
+      def setup_stack(line)
+        case line
+        when ITERATION_HEAD
+          @stack.pop_current_record
+        when BLOCK_HEAD
+          @stack.pop_current_record
+          pop_stack
+          @stack.push @readers[:block]
+        when SEPARATOR
+          @stack.pop_current_record
+          @stack.last_block_value.push @stack.push_new_record
+          @stack.push @readers[:key_value]
+        end
+      end
+
+      def read(line)
+        case line
+        when ITERATION_HEAD
+          setup_new_block(line, [])
+          @stack.push_new_record
+        end
+      end
+    end
+
+    def self.read_record(input, source_format=:default)
+      case source_format
+      when :default
+        ReaderState.new.read_record(input)
+      when :yaml
+        YAMLReader.read_record(input)
+      when :json
+        JSONReader.read_record(input)
+      when :csv, :tsv, Hash
+        CSVReader.read_record(input, source_format)
+      end
+    end
+  end
+end

data/lib/ad_hoc_template/version.rb

@@ -1,3 +1,3 @@
 module AdHocTemplate
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end