bismas 0.1.0

data/ChangeLog

@@ -0,0 +1,11 @@
+# markup: rd
+
+= Revision history for bismas
+
+== 0.1.0 [2015-11-20]
+
+* First release.
+
+== 0.0.0 [2015-09-17]
+
+* Birthday :-)

data/README

@@ -0,0 +1,41 @@
+= bismas - A Ruby client for BISMAS databases
+
+== VERSION
+
+This documentation refers to bismas version 0.1.0.
+
+
+== DESCRIPTION
+
+Access BISMAS databases from Ruby.
+
+
+== LINKS
+
+Documentation:: https://blackwinter.github.com/bismas
+Source code::   https://github.com/blackwinter/bismas
+RubyGem::       https://rubygems.org/gems/bismas
+Travis CI::     https://travis-ci.org/blackwinter/bismas
+
+
+== AUTHORS
+
+* Jens Wille <mailto:jens.wille@gmail.com>
+
+
+== LICENSE AND COPYRIGHT
+
+Copyright (C) 2015 Jens Wille
+
+bismas is free software: you can redistribute it and/or modify it
+under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or (at your
+option) any later version.
+
+bismas is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with bismas. If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,23 @@
+require_relative 'lib/bismas/version'
+
+begin
+  require 'hen'
+
+  Hen.lay! {{
+    gem: {
+      name:         %q{bismas},
+      version:      Bismas::VERSION,
+      summary:      %q{A Ruby client for BISMAS databases.},
+      description:  %q{Access BISMAS databases from Ruby.},
+      author:       %q{Jens Wille},
+      email:        %q{jens.wille@gmail.com},
+      license:      %q{AGPL-3.0},
+      homepage:     :blackwinter,
+      dependencies: { cyclops: '~> 0.2', nuggets: '~> 1.4' },
+
+      required_ruby_version: '>= 2.0'
+    }
+  }}
+rescue LoadError => err
+  warn "Please install the `hen' gem. (#{err})"
+end

data/bin/bismas

@@ -0,0 +1,30 @@
+#! /usr/bin/env ruby
+
+#--
+###############################################################################
+#                                                                             #
+# bismas -- A Ruby client for BISMAS databases                                #
+#                                                                             #
+# Copyright (C) 2015 Jens Wille                                               #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# bismas is free software; you can redistribute it and/or modify it           #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# bismas is distributed in the hope that it will be useful, but               #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public      #
+# License for more details.                                                   #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with bismas. If not, see <http://www.gnu.org/licenses/>.              #
+#                                                                             #
+###############################################################################
+#++
+
+require 'bismas/cli'
+Bismas::CLI.execute

data/lib/bismas/base.rb

@@ -0,0 +1,73 @@
+#--
+###############################################################################
+#                                                                             #
+# bismas -- A Ruby client for BISMAS databases                                #
+#                                                                             #
+# Copyright (C) 2015 Jens Wille                                               #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# bismas is free software; you can redistribute it and/or modify it           #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# bismas is distributed in the hope that it will be useful, but               #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public      #
+# License for more details.                                                   #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with bismas. If not, see <http://www.gnu.org/licenses/>.              #
+#                                                                             #
+###############################################################################
+#++
+
+require 'nuggets/file/open_file'
+require 'nuggets/array/extract_options'
+
+module Bismas
+
+  class Base
+
+    class << self
+
+      private
+
+      def file_method(method, mode, file, options = {}, *args, &block)
+        Bismas.amend_encoding(options)
+
+        File.open_file(file, options, mode) { |io|
+          args.unshift(options.merge(io: io))
+          method ? send(method, *args, &block) : block[new(*args)]
+        }
+      end
+
+    end
+
+    def initialize(options = {}, &block)
+      self.key = options[:key]
+      self.io  = options.fetch(:io, self.class::DEFAULT_IO)
+
+      @auto_id_block = options.fetch(:auto_id, block)
+      @options = options
+
+      reset
+    end
+
+    attr_accessor :key, :io, :auto_id
+
+    def reset
+      @auto_id = @auto_id_block ? @auto_id_block.call : default_auto_id
+    end
+
+    private
+
+    def default_auto_id(n = 0)
+      lambda { n += 1 }
+    end
+
+  end
+
+end

data/lib/bismas/cli.rb

@@ -0,0 +1,139 @@
+#--
+###############################################################################
+#                                                                             #
+# bismas -- A Ruby client for BISMAS databases                                #
+#                                                                             #
+# Copyright (C) 2015 Jens Wille                                               #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# bismas is free software; you can redistribute it and/or modify it           #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# bismas is distributed in the hope that it will be useful, but               #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public      #
+# License for more details.                                                   #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with bismas. If not, see <http://www.gnu.org/licenses/>.              #
+#                                                                             #
+###############################################################################
+#++
+
+require 'cyclops'
+require 'bismas'
+
+module Bismas
+
+  class CLI < Cyclops
+
+    TYPES = %w[dat dbm]
+
+    class << self
+
+      def defaults
+        super.merge(
+          config:     'config.yaml',
+          input:      '-',
+          output:     '-',
+          type:       TYPES.first,
+          key_format: '%s'
+        )
+      end
+
+    end
+
+    def run(arguments)
+      quit unless arguments.empty?
+
+      klass = case type = options[:type]
+        when 'dat'
+          Writer
+        when 'dbm'
+          require 'midos'
+          options[:output_encoding] ||= Midos::DEFAULT_ENCODING
+          Midos::Writer
+        else
+          quit "Unsupported type: #{type}. Must be one of: #{TYPES.join(', ')}."
+      end
+
+      Bismas.filter(klass, options, &method(:quit))
+    rescue LoadError => err
+      abort "Please install the `#{File.dirname(err.path)}' gem. (#{err})"
+    end
+
+    private
+
+    def opts(opts)
+      opts.option(:input__FILE, 'Path to input file [Default: STDIN]')
+
+      opts.option(:output__FILE, 'Path to output file [Default: STDOUT]')
+
+      opts.separator
+      opts.separator 'Writer options:'
+
+      opts.option(:type__TYPE, "Output file type (#{TYPES.join(', ')}) [Default: #{TYPES.first}]")
+
+      opts.separator
+
+      opts.option(:output_encoding__ENCODING, :n, 'Output encoding [Default: depends on TYPE]')
+
+      opts.separator
+
+      opts.option(:output_key__KEY, :k, 'ID key of output file')
+      opts.option(:key_format__KEY_FORMAT, :f, 'Key format [Default: %s]')
+
+      opts.separator
+
+      opts.option(:mapping__FILE_OR_YAML, 'Path to mapping file or YAML string')
+
+      opts.separator
+
+      opts.switch(:sort, 'Sort each record')
+
+      opts.separator
+
+      opts.option(:execute__CODE, 'Code to execute for each _record_ before mapping') { |e|
+        options[:execute] << e
+      }
+
+      opts.option(:execute_mapped__CODE, :E, 'Code to execute for each _record_ after mapping') { |e|
+        options[:execute_mapped] << e
+      }
+
+      opts.separator
+
+      opts.option(:padding_length__LENGTH, :P, Integer, "Length of padding for TYPE=dat [Default: #{DEFAULT_PADDING_LENGTH}]")
+
+      opts.separator
+      opts.separator 'Reader options:'
+
+      opts.option(:input_encoding__ENCODING, :N, "Input encoding [Default: #{DEFAULT_ENCODING}]")
+
+      opts.separator
+
+      opts.option(:input_key__KEY, :K, 'ID key of input file')
+
+      opts.separator
+
+      opts.switch(:strict, :S, 'Turn parse warnings into errors')
+
+      opts.switch(:silent, :T, 'Silence parse warnings')
+
+      opts.separator
+
+      opts.switch(:legacy, :L, 'Use the legacy parser')
+
+      opts.separator
+      opts.separator 'Common options:'
+
+      opts.option(:category_length__LENGTH, :C, Integer, "Length of category for TYPE=dat [Default: #{DEFAULT_CATEGORY_LENGTH}]")
+    end
+
+  end
+
+end

data/lib/bismas/mapping.rb

@@ -0,0 +1,96 @@
+#--
+###############################################################################
+#                                                                             #
+# bismas -- A Ruby client for BISMAS databases                                #
+#                                                                             #
+# Copyright (C) 2015 Jens Wille                                               #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# bismas is free software; you can redistribute it and/or modify it           #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# bismas is distributed in the hope that it will be useful, but               #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public      #
+# License for more details.                                                   #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with bismas. If not, see <http://www.gnu.org/licenses/>.              #
+#                                                                             #
+###############################################################################
+#++
+
+module Bismas
+
+  class Mapping
+
+    DEFAULT_MAPPING = true
+
+    LITERALS = { '~' => nil, 'false' => false, 'true' => true }
+
+    NULL = Object.new.tap { |null| def null.apply(hash); hash; end }
+
+    def self.[](mapping)
+      mapping ? new(mapping) : NULL
+    end
+
+    def initialize(mapping)
+      @mapping = default_hash.update(default: Array(DEFAULT_MAPPING))
+
+      mapping.each { |key, value|
+        value = Array(value.is_a?(String) ? range(value) : value)
+
+        !key.is_a?(String) ? @mapping[key] = value :
+          range(key) { |m| @mapping[m].concat(value) }
+      }
+
+      @mapping.each_value(&:uniq!)
+    end
+
+    def apply(hash, new_hash = default_hash)
+      hash.each { |key, value| map(key) { |new_key|
+        new_hash[new_key].concat(value)
+      } }
+
+      new_hash
+    end
+
+    def [](key)
+      map(key).to_a
+    end
+
+    private
+
+    def default_hash
+      Hash.new { |h, k| h[k] = [] }
+    end
+
+    def range(list, &block)
+      return enum_for(__method__, list) unless block
+
+      list.split(/\s*,\s*/).each { |part|
+        LITERALS.key?(part) ? block[LITERALS[part]] : begin
+          from, to = part.split('-')
+          from.upto(to || from, &block)
+        end
+      }
+    end
+
+    def fetch(key)
+      @mapping.key?(key) ? @mapping[key] : @mapping[:default]
+    end
+
+    def map(key)
+      return enum_for(__method__, key) unless block_given?
+
+      fetch(key).each { |new_key|
+        yield new_key == true ? key : new_key if new_key }
+    end
+
+  end
+
+end

data/lib/bismas/parser.rb

@@ -0,0 +1,167 @@
+#--
+###############################################################################
+#                                                                             #
+# bismas -- A Ruby client for BISMAS databases                                #
+#                                                                             #
+# Copyright (C) 2015 Jens Wille                                               #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# bismas is free software; you can redistribute it and/or modify it           #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# bismas is distributed in the hope that it will be useful, but               #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public      #
+# License for more details.                                                   #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with bismas. If not, see <http://www.gnu.org/licenses/>.              #
+#                                                                             #
+###############################################################################
+#++
+
+require 'strscan'
+
+module Bismas
+
+  # 0. Each _record_ is terminated by +0x0D+ +0x0A+ (<tt>CHARS[:newline]</tt>).
+  # 0. Each _record_ starts with +0x01+ (<tt>CHARS[:rs]</tt>) or, if it's a
+  #    deleted _record_, with +0xFF+ (<tt>CHARS[:deleted]</tt>).
+  # 0. Each _field_ is terminated by +0x00+ (<tt>CHARS[:fs]</tt>).
+  # 0. Each _field_ starts with the _category_ "number", a run of
+  #    +category_length+ characters except +0x00+, +0x01+, +0xDB+ or +0xFF+;
+  #    trailing space is stripped.
+  # 0. The remaining characters of a _field_ form the _category_ content;
+  #    trailing padding +0xDB+ (<tt>CHARS[:padding]</tt>) is stripped.
+  #
+  # To quote the BISMAS handbook: <i>"Konkret wird bei BISMAS jeder Datensatz
+  # durch ASCII(1) eingeleitet. Es folgt die erste Kategorienummer mit dem
+  # Kategorieinhalt. Abgeschlossen wird jede Kategorie mit ASCII(0), danach
+  # folgt die nächste Kategorienummer und -inhalt usw. Der gesamte Datensatz
+  # wird mit ASCII (13)(10) abgeschlossen."</i>
+
+  class Parser
+
+    # Legacy version of Parser that more closely mimics the behaviour of the
+    # original BISMAS software. Deviations from Parser:
+    #
+    # 0. Records are not required to start with +0x01+, any character will do.
+    # 0. Category numbers are extended to any character except +0x0D+ +0x0A+.
+    # [NOT IMPLEMENTED YET]
+
+    class Legacy < self
+
+      def initialize(options = {})
+        raise NotImplementedError, 'not implemented yet'
+
+        @category_char = '.'
+        super
+        @regex[:category] = /#{@regex[:category]}(?<!#{@chars[:newline]})/
+      end
+
+      private
+
+      def match_record
+        @input.skip(/./)
+      end
+
+    end
+
+    def self.parse(io, options = {}, &block)
+      klass = options[:legacy] ? Legacy : self
+      klass.new(options).parse(io, &block)
+    end
+
+    def initialize(options = {})
+      @regex = Bismas.regex(options)
+
+      @strict, @silent = options.values_at(:strict, :silent)
+    end
+
+    def parse(io, &block)
+      @input = StringScanner.new('')
+
+      io.each { |input|
+        @input << input
+
+        parse_record(&block) while @input.check_until(@regex[:newline])
+        @input.string = @input.string.byteslice(@input.pos..-1)
+      }
+
+      error('Unexpected data') unless @input.eos?
+
+      self
+    end
+
+    def parse_record
+      if match(:deleted)
+        match(:skip_line)
+        return
+      elsif !match_record
+        error('Malformed record', :line)
+        return
+      end
+
+      r = Hash.new { |h, k| h[k] = [] }
+      parse_field  { |k, v| r[k] << v } until match(:newline)
+
+      block_given? ? yield(r) : r
+    end
+
+    def parse_field
+      k = match(:category, 0) and k.rstrip!
+
+      v = match(:field, 1) or error(k ?
+        "Unclosed field `#{k}'" : 'Unexpected data', :rest)
+
+      k ? block_given? ? yield(k, v) : [k, v] :
+        v.empty? ? nil : error('Malformed field', :field)
+    end
+
+    private
+
+    def match(key, index = nil)
+      res = @input.skip(@regex.fetch(key))
+      res && index ? @input[index] : res
+    end
+
+    def match_record
+      match(:rs)
+    end
+
+    def error(message, skip = nil)
+      err = parse_error(message)
+      raise err unless skip && !@strict
+
+      warn err.to_s unless @silent
+      match(:"skip_#{skip}")
+      nil
+    end
+
+    def parse_error(message)
+      ParseError.new(@input, message)
+    end
+
+    class ParseError < StandardError
+
+      def initialize(input, message)
+        @input, @message = input, message
+      end
+
+      def to_s
+        '%s at %d:%d: %s' % [@message]
+          .insert(*@input.eos? ?
+            [0, 'Unexpected end of input'] :
+            [-1, @input.peek(16).inspect])
+          .insert(1, $., @input.pos)
+      end
+
+    end
+
+  end
+
+end

data/lib/bismas/reader.rb

@@ -0,0 +1,90 @@
+#--
+###############################################################################
+#                                                                             #
+# bismas -- A Ruby client for BISMAS databases                                #
+#                                                                             #
+# Copyright (C) 2015 Jens Wille                                               #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# bismas is free software; you can redistribute it and/or modify it           #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# bismas is distributed in the hope that it will be useful, but               #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public      #
+# License for more details.                                                   #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with bismas. If not, see <http://www.gnu.org/licenses/>.              #
+#                                                                             #
+###############################################################################
+#++
+
+require_relative 'parser'
+
+module Bismas
+
+  class Reader < Base
+
+    DEFAULT_IO = $stdin
+
+    class << self
+
+      def parse(*args, &block)
+        reader = new(args.extract_options!).parse(*args, &block)
+        block ? reader : reader.records
+      end
+
+      def parse_file(*args, &block)
+        file_method(:parse, 'rb', *args, &block)
+      end
+
+    end
+
+    attr_reader :records
+
+    def reset
+      super
+      @records = {}
+    end
+
+    def parse(io = io(), &block)
+      unless block
+        records, block = @records, amend_block { |id, record|
+          records[id] = record
+        }
+      end
+
+      Parser.parse(io, @options) { |record|
+        block[key ? record[key].join : auto_id.call, record] }
+
+      self
+    end
+
+    private
+
+    def amend_block(&block)
+      return block unless $VERBOSE && k = @key
+
+      r, i = block.binding.eval('_ = records, io')
+
+      l = i.respond_to?(:lineno)
+      s = i.respond_to?(:path) ? i.path :
+        Object.instance_method(:inspect).bind(i).call
+
+      lambda { |id, *args|
+        if (r ||= block.binding.eval('records')).key?(id)
+          warn "Duplicate record in #{s}#{":#{i.lineno}" if l}: »#{k}:#{id}«"
+        end
+
+        block[id, *args]
+      }
+    end
+
+  end
+
+end