bismas 0.1.0

data/lib/bismas/version.rb

@@ -0,0 +1,27 @@
+module Bismas
+
+  module Version
+
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 0
+
+    class << self
+
+      # Returns array representation.
+      def to_a
+        [MAJOR, MINOR, TINY]
+      end
+
+      # Short-cut for version string.
+      def to_s
+        to_a.join('.')
+      end
+
+    end
+
+  end
+
+  VERSION = Version.to_s
+
+end

data/lib/bismas/writer.rb

@@ -0,0 +1,105 @@
+#--
+###############################################################################
+#                                                                             #
+# bismas -- A Ruby client for BISMAS databases                                #
+#                                                                             #
+# Copyright (C) 2015 Jens Wille                                               #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# bismas is free software; you can redistribute it and/or modify it           #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# bismas is distributed in the hope that it will be useful, but               #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public      #
+# License for more details.                                                   #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with bismas. If not, see <http://www.gnu.org/licenses/>.              #
+#                                                                             #
+###############################################################################
+#++
+
+require_relative 'parser'
+
+module Bismas
+
+  class Writer < Base
+
+    DEFAULT_IO = $stdout
+
+    class << self
+
+      def write(*args, &block)
+        new(args.extract_options!, &block).write(*args)
+      end
+
+      def write_file(*args, &block)
+        file_method(:write, 'wb', *args, &block)
+      end
+
+      def open(*args, &block)
+        file_method(nil, 'wb', *args, &block)
+      end
+
+    end
+
+    def initialize(options = {})
+      @category_length = options[:category_length] || DEFAULT_CATEGORY_LENGTH
+      @padding_length  = options[:padding_length]  || DEFAULT_PADDING_LENGTH
+      @sort            = options[:sort]
+
+      @chars = Bismas.chars(options)
+
+      super
+    end
+
+    def write(records, *args)
+      !records.is_a?(Hash) ?
+        records.each { |record| write_i(nil, record, *args) } :
+        records.each { |id, record| write_i(id, record, *args) }
+
+      self
+    end
+
+    def put(record, *args)
+      record.is_a?(Hash) ?
+        write_i(nil, record, *args) :
+        write_i(*args.unshift(*record))
+
+      self
+    end
+
+    alias_method :<<, :put
+
+    def []=(id, record)
+      write_i(id, record)
+    end
+
+    private
+
+    def write_i(id, record, io = io())
+      return if record.empty?
+
+      category_format, fs = "%-#{@category_length}s", @chars[:fs]
+
+      record[key] = id || auto_id.call if key && !record.key?(key)
+
+      io << @chars[:rs]
+
+      (@sort ? Hash[record.sort] : record).each { |k, v| Array(v).each { |w|
+        io << category_format % k if k
+        io << w.to_s << fs
+      } if v }
+
+      io << @chars[:padding] * @padding_length << fs if @padding_length > 0
+      io << @chars[:newline]
+    end
+
+  end
+
+end

data/lib/bismas.rb

@@ -0,0 +1,146 @@
+#--
+###############################################################################
+#                                                                             #
+# bismas -- A Ruby client for BISMAS databases                                #
+#                                                                             #
+# Copyright (C) 2015 Jens Wille                                               #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@gmail.com>                                       #
+#                                                                             #
+# bismas is free software; you can redistribute it and/or modify it           #
+# under the terms of the GNU Affero General Public License as published by    #
+# the Free Software Foundation; either version 3 of the License, or (at your  #
+# option) any later version.                                                  #
+#                                                                             #
+# bismas is distributed in the hope that it will be useful, but               #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public      #
+# License for more details.                                                   #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with bismas. If not, see <http://www.gnu.org/licenses/>.              #
+#                                                                             #
+###############################################################################
+#++
+
+module Bismas
+
+  # Default file encoding
+  DEFAULT_ENCODING = 'CP850'
+
+  CHARS = { rs: 1, deleted: 255, padding: 219, fs: 0, newline: "\r\n" }
+
+  REGEX = Hash[CHARS.keys.map { |k| [k, "%{#{k}}"] }].update(
+    field:      '(.*?)%{padding}*%{fs}',
+    skip_rest:  '.*?(?=%{newline})',
+    skip_line:  '.*?%{newline}',
+    skip_field: '.*?%{fs}'
+  )
+
+  CATEGORY_CHAR_SKIP = %i[rs deleted padding fs].each { |k|
+    CHARS[k] = CHARS[k].chr.force_encoding(DEFAULT_ENCODING) }
+
+  # See parameter +FELD+ in BISMAS <tt>*.CAT</tt>
+  DEFAULT_CATEGORY_LENGTH = 4
+
+  # See parameter +FUELLZEICHEN+ in BISMAS <tt>*.CFG</tt>
+  DEFAULT_PADDING_LENGTH = 20
+
+  class << self
+
+    def chars(options = {})
+      encoding = amend_encoding(options).split(':').last
+
+      Hash[CHARS.map { |k, v| [k, begin
+        v.encode(encoding)
+      rescue Encoding::UndefinedConversionError
+        v.dup.force_encoding(encoding)
+      end] }]
+    end
+
+    def regex(options = {}, chars = chars(options))
+      category_length = options[:category_length] || DEFAULT_CATEGORY_LENGTH
+
+      Hash[REGEX.map { |k, v| [k, Regexp.new(v % chars)] }].update(category:
+        /[^#{chars.values_at(*CATEGORY_CHAR_SKIP).join}]{#{category_length}}/)
+    end
+
+    def filter(klass, options, &block)
+      execute = %i[execute execute_mapped].map { |key|
+        value = Array(options[key]); lambda { |bind|
+        value.each { |code| eval(code, bind) } } }
+
+      mapping = mapping(options[:mapping], &block)
+
+      key_format = options[:key_format]
+
+      writer_options = {
+        encoding:        encoding = options[:output_encoding],
+        key:             options[:output_key],
+        sort:            options[:sort],
+        padding_length:  options[:padding_length],
+        category_length: options[:category_length]
+      }
+
+      reader_options = {
+        encoding:        options[:input_encoding],
+        key:             options[:input_key],
+        strict:          options[:strict],
+        silent:          options[:silent],
+        legacy:          options[:legacy],
+        category_length: options[:category_length]
+      }
+
+      klass.open(options[:output], writer_options) { |writer|
+        Reader.parse_file(options[:input], reader_options) { |id, record|
+          execute[0][bind = binding]
+          record = mapping.apply(encode(record, encoding))
+
+          execute[1][bind]
+          writer[key_format % id] = record
+        }
+      }
+    end
+
+    def mapping(mapping, &block)
+      block ||= method(:abort)
+
+      Mapping[case mapping
+        when nil, Hash    then mapping
+        when /\A\{.*\}\z/ then SafeYAML.load(mapping)
+        when String       then File.readable?(mapping) ?
+          SafeYAML.load_file(mapping) : block["No such file: #{mapping}"]
+        else block["Invalid mapping: #{mapping.inspect}"]
+      end]
+    end
+
+    def encode(record, encoding)
+      return record unless encoding
+
+      fallback = Hash.new { |h, k| h[k] = '?' }
+
+      record.each { |key, values|
+        values.each { |value| value.encode!(encoding, fallback: fallback) }
+
+        unless fallback.empty?
+          chars = fallback.keys.map(&:inspect).join(', '); fallback.clear
+          warn "Undefined characters at #{$.}:#{key}: #{chars}"
+        end
+      }
+    end
+
+    def amend_encoding(options, default_encoding = DEFAULT_ENCODING)
+      (options[:encoding] ||= default_encoding).tap { |encoding|
+        encoding.prepend(default_encoding) if encoding.start_with?(':') }
+    end
+
+  end
+
+end
+
+require_relative 'bismas/base'
+require_relative 'bismas/reader'
+require_relative 'bismas/writer'
+require_relative 'bismas/mapping'
+require_relative 'bismas/version'

data/spec/bismas/parser_spec.rb

@@ -0,0 +1,18 @@
+describe Bismas::Parser do
+
+  subject { described_class.new(strict: true) }
+
+  def self.expect_parse_error(msg, input)
+    example { expect{subject.parse(StringIO.new(*encode(input)))}
+      .to(raise_error(described_class::ParseError, /\A#{msg}/)) }
+  end
+
+  expect_parse_error('Malformed record', "foo\r\n")
+  expect_parse_error('Unexpected data',  "\x010\r\n")
+  expect_parse_error('Unclosed field',   "\x01000\r\n")
+  expect_parse_error('Unclosed field',   "\x01foo\r\n")
+  expect_parse_error('Malformed field',  "\x01foo\x00\r\n")
+  expect_parse_error('Malformed record', "\x01\r\nfoo\r\n")
+  expect_parse_error('Malformed record', "\x01\r\nfoo\r\n000")
+
+end

data/spec/bismas/reader_spec.rb

@@ -0,0 +1,22 @@
+describe Bismas::Reader do
+
+  example do
+    expect(described_class.parse_file(data('test.dat'), strict: true)).to eq({
+      1  => { "005" => ["Bock, F."], "020" => ["Zur Geschichte des Schlagwortkatalogs in Praxis und Theorie"], "030" => ["Zentralblatt f\x81r Bibliothekswesen. 40(1923), S.494-502."], "055" => ["1923"], "060" => ["Geschichte des Schlagwortkataloges"], "059" => ["d"], "053" => ["a"], "120" => ["D"] },
+      2  => { "005" => ["Bravo, B.R. => Rodriguez Bravo, B."], "150" => ["22. 4.2007 19:43:53"] },
+      3  => { "005" => ["Simmons, P."], "020" => ["Microcomputer software for ISO 2709 record conversion"], "030" => ["Microcomputers for information management. 6(1989), S.197-205."], "055" => ["1989"], "060" => ["Bibliographische Software"], "053" => ["a"], "059" => ["e"], "100" => ["ISO 2709"], "065" => ["Datenformate"] },
+      4  => { "005" => ["Fugmann, R."], "020" => ["\xAADie Aufgabenteilung zwischen Wortschatz und Grammatik in einer Indexsprache"], "030" => ["Datenbasen, Datenbanken, Netzwerke. Bd.1."], "045" => ["M\x81nchen"], "055" => ["1979"], "060" => ["Theorie verbaler Dokumentationssprachen"], "051" => ["S.67-93"], "059" => ["d"], "053" => ["a"] },
+      5  => { "005" => ["Babu, B.Ramesh => Ramesh Babu, B."], "150" => ["10.12.2005 11:31:20"] },
+      6  => { "005" => ["Rau, P."], "020" => ["\xAADie Facettenmethode und ihre Anwendung auf die Philologie"], "055" => ["1970"], "045" => ["Hamburg"], "058" => ["Assessorarbeit"], "053" => ["x"], "059" => ["d"], "115" => ["Geisteswissenschaften"] },
+      7  => { "005" => ["Bo\xE1meyer, C."], "020" => ["RSWK-Anwendung und Schlagwortnormdatei unter Einsatz der Datenverarbeitung"], "030" => ["Zeitschrift f\x81r Bibliothekswesen und Bibliographie. 35(1988), S.113-121."], "055" => ["1988"], "060" => ["Regeln f\x81r den Schlagwortkatalog (RSWK)"], "100" => ["RSWK"], "101" => ["SWD"], "059" => ["d"], "053" => ["a"] },
+      8  => { "005" => ["Bock, K."], "020" => ["RSWK oder der S\x81ndenkatalog"], "030" => ["BuB. 40(1988), S.262-267."], "055" => ["1988"], "060" => ["Regeln f\x81r den Schlagwortkatalog (RSWK)"], "058" => ["Forts. in: BuB 40(1988) S.926-927."], "059" => ["d"], "053" => ["a"] },
+      9  => { "005" => ["Bock, K."], "020" => ["RSWK und die entt\x84uschte Liebe"], "030" => ["BuB. 40(1988), S.926-927."], "055" => ["1988"], "060" => ["Regeln f\x81r den Schlagwortkatalog (RSWK)"], "025" => ["ein Nachtrag"], "059" => ["d"], "053" => ["a"], "150" => ["                   "], "152" => [" 7. 4.2001 15:39:33"], "120" => ["D"] },
+      10 => { "020" => ["SISIS"], "030" => ["Bibliotheksdienst. 35(2001) H.9, S.1179-1182."], "055" => ["2001"], "053" => ["a"], "059" => ["d"], "150" => ["                   "], "152" => ["29. 9.2001 11:38:17"], "005" => ["Klau\xE1, H."], "025" => ["15. Anwenderforum Berlin-Brandenburg"], "100" => ["SISIS"], "120" => ["Berlin", "Brandenburg", "D"] },
+      11 => { "005" => ["Notess, G.R."], "020" => ["Tracking title search capabilities"], "030" => ["Online. 25(2001) no.3, S.72-74."], "053" => ["a"], "055" => ["2001"], "059" => ["e"], "060" => ["Volltextretrieval"], "150" => [" 5. 6.2001 11:36:02"] },
+      12 => { "005" => ["G\x94dert, W."], "020" => ["Online-Katalog und bibliothekarische Inhaltserschlie\xE1ung"], "030" => ["77. Deutscher Bibliothekartag in Augsburg 1987. Reden und Vortr\x84ge. Hrsg.: Y.A. Haase u.a."], "045" => ["Frankfurt a.M."], "055" => ["1988"], "060" => ["Grundlagen u. Einf\x81hrungen: Allgemeine Literatur"], "065" => ["Regeln f\x81r den Schlagwortkatalog (RSWK)"], "050" => ["Klostermann"], "035" => ["Zeitschrift f\x81r Bibliothekswesen und Bibliographie: Sonderh.46"], "051" => ["S.279-302"], "053" => ["a"], "059" => ["d"] },
+      13 => { "005" => ["Aluri, R.D."], "010" => ["Kemp, A."], "015" => ["Boll, J.J."], "020" => ["Subject analysis in online catalogs"], "045" => ["Englewood, CO"], "050" => ["Libraries Unlimited"], "055" => ["1991"], "060" => ["Klassifikationssysteme im Online-Retrieval"], "065" => ["Verbale Doksprachen im Online-Retrieval"], "058" => ["Rez. in: Technical services quarterly. 9(1992) no.3, S.87-88 (H.L. Hoerman); Knowledge organization 20(1993) no.3, S.165-166 (O. Oberhauser); JASIS 44(1993) S.593 (D. Vizine-Goetz)", "\r\r2. Aufl. unter: Olson, H.A., J.J. Boll: Subject access in online catalogs. 2nd ed. Englewood, CO: Libraries Unlimited 2001. xv, 333 S. ISBN 1-56308-800-2"], "051" => ["XII,303 S"], "052" => ["0-87287-670-5"], "053" => ["m"], "059" => ["e"], "130" => ["025.3'132--dc20"], "131" => ["Z699.35.S92A46 1990"], "136" => ["Catalogs, On-line--Subject access", "Subject cataloguing--Data processing", "Machine-readable bibliographic data", "Information retrieval"], "150" => ["                   "], "152" => ["26. 8.2005 15:16:28"] },
+      14 => { "020" => ["Multilingual information management"], "055" => ["1999"], "059" => ["e"], "060" => ["Multilinguale Probleme"], "150" => [" 2. 8.2001  9:04:29"], "017" => ["Hovy, N. et al."], "025" => ["current levels and future abilities. A report commissioned by the US National Science Foundation and also delivered to the European's Commission's Language Engineering Office and the US Defense Advanced Research Projects Agency"], "050" => ["US National Science Foundation"], "058" => ["Vgl. auch: http://www.cs.cmu.edu/~ref/mlim/index.shtml"], "045" => ["?"], "056" => ["Over the past 50 years, a variety of language-related capabilities has been developed in machine translation, information retrieval, speech recognition, text summarization, and so on. These applications rest upon a set of core techniques such as language modeling, information extraction, parsing, generation, and multimedia planning and integration; and they involve methods using statistics, rules, grammars, lexicons, ontologies, training techniques, and so on. It is a puzzling fact that although all of this work deals with language in some form or other, the major applications have each developed a separate research field. For example, there is no reason why speech recognition techniques involving n-grams and hidden Markov models could not have been used in machine translation 15 years earlier than they were, or why some of the lexical and semantic insights from the subarea called Computational Linguistics are still not used in information retrieval. This picture will rapidly change. The twin challenges of massive information overload via the web and ubiquitous computers present us with an unavoidable task: developing techniques to handle multilingual and multi-modal information robustly and efficiently, with as high quality performanceas possible.  The most effective way for us to address such a mammoth task, and to ensure that our various techniques and applications fit together, is to start talking  across the artificial research boundaries. Extending the current technologies will require integrating the various capabilities into multi-functional and multi-lingual natural language systems. However, at this time there is no clear vision of how these technologies could or should be assembled into a coherent framework. What would be involved in connecting a speech recognition system to an information retrieval engine, and then using machine translation and summarization software to process the retrieved text? How can traditional parsing and generation be enhanced with statistical techniques? What would be the effect of carefully crafted lexicons on traditional information retrieval? At which points should machine translation be interleaved within information retrieval systems to"] }
+    }.each_value { |h| h.each_value { |a| encode(*a) } })
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+$:.unshift('lib') unless $:.first == 'lib'
+
+require 'bismas'
+
+RSpec.configure { |config|
+  config.include(Module.new {
+    def encode(*args)
+      args.each { |s| s.force_encoding(Bismas::DEFAULT_ENCODING) }
+    end
+
+    def data(file)
+      File.join(File.dirname(__FILE__), 'data', file)
+    end
+  })
+}

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification
+name: bismas
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jens Wille
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cyclops
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: nuggets
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: hen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Access BISMAS databases from Ruby.
+email: jens.wille@gmail.com
+executables:
+- bismas
+extensions: []
+extra_rdoc_files:
+- README
+- COPYING
+- ChangeLog
+files:
+- COPYING
+- ChangeLog
+- README
+- Rakefile
+- bin/bismas
+- lib/bismas.rb
+- lib/bismas/base.rb
+- lib/bismas/cli.rb
+- lib/bismas/mapping.rb
+- lib/bismas/parser.rb
+- lib/bismas/reader.rb
+- lib/bismas/version.rb
+- lib/bismas/writer.rb
+- spec/bismas/parser_spec.rb
+- spec/bismas/reader_spec.rb
+- spec/data/test.dat
+- spec/spec_helper.rb
+homepage: http://github.com/blackwinter/bismas
+licenses:
+- AGPL-3.0
+metadata: {}
+post_install_message: |2+
+
+  bismas-0.1.0 [2015-11-20]:
+
+  * First release.
+
+rdoc_options:
+- "--title"
+- bismas Application documentation (v0.1.0)
+- "--charset"
+- UTF-8
+- "--line-numbers"
+- "--all"
+- "--main"
+- README
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.0
+signing_key: 
+specification_version: 4
+summary: A Ruby client for BISMAS databases.
+test_files: []