bismas 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/ChangeLog ADDED
@@ -0,0 +1,11 @@
1
+ # markup: rd
2
+
3
+ = Revision history for bismas
4
+
5
+ == 0.1.0 [2015-11-20]
6
+
7
+ * First release.
8
+
9
+ == 0.0.0 [2015-09-17]
10
+
11
+ * Birthday :-)
data/README ADDED
@@ -0,0 +1,41 @@
1
+ = bismas - A Ruby client for BISMAS databases
2
+
3
+ == VERSION
4
+
5
+ This documentation refers to bismas version 0.1.0.
6
+
7
+
8
+ == DESCRIPTION
9
+
10
+ Access BISMAS databases from Ruby.
11
+
12
+
13
+ == LINKS
14
+
15
+ Documentation:: https://blackwinter.github.com/bismas
16
+ Source code:: https://github.com/blackwinter/bismas
17
+ RubyGem:: https://rubygems.org/gems/bismas
18
+ Travis CI:: https://travis-ci.org/blackwinter/bismas
19
+
20
+
21
+ == AUTHORS
22
+
23
+ * Jens Wille <mailto:jens.wille@gmail.com>
24
+
25
+
26
+ == LICENSE AND COPYRIGHT
27
+
28
+ Copyright (C) 2015 Jens Wille
29
+
30
+ bismas is free software: you can redistribute it and/or modify it
31
+ under the terms of the GNU Affero General Public License as published by
32
+ the Free Software Foundation, either version 3 of the License, or (at your
33
+ option) any later version.
34
+
35
+ bismas is distributed in the hope that it will be useful, but
36
+ WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
37
+ or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
38
+ License for more details.
39
+
40
+ You should have received a copy of the GNU Affero General Public License
41
+ along with bismas. If not, see <http://www.gnu.org/licenses/>.
data/Rakefile ADDED
@@ -0,0 +1,23 @@
1
+ require_relative 'lib/bismas/version'
2
+
3
+ begin
4
+ require 'hen'
5
+
6
+ Hen.lay! {{
7
+ gem: {
8
+ name: %q{bismas},
9
+ version: Bismas::VERSION,
10
+ summary: %q{A Ruby client for BISMAS databases.},
11
+ description: %q{Access BISMAS databases from Ruby.},
12
+ author: %q{Jens Wille},
13
+ email: %q{jens.wille@gmail.com},
14
+ license: %q{AGPL-3.0},
15
+ homepage: :blackwinter,
16
+ dependencies: { cyclops: '~> 0.2', nuggets: '~> 1.4' },
17
+
18
+ required_ruby_version: '>= 2.0'
19
+ }
20
+ }}
21
+ rescue LoadError => err
22
+ warn "Please install the `hen' gem. (#{err})"
23
+ end
data/bin/bismas ADDED
@@ -0,0 +1,30 @@
1
+ #! /usr/bin/env ruby
2
+
3
+ #--
4
+ ###############################################################################
5
+ # #
6
+ # bismas -- A Ruby client for BISMAS databases #
7
+ # #
8
+ # Copyright (C) 2015 Jens Wille #
9
+ # #
10
+ # Authors: #
11
+ # Jens Wille <jens.wille@gmail.com> #
12
+ # #
13
+ # bismas is free software; you can redistribute it and/or modify it #
14
+ # under the terms of the GNU Affero General Public License as published by #
15
+ # the Free Software Foundation; either version 3 of the License, or (at your #
16
+ # option) any later version. #
17
+ # #
18
+ # bismas is distributed in the hope that it will be useful, but #
19
+ # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
20
+ # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public #
21
+ # License for more details. #
22
+ # #
23
+ # You should have received a copy of the GNU Affero General Public License #
24
+ # along with bismas. If not, see <http://www.gnu.org/licenses/>. #
25
+ # #
26
+ ###############################################################################
27
+ #++
28
+
29
+ require 'bismas/cli'
30
+ Bismas::CLI.execute
@@ -0,0 +1,73 @@
1
+ #--
2
+ ###############################################################################
3
+ # #
4
+ # bismas -- A Ruby client for BISMAS databases #
5
+ # #
6
+ # Copyright (C) 2015 Jens Wille #
7
+ # #
8
+ # Authors: #
9
+ # Jens Wille <jens.wille@gmail.com> #
10
+ # #
11
+ # bismas is free software; you can redistribute it and/or modify it #
12
+ # under the terms of the GNU Affero General Public License as published by #
13
+ # the Free Software Foundation; either version 3 of the License, or (at your #
14
+ # option) any later version. #
15
+ # #
16
+ # bismas is distributed in the hope that it will be useful, but #
17
+ # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
18
+ # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public #
19
+ # License for more details. #
20
+ # #
21
+ # You should have received a copy of the GNU Affero General Public License #
22
+ # along with bismas. If not, see <http://www.gnu.org/licenses/>. #
23
+ # #
24
+ ###############################################################################
25
+ #++
26
+
27
+ require 'nuggets/file/open_file'
28
+ require 'nuggets/array/extract_options'
29
+
30
+ module Bismas
31
+
32
+ class Base
33
+
34
+ class << self
35
+
36
+ private
37
+
38
+ def file_method(method, mode, file, options = {}, *args, &block)
39
+ Bismas.amend_encoding(options)
40
+
41
+ File.open_file(file, options, mode) { |io|
42
+ args.unshift(options.merge(io: io))
43
+ method ? send(method, *args, &block) : block[new(*args)]
44
+ }
45
+ end
46
+
47
+ end
48
+
49
+ def initialize(options = {}, &block)
50
+ self.key = options[:key]
51
+ self.io = options.fetch(:io, self.class::DEFAULT_IO)
52
+
53
+ @auto_id_block = options.fetch(:auto_id, block)
54
+ @options = options
55
+
56
+ reset
57
+ end
58
+
59
+ attr_accessor :key, :io, :auto_id
60
+
61
+ def reset
62
+ @auto_id = @auto_id_block ? @auto_id_block.call : default_auto_id
63
+ end
64
+
65
+ private
66
+
67
+ def default_auto_id(n = 0)
68
+ lambda { n += 1 }
69
+ end
70
+
71
+ end
72
+
73
+ end
data/lib/bismas/cli.rb ADDED
@@ -0,0 +1,139 @@
1
+ #--
2
+ ###############################################################################
3
+ # #
4
+ # bismas -- A Ruby client for BISMAS databases #
5
+ # #
6
+ # Copyright (C) 2015 Jens Wille #
7
+ # #
8
+ # Authors: #
9
+ # Jens Wille <jens.wille@gmail.com> #
10
+ # #
11
+ # bismas is free software; you can redistribute it and/or modify it #
12
+ # under the terms of the GNU Affero General Public License as published by #
13
+ # the Free Software Foundation; either version 3 of the License, or (at your #
14
+ # option) any later version. #
15
+ # #
16
+ # bismas is distributed in the hope that it will be useful, but #
17
+ # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
18
+ # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public #
19
+ # License for more details. #
20
+ # #
21
+ # You should have received a copy of the GNU Affero General Public License #
22
+ # along with bismas. If not, see <http://www.gnu.org/licenses/>. #
23
+ # #
24
+ ###############################################################################
25
+ #++
26
+
27
+ require 'cyclops'
28
+ require 'bismas'
29
+
30
+ module Bismas
31
+
32
+ class CLI < Cyclops
33
+
34
+ TYPES = %w[dat dbm]
35
+
36
+ class << self
37
+
38
+ def defaults
39
+ super.merge(
40
+ config: 'config.yaml',
41
+ input: '-',
42
+ output: '-',
43
+ type: TYPES.first,
44
+ key_format: '%s'
45
+ )
46
+ end
47
+
48
+ end
49
+
50
+ def run(arguments)
51
+ quit unless arguments.empty?
52
+
53
+ klass = case type = options[:type]
54
+ when 'dat'
55
+ Writer
56
+ when 'dbm'
57
+ require 'midos'
58
+ options[:output_encoding] ||= Midos::DEFAULT_ENCODING
59
+ Midos::Writer
60
+ else
61
+ quit "Unsupported type: #{type}. Must be one of: #{TYPES.join(', ')}."
62
+ end
63
+
64
+ Bismas.filter(klass, options, &method(:quit))
65
+ rescue LoadError => err
66
+ abort "Please install the `#{File.dirname(err.path)}' gem. (#{err})"
67
+ end
68
+
69
+ private
70
+
71
+ def opts(opts)
72
+ opts.option(:input__FILE, 'Path to input file [Default: STDIN]')
73
+
74
+ opts.option(:output__FILE, 'Path to output file [Default: STDOUT]')
75
+
76
+ opts.separator
77
+ opts.separator 'Writer options:'
78
+
79
+ opts.option(:type__TYPE, "Output file type (#{TYPES.join(', ')}) [Default: #{TYPES.first}]")
80
+
81
+ opts.separator
82
+
83
+ opts.option(:output_encoding__ENCODING, :n, 'Output encoding [Default: depends on TYPE]')
84
+
85
+ opts.separator
86
+
87
+ opts.option(:output_key__KEY, :k, 'ID key of output file')
88
+ opts.option(:key_format__KEY_FORMAT, :f, 'Key format [Default: %s]')
89
+
90
+ opts.separator
91
+
92
+ opts.option(:mapping__FILE_OR_YAML, 'Path to mapping file or YAML string')
93
+
94
+ opts.separator
95
+
96
+ opts.switch(:sort, 'Sort each record')
97
+
98
+ opts.separator
99
+
100
+ opts.option(:execute__CODE, 'Code to execute for each _record_ before mapping') { |e|
101
+ options[:execute] << e
102
+ }
103
+
104
+ opts.option(:execute_mapped__CODE, :E, 'Code to execute for each _record_ after mapping') { |e|
105
+ options[:execute_mapped] << e
106
+ }
107
+
108
+ opts.separator
109
+
110
+ opts.option(:padding_length__LENGTH, :P, Integer, "Length of padding for TYPE=dat [Default: #{DEFAULT_PADDING_LENGTH}]")
111
+
112
+ opts.separator
113
+ opts.separator 'Reader options:'
114
+
115
+ opts.option(:input_encoding__ENCODING, :N, "Input encoding [Default: #{DEFAULT_ENCODING}]")
116
+
117
+ opts.separator
118
+
119
+ opts.option(:input_key__KEY, :K, 'ID key of input file')
120
+
121
+ opts.separator
122
+
123
+ opts.switch(:strict, :S, 'Turn parse warnings into errors')
124
+
125
+ opts.switch(:silent, :T, 'Silence parse warnings')
126
+
127
+ opts.separator
128
+
129
+ opts.switch(:legacy, :L, 'Use the legacy parser')
130
+
131
+ opts.separator
132
+ opts.separator 'Common options:'
133
+
134
+ opts.option(:category_length__LENGTH, :C, Integer, "Length of category for TYPE=dat [Default: #{DEFAULT_CATEGORY_LENGTH}]")
135
+ end
136
+
137
+ end
138
+
139
+ end
@@ -0,0 +1,96 @@
1
+ #--
2
+ ###############################################################################
3
+ # #
4
+ # bismas -- A Ruby client for BISMAS databases #
5
+ # #
6
+ # Copyright (C) 2015 Jens Wille #
7
+ # #
8
+ # Authors: #
9
+ # Jens Wille <jens.wille@gmail.com> #
10
+ # #
11
+ # bismas is free software; you can redistribute it and/or modify it #
12
+ # under the terms of the GNU Affero General Public License as published by #
13
+ # the Free Software Foundation; either version 3 of the License, or (at your #
14
+ # option) any later version. #
15
+ # #
16
+ # bismas is distributed in the hope that it will be useful, but #
17
+ # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
18
+ # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public #
19
+ # License for more details. #
20
+ # #
21
+ # You should have received a copy of the GNU Affero General Public License #
22
+ # along with bismas. If not, see <http://www.gnu.org/licenses/>. #
23
+ # #
24
+ ###############################################################################
25
+ #++
26
+
27
+ module Bismas
28
+
29
+ class Mapping
30
+
31
+ DEFAULT_MAPPING = true
32
+
33
+ LITERALS = { '~' => nil, 'false' => false, 'true' => true }
34
+
35
+ NULL = Object.new.tap { |null| def null.apply(hash); hash; end }
36
+
37
+ def self.[](mapping)
38
+ mapping ? new(mapping) : NULL
39
+ end
40
+
41
+ def initialize(mapping)
42
+ @mapping = default_hash.update(default: Array(DEFAULT_MAPPING))
43
+
44
+ mapping.each { |key, value|
45
+ value = Array(value.is_a?(String) ? range(value) : value)
46
+
47
+ !key.is_a?(String) ? @mapping[key] = value :
48
+ range(key) { |m| @mapping[m].concat(value) }
49
+ }
50
+
51
+ @mapping.each_value(&:uniq!)
52
+ end
53
+
54
+ def apply(hash, new_hash = default_hash)
55
+ hash.each { |key, value| map(key) { |new_key|
56
+ new_hash[new_key].concat(value)
57
+ } }
58
+
59
+ new_hash
60
+ end
61
+
62
+ def [](key)
63
+ map(key).to_a
64
+ end
65
+
66
+ private
67
+
68
+ def default_hash
69
+ Hash.new { |h, k| h[k] = [] }
70
+ end
71
+
72
+ def range(list, &block)
73
+ return enum_for(__method__, list) unless block
74
+
75
+ list.split(/\s*,\s*/).each { |part|
76
+ LITERALS.key?(part) ? block[LITERALS[part]] : begin
77
+ from, to = part.split('-')
78
+ from.upto(to || from, &block)
79
+ end
80
+ }
81
+ end
82
+
83
+ def fetch(key)
84
+ @mapping.key?(key) ? @mapping[key] : @mapping[:default]
85
+ end
86
+
87
+ def map(key)
88
+ return enum_for(__method__, key) unless block_given?
89
+
90
+ fetch(key).each { |new_key|
91
+ yield new_key == true ? key : new_key if new_key }
92
+ end
93
+
94
+ end
95
+
96
+ end
@@ -0,0 +1,167 @@
1
+ #--
2
+ ###############################################################################
3
+ # #
4
+ # bismas -- A Ruby client for BISMAS databases #
5
+ # #
6
+ # Copyright (C) 2015 Jens Wille #
7
+ # #
8
+ # Authors: #
9
+ # Jens Wille <jens.wille@gmail.com> #
10
+ # #
11
+ # bismas is free software; you can redistribute it and/or modify it #
12
+ # under the terms of the GNU Affero General Public License as published by #
13
+ # the Free Software Foundation; either version 3 of the License, or (at your #
14
+ # option) any later version. #
15
+ # #
16
+ # bismas is distributed in the hope that it will be useful, but #
17
+ # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
18
+ # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public #
19
+ # License for more details. #
20
+ # #
21
+ # You should have received a copy of the GNU Affero General Public License #
22
+ # along with bismas. If not, see <http://www.gnu.org/licenses/>. #
23
+ # #
24
+ ###############################################################################
25
+ #++
26
+
27
+ require 'strscan'
28
+
29
+ module Bismas
30
+
31
+ # 0. Each _record_ is terminated by +0x0D+ +0x0A+ (<tt>CHARS[:newline]</tt>).
32
+ # 0. Each _record_ starts with +0x01+ (<tt>CHARS[:rs]</tt>) or, if it's a
33
+ # deleted _record_, with +0xFF+ (<tt>CHARS[:deleted]</tt>).
34
+ # 0. Each _field_ is terminated by +0x00+ (<tt>CHARS[:fs]</tt>).
35
+ # 0. Each _field_ starts with the _category_ "number", a run of
36
+ # +category_length+ characters except +0x00+, +0x01+, +0xDB+ or +0xFF+;
37
+ # trailing space is stripped.
38
+ # 0. The remaining characters of a _field_ form the _category_ content;
39
+ # trailing padding +0xDB+ (<tt>CHARS[:padding]</tt>) is stripped.
40
+ #
41
+ # To quote the BISMAS handbook: <i>"Konkret wird bei BISMAS jeder Datensatz
42
+ # durch ASCII(1) eingeleitet. Es folgt die erste Kategorienummer mit dem
43
+ # Kategorieinhalt. Abgeschlossen wird jede Kategorie mit ASCII(0), danach
44
+ # folgt die nächste Kategorienummer und -inhalt usw. Der gesamte Datensatz
45
+ # wird mit ASCII (13)(10) abgeschlossen."</i>
46
+
47
+ class Parser
48
+
49
+ # Legacy version of Parser that more closely mimics the behaviour of the
50
+ # original BISMAS software. Deviations from Parser:
51
+ #
52
+ # 0. Records are not required to start with +0x01+, any character will do.
53
+ # 0. Category numbers are extended to any character except +0x0D+ +0x0A+.
54
+ # [NOT IMPLEMENTED YET]
55
+
56
+ class Legacy < self
57
+
58
+ def initialize(options = {})
59
+ raise NotImplementedError, 'not implemented yet'
60
+
61
+ @category_char = '.'
62
+ super
63
+ @regex[:category] = /#{@regex[:category]}(?<!#{@chars[:newline]})/
64
+ end
65
+
66
+ private
67
+
68
+ def match_record
69
+ @input.skip(/./)
70
+ end
71
+
72
+ end
73
+
74
+ def self.parse(io, options = {}, &block)
75
+ klass = options[:legacy] ? Legacy : self
76
+ klass.new(options).parse(io, &block)
77
+ end
78
+
79
+ def initialize(options = {})
80
+ @regex = Bismas.regex(options)
81
+
82
+ @strict, @silent = options.values_at(:strict, :silent)
83
+ end
84
+
85
+ def parse(io, &block)
86
+ @input = StringScanner.new('')
87
+
88
+ io.each { |input|
89
+ @input << input
90
+
91
+ parse_record(&block) while @input.check_until(@regex[:newline])
92
+ @input.string = @input.string.byteslice(@input.pos..-1)
93
+ }
94
+
95
+ error('Unexpected data') unless @input.eos?
96
+
97
+ self
98
+ end
99
+
100
+ def parse_record
101
+ if match(:deleted)
102
+ match(:skip_line)
103
+ return
104
+ elsif !match_record
105
+ error('Malformed record', :line)
106
+ return
107
+ end
108
+
109
+ r = Hash.new { |h, k| h[k] = [] }
110
+ parse_field { |k, v| r[k] << v } until match(:newline)
111
+
112
+ block_given? ? yield(r) : r
113
+ end
114
+
115
+ def parse_field
116
+ k = match(:category, 0) and k.rstrip!
117
+
118
+ v = match(:field, 1) or error(k ?
119
+ "Unclosed field `#{k}'" : 'Unexpected data', :rest)
120
+
121
+ k ? block_given? ? yield(k, v) : [k, v] :
122
+ v.empty? ? nil : error('Malformed field', :field)
123
+ end
124
+
125
+ private
126
+
127
+ def match(key, index = nil)
128
+ res = @input.skip(@regex.fetch(key))
129
+ res && index ? @input[index] : res
130
+ end
131
+
132
+ def match_record
133
+ match(:rs)
134
+ end
135
+
136
+ def error(message, skip = nil)
137
+ err = parse_error(message)
138
+ raise err unless skip && !@strict
139
+
140
+ warn err.to_s unless @silent
141
+ match(:"skip_#{skip}")
142
+ nil
143
+ end
144
+
145
+ def parse_error(message)
146
+ ParseError.new(@input, message)
147
+ end
148
+
149
+ class ParseError < StandardError
150
+
151
+ def initialize(input, message)
152
+ @input, @message = input, message
153
+ end
154
+
155
+ def to_s
156
+ '%s at %d:%d: %s' % [@message]
157
+ .insert(*@input.eos? ?
158
+ [0, 'Unexpected end of input'] :
159
+ [-1, @input.peek(16).inspect])
160
+ .insert(1, $., @input.pos)
161
+ end
162
+
163
+ end
164
+
165
+ end
166
+
167
+ end
@@ -0,0 +1,90 @@
1
+ #--
2
+ ###############################################################################
3
+ # #
4
+ # bismas -- A Ruby client for BISMAS databases #
5
+ # #
6
+ # Copyright (C) 2015 Jens Wille #
7
+ # #
8
+ # Authors: #
9
+ # Jens Wille <jens.wille@gmail.com> #
10
+ # #
11
+ # bismas is free software; you can redistribute it and/or modify it #
12
+ # under the terms of the GNU Affero General Public License as published by #
13
+ # the Free Software Foundation; either version 3 of the License, or (at your #
14
+ # option) any later version. #
15
+ # #
16
+ # bismas is distributed in the hope that it will be useful, but #
17
+ # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
18
+ # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public #
19
+ # License for more details. #
20
+ # #
21
+ # You should have received a copy of the GNU Affero General Public License #
22
+ # along with bismas. If not, see <http://www.gnu.org/licenses/>. #
23
+ # #
24
+ ###############################################################################
25
+ #++
26
+
27
+ require_relative 'parser'
28
+
29
+ module Bismas
30
+
31
+ class Reader < Base
32
+
33
+ DEFAULT_IO = $stdin
34
+
35
+ class << self
36
+
37
+ def parse(*args, &block)
38
+ reader = new(args.extract_options!).parse(*args, &block)
39
+ block ? reader : reader.records
40
+ end
41
+
42
+ def parse_file(*args, &block)
43
+ file_method(:parse, 'rb', *args, &block)
44
+ end
45
+
46
+ end
47
+
48
+ attr_reader :records
49
+
50
+ def reset
51
+ super
52
+ @records = {}
53
+ end
54
+
55
+ def parse(io = io(), &block)
56
+ unless block
57
+ records, block = @records, amend_block { |id, record|
58
+ records[id] = record
59
+ }
60
+ end
61
+
62
+ Parser.parse(io, @options) { |record|
63
+ block[key ? record[key].join : auto_id.call, record] }
64
+
65
+ self
66
+ end
67
+
68
+ private
69
+
70
+ def amend_block(&block)
71
+ return block unless $VERBOSE && k = @key
72
+
73
+ r, i = block.binding.eval('_ = records, io')
74
+
75
+ l = i.respond_to?(:lineno)
76
+ s = i.respond_to?(:path) ? i.path :
77
+ Object.instance_method(:inspect).bind(i).call
78
+
79
+ lambda { |id, *args|
80
+ if (r ||= block.binding.eval('records')).key?(id)
81
+ warn "Duplicate record in #{s}#{":#{i.lineno}" if l}: »#{k}:#{id}«"
82
+ end
83
+
84
+ block[id, *args]
85
+ }
86
+ end
87
+
88
+ end
89
+
90
+ end