traject 0.0.2 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. data/Gemfile +4 -0
  2. data/README.md +85 -61
  3. data/Rakefile +5 -0
  4. data/bin/traject +31 -3
  5. data/doc/settings.md +74 -13
  6. data/lib/tasks/load_maps.rake +48 -0
  7. data/lib/traject/indexer/settings.rb +75 -0
  8. data/lib/traject/indexer.rb +255 -45
  9. data/lib/traject/json_writer.rb +4 -2
  10. data/lib/traject/macros/marc21.rb +18 -6
  11. data/lib/traject/macros/marc21_semantics.rb +405 -0
  12. data/lib/traject/macros/marc_format_classifier.rb +180 -0
  13. data/lib/traject/marc4j_reader.rb +160 -0
  14. data/lib/traject/marc_extractor.rb +33 -17
  15. data/lib/traject/marc_reader.rb +14 -11
  16. data/lib/traject/solrj_writer.rb +247 -9
  17. data/lib/traject/thread_pool.rb +154 -0
  18. data/lib/traject/translation_map.rb +46 -4
  19. data/lib/traject/util.rb +30 -0
  20. data/lib/traject/version.rb +1 -1
  21. data/lib/translation_maps/lcc_top_level.yaml +26 -0
  22. data/lib/translation_maps/marc_genre_007.yaml +9 -0
  23. data/lib/translation_maps/marc_genre_leader.yaml +22 -0
  24. data/lib/translation_maps/marc_geographic.yaml +589 -0
  25. data/lib/translation_maps/marc_instruments.yaml +102 -0
  26. data/lib/translation_maps/marc_languages.yaml +490 -0
  27. data/test/indexer/each_record_test.rb +34 -0
  28. data/test/indexer/macros_marc21_semantics_test.rb +206 -0
  29. data/test/indexer/macros_marc21_test.rb +10 -1
  30. data/test/indexer/map_record_test.rb +78 -8
  31. data/test/indexer/read_write_test.rb +43 -10
  32. data/test/indexer/settings_test.rb +60 -4
  33. data/test/indexer/to_field_test.rb +39 -0
  34. data/test/marc4j_reader_test.rb +75 -0
  35. data/test/marc_extractor_test.rb +62 -0
  36. data/test/marc_format_classifier_test.rb +91 -0
  37. data/test/marc_reader_test.rb +12 -0
  38. data/test/solrj_writer_test.rb +146 -43
  39. data/test/test_helper.rb +50 -0
  40. data/test/test_support/245_no_ab.marc +1 -0
  41. data/test/test_support/880_with_no_6.utf8.marc +1 -0
  42. data/test/test_support/bad_subfield_code.marc +1 -0
  43. data/test/test_support/date_resort_to_260.marc +1 -0
  44. data/test/test_support/date_type_r_missing_date2.marc +1 -0
  45. data/test/test_support/date_with_u.marc +1 -0
  46. data/test/test_support/demo_config.rb +153 -0
  47. data/test/test_support/emptyish_record.marc +1 -0
  48. data/test/test_support/louis_armstrong.marc +1 -0
  49. data/test/test_support/manuscript_online_thesis.marc +1 -0
  50. data/test/test_support/microform_online_conference.marc +1 -0
  51. data/test/test_support/multi_era.marc +1 -0
  52. data/test/test_support/multi_geo.marc +1 -0
  53. data/test/test_support/musical_cage.marc +1 -0
  54. data/test/test_support/one-marc8.mrc +1 -0
  55. data/test/test_support/online_only.marc +1 -0
  56. data/test/test_support/packed_041a_lang.marc +1 -0
  57. data/test/test_support/the_business_ren.marc +1 -0
  58. data/test/translation_map_test.rb +8 -0
  59. data/test/translation_maps/properties_map.properties +5 -0
  60. data/traject.gemspec +1 -1
  61. data/vendor/marc4j/README.md +17 -0
  62. data/vendor/marc4j/lib/marc4j-2.5.1-beta.jar +0 -0
  63. metadata +81 -2
metadata CHANGED
@@ -2,14 +2,14 @@
2
2
  name: traject
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease:
5
- version: 0.0.2
5
+ version: 0.9.1
6
6
  platform: ruby
7
7
  authors:
8
8
  - Jonathan Rochkind
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-07-17 00:00:00.000000000 Z
12
+ date: 2013-08-19 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: marc
@@ -71,6 +71,22 @@ dependencies:
71
71
  none: false
72
72
  prerelease: false
73
73
  type: :runtime
74
+ - !ruby/object:Gem::Dependency
75
+ name: yell
76
+ version_requirements: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - '>='
79
+ - !ruby/object:Gem::Version
80
+ version: '0'
81
+ none: false
82
+ requirement: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - '>='
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
87
+ none: false
88
+ prerelease: false
89
+ type: :runtime
74
90
  - !ruby/object:Gem::Dependency
75
91
  name: bundler
76
92
  version_requirements: !ruby/object:Gem::Requirement
@@ -151,30 +167,66 @@ files:
151
167
  - bin/traject
152
168
  - doc/macros.md
153
169
  - doc/settings.md
170
+ - lib/tasks/load_maps.rake
154
171
  - lib/traject.rb
155
172
  - lib/traject/indexer.rb
173
+ - lib/traject/indexer/settings.rb
156
174
  - lib/traject/json_writer.rb
157
175
  - lib/traject/macros/basic.rb
158
176
  - lib/traject/macros/marc21.rb
177
+ - lib/traject/macros/marc21_semantics.rb
178
+ - lib/traject/macros/marc_format_classifier.rb
179
+ - lib/traject/marc4j_reader.rb
159
180
  - lib/traject/marc_extractor.rb
160
181
  - lib/traject/marc_reader.rb
161
182
  - lib/traject/qualified_const_get.rb
162
183
  - lib/traject/solrj_writer.rb
184
+ - lib/traject/thread_pool.rb
163
185
  - lib/traject/translation_map.rb
186
+ - lib/traject/util.rb
164
187
  - lib/traject/version.rb
188
+ - lib/translation_maps/lcc_top_level.yaml
189
+ - lib/translation_maps/marc_genre_007.yaml
190
+ - lib/translation_maps/marc_genre_leader.yaml
191
+ - lib/translation_maps/marc_geographic.yaml
192
+ - lib/translation_maps/marc_instruments.yaml
193
+ - lib/translation_maps/marc_languages.yaml
194
+ - test/indexer/each_record_test.rb
195
+ - test/indexer/macros_marc21_semantics_test.rb
165
196
  - test/indexer/macros_marc21_test.rb
166
197
  - test/indexer/macros_test.rb
167
198
  - test/indexer/map_record_test.rb
168
199
  - test/indexer/read_write_test.rb
169
200
  - test/indexer/settings_test.rb
201
+ - test/indexer/to_field_test.rb
202
+ - test/marc4j_reader_test.rb
170
203
  - test/marc_extractor_test.rb
204
+ - test/marc_format_classifier_test.rb
171
205
  - test/marc_reader_test.rb
172
206
  - test/solrj_writer_test.rb
173
207
  - test/test_helper.rb
208
+ - test/test_support/245_no_ab.marc
209
+ - test/test_support/880_with_no_6.utf8.marc
210
+ - test/test_support/bad_subfield_code.marc
211
+ - test/test_support/date_resort_to_260.marc
212
+ - test/test_support/date_type_r_missing_date2.marc
213
+ - test/test_support/date_with_u.marc
214
+ - test/test_support/demo_config.rb
215
+ - test/test_support/emptyish_record.marc
174
216
  - test/test_support/hebrew880s.marc
217
+ - test/test_support/louis_armstrong.marc
175
218
  - test/test_support/manufacturing_consent.marc
219
+ - test/test_support/manuscript_online_thesis.marc
220
+ - test/test_support/microform_online_conference.marc
221
+ - test/test_support/multi_era.marc
222
+ - test/test_support/multi_geo.marc
223
+ - test/test_support/musical_cage.marc
224
+ - test/test_support/one-marc8.mrc
225
+ - test/test_support/online_only.marc
226
+ - test/test_support/packed_041a_lang.marc
176
227
  - test/test_support/test_data.utf8.marc.xml
177
228
  - test/test_support/test_data.utf8.mrc
229
+ - test/test_support/the_business_ren.marc
178
230
  - test/translation_map_test.rb
179
231
  - test/translation_maps/bad_ruby.rb
180
232
  - test/translation_maps/bad_yaml.yaml
@@ -183,10 +235,13 @@ files:
183
235
  - test/translation_maps/default_literal.rb
184
236
  - test/translation_maps/default_passthrough.rb
185
237
  - test/translation_maps/marc_040a_translate_test.yaml
238
+ - test/translation_maps/properties_map.properties
186
239
  - test/translation_maps/ruby_map.rb
187
240
  - test/translation_maps/translate_array_test.yaml
188
241
  - test/translation_maps/yaml_map.yaml
189
242
  - traject.gemspec
243
+ - vendor/marc4j/README.md
244
+ - vendor/marc4j/lib/marc4j-2.5.1-beta.jar
190
245
  - vendor/solrj/README
191
246
  - vendor/solrj/build.xml
192
247
  - vendor/solrj/ivy.xml
@@ -238,19 +293,42 @@ signing_key:
238
293
  specification_version: 3
239
294
  summary: Index MARC to Solr; or generally process source records to hash-like structures
240
295
  test_files:
296
+ - test/indexer/each_record_test.rb
297
+ - test/indexer/macros_marc21_semantics_test.rb
241
298
  - test/indexer/macros_marc21_test.rb
242
299
  - test/indexer/macros_test.rb
243
300
  - test/indexer/map_record_test.rb
244
301
  - test/indexer/read_write_test.rb
245
302
  - test/indexer/settings_test.rb
303
+ - test/indexer/to_field_test.rb
304
+ - test/marc4j_reader_test.rb
246
305
  - test/marc_extractor_test.rb
306
+ - test/marc_format_classifier_test.rb
247
307
  - test/marc_reader_test.rb
248
308
  - test/solrj_writer_test.rb
249
309
  - test/test_helper.rb
310
+ - test/test_support/245_no_ab.marc
311
+ - test/test_support/880_with_no_6.utf8.marc
312
+ - test/test_support/bad_subfield_code.marc
313
+ - test/test_support/date_resort_to_260.marc
314
+ - test/test_support/date_type_r_missing_date2.marc
315
+ - test/test_support/date_with_u.marc
316
+ - test/test_support/demo_config.rb
317
+ - test/test_support/emptyish_record.marc
250
318
  - test/test_support/hebrew880s.marc
319
+ - test/test_support/louis_armstrong.marc
251
320
  - test/test_support/manufacturing_consent.marc
321
+ - test/test_support/manuscript_online_thesis.marc
322
+ - test/test_support/microform_online_conference.marc
323
+ - test/test_support/multi_era.marc
324
+ - test/test_support/multi_geo.marc
325
+ - test/test_support/musical_cage.marc
326
+ - test/test_support/one-marc8.mrc
327
+ - test/test_support/online_only.marc
328
+ - test/test_support/packed_041a_lang.marc
252
329
  - test/test_support/test_data.utf8.marc.xml
253
330
  - test/test_support/test_data.utf8.mrc
331
+ - test/test_support/the_business_ren.marc
254
332
  - test/translation_map_test.rb
255
333
  - test/translation_maps/bad_ruby.rb
256
334
  - test/translation_maps/bad_yaml.yaml
@@ -259,6 +337,7 @@ test_files:
259
337
  - test/translation_maps/default_literal.rb
260
338
  - test/translation_maps/default_passthrough.rb
261
339
  - test/translation_maps/marc_040a_translate_test.yaml
340
+ - test/translation_maps/properties_map.properties
262
341
  - test/translation_maps/ruby_map.rb
263
342
  - test/translation_maps/translate_array_test.yaml
264
343
  - test/translation_maps/yaml_map.yaml