fbo 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/fbo/chunked_file.rb +6 -19
- data/lib/fbo/file.rb +7 -2
- data/lib/fbo/interpreter.rb +31 -60
- data/lib/fbo/segmented_file.rb +2 -2
- data/lib/fbo/version.rb +1 -1
- data/spec/fbo/chunked_file_spec.rb +2 -2
- data/spec/fbo/interpreter_spec.rb +11 -91
- data/spec/fbo/remote_file_spec.rb +1 -1
- metadata +18 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cb42052147996e0d44fdbaee04e245b538affecb
|
4
|
+
data.tar.gz: 4fb9642b21ee11a0840110cd21ed1488ba779662
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5cdded369892b66f85d27ee09eacff5dcbca57f96508eed2d119b01d93e7c66b1f4ed86c8185dadbf4797ab211d1b59c8dad59040343b88db4ec7fd515a7b71f
|
7
|
+
data.tar.gz: 0c496f7dc170fcb6bd98eef097fcc74a422049c6ee10ab3e3b0950881bff49f31dd900b77a4f437d025795bcd2eca6b0067d427e218e95ab227d9fa77af9a050
|
data/lib/fbo/chunked_file.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
require 'forwardable'
|
2
2
|
|
3
3
|
module FBO
|
4
|
-
class ChunkedFile
|
4
|
+
class ChunkedFile < File
|
5
5
|
extend Forwardable
|
6
6
|
|
7
7
|
attr_reader :file, :chunk_size
|
8
|
-
def_delegators :@file, :
|
8
|
+
def_delegators :@file, :eof?, :eof
|
9
9
|
|
10
10
|
|
11
11
|
DEFAULT_CHUNK_SIZE = 250 * 1024 # 250KB
|
@@ -19,7 +19,7 @@ module FBO
|
|
19
19
|
def contents
|
20
20
|
if @contents.nil?
|
21
21
|
@contents = []
|
22
|
-
while !eof
|
22
|
+
while !eof
|
23
23
|
@contents << next_chunk
|
24
24
|
end
|
25
25
|
@contents.compact!
|
@@ -31,7 +31,7 @@ module FBO
|
|
31
31
|
private
|
32
32
|
|
33
33
|
def next_chunk
|
34
|
-
return nil if eof
|
34
|
+
return nil if eof
|
35
35
|
|
36
36
|
chunk, line = "", ""
|
37
37
|
|
@@ -40,7 +40,7 @@ module FBO
|
|
40
40
|
line = gets
|
41
41
|
break unless line
|
42
42
|
chunk += line
|
43
|
-
end while (chunk.bytesize < @chunk_size)
|
43
|
+
end while (chunk.bytesize < @chunk_size && !eof)
|
44
44
|
|
45
45
|
# Add lines up to the end of a notice.
|
46
46
|
if line && line !~ FBO::NOTICE_CLOSE_REGEXP
|
@@ -49,23 +49,10 @@ module FBO
|
|
49
49
|
break unless line
|
50
50
|
chunk += line
|
51
51
|
break if line =~ FBO::NOTICE_CLOSE_REGEXP
|
52
|
-
end while (
|
52
|
+
end while (!eof)
|
53
53
|
end
|
54
54
|
|
55
55
|
return chunk.strip
|
56
56
|
end
|
57
|
-
|
58
|
-
def gets
|
59
|
-
line = file.gets
|
60
|
-
return unless line
|
61
|
-
cleanup_data(line)
|
62
|
-
end
|
63
|
-
|
64
|
-
def cleanup_data(data)
|
65
|
-
data.encode('UTF-16le', :invalid => :replace, :replace => '')
|
66
|
-
.encode('UTF-8')
|
67
|
-
.gsub(/\r\n/, "\n")
|
68
|
-
.gsub(/^M/, "")
|
69
|
-
end
|
70
57
|
end
|
71
58
|
end
|
data/lib/fbo/file.rb
CHANGED
@@ -5,7 +5,7 @@ module FBO
|
|
5
5
|
extend Forwardable
|
6
6
|
|
7
7
|
attr_reader :file
|
8
|
-
def_delegators :@file, :
|
8
|
+
def_delegators :@file, :eof?, :eof
|
9
9
|
|
10
10
|
class << self
|
11
11
|
def filename_for_date(date)
|
@@ -18,6 +18,11 @@ module FBO
|
|
18
18
|
@file = ::File.new(filename)
|
19
19
|
end
|
20
20
|
|
21
|
+
def gets
|
22
|
+
str = @file.gets
|
23
|
+
str.nil? ? nil : cleanup_data(str)
|
24
|
+
end
|
25
|
+
|
21
26
|
def contents
|
22
27
|
if @contents.nil?
|
23
28
|
@contents = cleanup_data(@file.read)
|
@@ -25,7 +30,7 @@ module FBO
|
|
25
30
|
@contents
|
26
31
|
end
|
27
32
|
|
28
|
-
|
33
|
+
protected
|
29
34
|
|
30
35
|
def cleanup_data(data)
|
31
36
|
data.encode('UTF-16le', :invalid => :replace, :replace => '')
|
data/lib/fbo/interpreter.rb
CHANGED
@@ -1,76 +1,47 @@
|
|
1
1
|
module FBO
|
2
2
|
class Interpreter
|
3
|
-
class << self
|
4
|
-
def notice_enumeration(name, type)
|
5
|
-
define_method(name) do |&block|
|
6
|
-
selected = notice_nodes(type)
|
7
|
-
return unless selected
|
8
|
-
selected.each do |node|
|
9
|
-
block.call node.to_hash.merge({ type: node.type })
|
10
|
-
end
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
notice_enumeration :each_notice, :all
|
16
|
-
notice_enumeration :each_presolicitation, :presol
|
17
|
-
notice_enumeration :each_combined_solicitation, :combine
|
18
|
-
notice_enumeration :each_amendment, :amdcss
|
19
|
-
notice_enumeration :each_modification, :mod
|
20
|
-
notice_enumeration :each_award, :award
|
21
|
-
notice_enumeration :each_justification_and_approval, :ja
|
22
|
-
notice_enumeration :each_intent_to_bundle, :itb
|
23
|
-
notice_enumeration :each_fair_opportunity, :fairopp
|
24
|
-
notice_enumeration :each_sources_sought, :srcsgt
|
25
|
-
notice_enumeration :each_foreign_standard, :fstd
|
26
|
-
notice_enumeration :each_special_notice, :snote
|
27
|
-
notice_enumeration :each_sale_of_surplus, :ssale
|
28
|
-
notice_enumeration :each_document_upload, :epsupload
|
29
|
-
notice_enumeration :each_document_delete, :delete
|
30
|
-
notice_enumeration :each_document_archival, :archive
|
31
|
-
notice_enumeration :each_document_unarchival, :unarchive
|
32
|
-
|
33
3
|
def initialize(file)
|
34
4
|
@file = file
|
5
|
+
@parser = Parser.new
|
35
6
|
end
|
36
7
|
|
37
|
-
|
8
|
+
# Walk through the file returning each notice text block and structure.
|
9
|
+
# If a block is given, yield passing the text block and structure as args.
|
10
|
+
#
|
11
|
+
def next_notice(&block)
|
12
|
+
return unless notice_text = next_notice_string
|
13
|
+
return if notice_text.empty?
|
38
14
|
|
39
|
-
|
40
|
-
if
|
41
|
-
|
42
|
-
end
|
43
|
-
|
44
|
-
if @file.is_a?(FBO::SegmentedFile) && type != :all
|
45
|
-
@notice_nodes[type] = parse_segment(type)
|
46
|
-
else
|
47
|
-
@notice_nodes[:all] = parse_file
|
48
|
-
if type == :all
|
49
|
-
@notice_nodes[:all]
|
50
|
-
else
|
51
|
-
@notice_nodes[type] = @notice_nodes[:all].select { |n| n.type == type }
|
52
|
-
end
|
53
|
-
end
|
15
|
+
notice_node = parse_notice(notice_text)
|
16
|
+
yield notice_text, notice_node if block_given?
|
17
|
+
return notice_text, notice_node
|
54
18
|
end
|
55
19
|
|
56
|
-
def parse_file
|
57
|
-
tree = FBO::Parser.new(@file).parse
|
58
|
-
tree.elements
|
59
|
-
end
|
60
20
|
|
61
|
-
|
62
|
-
return unless @file.is_a? FBO::SegmentedFile
|
21
|
+
private
|
63
22
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
23
|
+
# Get the next whole notice from the FBO dump file.
|
24
|
+
# Return an empty string if no more exist
|
25
|
+
#
|
26
|
+
def next_notice_string
|
27
|
+
return if @file.eof?
|
28
|
+
|
29
|
+
line, entry_string = '', ''
|
30
|
+
while !@file.eof
|
31
|
+
line = @file.gets
|
32
|
+
break unless line
|
33
|
+
next unless line =~ /\S/
|
34
|
+
entry_string += line
|
35
|
+
break if line =~ FBO::NOTICE_CLOSE_REGEXP
|
69
36
|
end
|
70
|
-
return
|
37
|
+
return entry_string.strip
|
38
|
+
end
|
71
39
|
|
72
|
-
|
73
|
-
|
40
|
+
# Parse the text to extract a structure of data for the notice.
|
41
|
+
#
|
42
|
+
def parse_notice(text)
|
43
|
+
tree = @parser.parse(text)
|
44
|
+
tree.elements.first
|
74
45
|
end
|
75
46
|
end
|
76
47
|
end
|
data/lib/fbo/segmented_file.rb
CHANGED
data/lib/fbo/version.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe FBO::ChunkedFile do
|
4
|
-
let(:filename) { File.join(File.dirname(__FILE__), '..', 'fixtures', '
|
4
|
+
let(:filename) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'FBOFeed20131013') }
|
5
5
|
let(:file) { FBO::File.new(filename) }
|
6
|
-
let(:chunk_size) {
|
6
|
+
let(:chunk_size) { 25 * 1024 } # 25KB
|
7
7
|
subject { FBO::ChunkedFile.new(file, chunk_size) }
|
8
8
|
|
9
9
|
describe '#contents' do
|
@@ -3,102 +3,22 @@ require 'spec_helper'
|
|
3
3
|
describe FBO::Interpreter do
|
4
4
|
let(:filename) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'FBOFeed20130331') }
|
5
5
|
let(:file) { FBO::File.new(filename) }
|
6
|
-
let(:actor) {
|
6
|
+
let(:actor) { MiniTest::Mock.new }
|
7
|
+
|
7
8
|
|
8
9
|
context 'with input FBO::File' do
|
9
10
|
subject { FBO::Interpreter.new(file) }
|
10
11
|
|
11
|
-
describe '#
|
12
|
-
it 'acts on
|
13
|
-
actor.
|
14
|
-
subject.
|
15
|
-
|
16
|
-
end
|
17
|
-
|
18
|
-
describe '#each_presolicitation' do
|
19
|
-
it 'acts on each PRESOL notice' do
|
20
|
-
actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
|
21
|
-
subject.each_presolicitation { |n| actor.process(n) }
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
describe '#each_sources_sought' do
|
26
|
-
it 'acts on each SRCSGT notice' do
|
27
|
-
actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
|
28
|
-
subject.each_sources_sought { |n| actor.process(n) }
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
describe '#each_special_notice' do
|
33
|
-
it 'does nothing (no SNOTE notices)' do
|
34
|
-
actor.expects(:process).never
|
35
|
-
subject.each_special_notice { |n| actor.process(n) }
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
context 'with input FBO::ChunkedFile' do
|
41
|
-
let(:chunked) { FBO::ChunkedFile.new(file) }
|
42
|
-
subject { FBO::Interpreter.new(chunked) }
|
43
|
-
|
44
|
-
describe '#each_notice' do
|
45
|
-
it 'acts on each notice' do
|
46
|
-
actor.expects(:process).with(instance_of(Hash)).times(8)
|
47
|
-
subject.each_notice { |n| actor.process(n) }
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
describe '#each_presolicitation' do
|
52
|
-
it 'acts on each PRESOL notice' do
|
53
|
-
actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
|
54
|
-
subject.each_presolicitation { |n| actor.process(n) }
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
describe '#each_sources_sought' do
|
59
|
-
it 'acts on each SRCSGT notice' do
|
60
|
-
actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
|
61
|
-
subject.each_sources_sought { |n| actor.process(n) }
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
describe '#each_special_notice' do
|
66
|
-
it 'does nothing (no SNOTE notices)' do
|
67
|
-
actor.expects(:process).never
|
68
|
-
subject.each_special_notice { |n| actor.process(n) }
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
context 'with input FBO::SegmentedFile' do
|
74
|
-
let(:segmented) { FBO::SegmentedFile.new(file) }
|
75
|
-
subject { FBO::Interpreter.new(segmented) }
|
76
|
-
|
77
|
-
describe '#each_notice' do
|
78
|
-
it 'acts on each notice' do
|
79
|
-
actor.expects(:process).with(instance_of(Hash)).times(8)
|
80
|
-
subject.each_notice { |n| actor.process(n) }
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
describe '#each_presolicitation' do
|
85
|
-
it 'acts on each PRESOL notice' do
|
86
|
-
actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
|
87
|
-
subject.each_presolicitation { |n| actor.process(n) }
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
describe '#each_sources_sought' do
|
92
|
-
it 'acts on each SRCSGT notice' do
|
93
|
-
actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
|
94
|
-
subject.each_sources_sought { |n| actor.process(n) }
|
95
|
-
end
|
96
|
-
end
|
12
|
+
describe '#next_notice' do
|
13
|
+
it 'acts on the first notice' do
|
14
|
+
actor.expect(:process, nil, [String, Treetop::Runtime::SyntaxNode ])
|
15
|
+
string, structure = subject.next_notice { |str, struct| actor.process(str, struct) }
|
16
|
+
actor.verify
|
97
17
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
18
|
+
string.must_match /^\<PRESOL\>/
|
19
|
+
string.must_match /\<SOLNBR\>SPE4A713R0795/
|
20
|
+
structure.type.must_equal :presol
|
21
|
+
structure.to_hash[:solicitation_number].must_equal 'SPE4A713R0795'
|
102
22
|
end
|
103
23
|
end
|
104
24
|
end
|
metadata
CHANGED
@@ -1,97 +1,97 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fbo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Kottom
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2014-03-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: treetop
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: bundler
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - ~>
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '1.3'
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - ~>
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '1.3'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :development
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: minitest
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - ~>
|
59
|
+
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: 4.7.5
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - ~>
|
66
|
+
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: 4.7.5
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: mocha
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: turn
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - ">="
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '0'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- -
|
94
|
+
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
97
|
description: |2
|
@@ -107,7 +107,7 @@ executables: []
|
|
107
107
|
extensions: []
|
108
108
|
extra_rdoc_files: []
|
109
109
|
files:
|
110
|
-
- .gitignore
|
110
|
+
- ".gitignore"
|
111
111
|
- Gemfile
|
112
112
|
- LICENSE.txt
|
113
113
|
- README.md
|
@@ -143,17 +143,17 @@ require_paths:
|
|
143
143
|
- lib
|
144
144
|
required_ruby_version: !ruby/object:Gem::Requirement
|
145
145
|
requirements:
|
146
|
-
- -
|
146
|
+
- - ">="
|
147
147
|
- !ruby/object:Gem::Version
|
148
148
|
version: '0'
|
149
149
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
150
150
|
requirements:
|
151
|
-
- -
|
151
|
+
- - ">="
|
152
152
|
- !ruby/object:Gem::Version
|
153
153
|
version: '0'
|
154
154
|
requirements: []
|
155
155
|
rubyforge_project:
|
156
|
-
rubygems_version: 2.
|
156
|
+
rubygems_version: 2.2.2
|
157
157
|
signing_key:
|
158
158
|
specification_version: 4
|
159
159
|
summary: Parse and process FBO bulk listings
|