fbo 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +37 -0
- data/.rspec +2 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +27 -0
- data/LICENSE.txt +22 -0
- data/README.md +39 -0
- data/Rakefile +1 -0
- data/fbo.gemspec +30 -0
- data/lib/fbo.rb +25 -0
- data/lib/fbo/file.rb +14 -0
- data/lib/fbo/notice.rb +9 -0
- data/lib/fbo/notices.rb +6 -0
- data/lib/fbo/notices/amendment.rb +13 -0
- data/lib/fbo/notices/award.rb +14 -0
- data/lib/fbo/notices/combined_solicitation.rb +14 -0
- data/lib/fbo/notices/modification.rb +14 -0
- data/lib/fbo/notices/presolicitation.rb +14 -0
- data/lib/fbo/notices/sources_sought.rb +13 -0
- data/lib/fbo/notices/unknown.rb +7 -0
- data/lib/fbo/parser.rb +79 -0
- data/lib/fbo/parser/amendment_handler.rb +58 -0
- data/lib/fbo/parser/award_handler.rb +62 -0
- data/lib/fbo/parser/combined_solicitation_handler.rb +57 -0
- data/lib/fbo/parser/modification_handler.rb +66 -0
- data/lib/fbo/parser/notice_handler.rb +27 -0
- data/lib/fbo/parser/parser_helper.rb +275 -0
- data/lib/fbo/parser/presolicitation_handler.rb +57 -0
- data/lib/fbo/parser/sources_sought_handler.rb +57 -0
- data/lib/fbo/parser/unknown_handler.rb +20 -0
- data/lib/fbo/remote_file.rb +39 -0
- data/lib/fbo/version.rb +3 -0
- data/spec/fbo/file_spec.rb +20 -0
- data/spec/fbo/parser/amendment_handler_spec.rb +46 -0
- data/spec/fbo/parser/award_handler_spec.rb +51 -0
- data/spec/fbo/parser/combined_solicitation_handler_spec.rb +47 -0
- data/spec/fbo/parser/modification_handler_spec.rb +47 -0
- data/spec/fbo/parser/presolicitation_handler_spec.rb +46 -0
- data/spec/fbo/parser/sources_sought_handler_spec.rb +46 -0
- data/spec/fbo/parser/unknown_handler_spec.rb +19 -0
- data/spec/fbo/parser_spec.rb +85 -0
- data/spec/fbo/remote_file_spec.rb +55 -0
- data/spec/fixtures/FBOFeed20130331 +5668 -0
- data/spec/fixtures/FBOFeed20130404 +45653 -0
- data/spec/fixtures/FBOFeed20130406 +10152 -0
- data/spec/fixtures/FBOFeed20130407 +6610 -0
- data/spec/fixtures/notices/amdcss +26 -0
- data/spec/fixtures/notices/award +31 -0
- data/spec/fixtures/notices/combine +29 -0
- data/spec/fixtures/notices/mod +28 -0
- data/spec/fixtures/notices/presol +25 -0
- data/spec/fixtures/notices/snote +26 -0
- data/spec/fixtures/notices/srcsgt +27 -0
- data/spec/spec_helper.rb +23 -0
- metadata +154 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 341885ee905b5466fe116ddfbef59dc1ceb9fc1b
|
4
|
+
data.tar.gz: 0f9954197c0c2cf290b860756a0e13bbf482f415
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: df079c191ef00e41a23e3ce5b881131db4ef43ad786f39a4094a6b3b9e329b2d3f96a28406e608dc89cdb9947f13e5f8b5d5bf065787f6e802165a7be423f73d
|
7
|
+
data.tar.gz: 102f3ecbb75bb265890745ff7da8ae0a491977b1c0d9be81779d315bcef655ff5bef5c5adc651beb0cfed5dfeae50e316a1e538303895811a85b5e7422b1fd51
|
data/.gitignore
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# Emacs temporary files
|
2
|
+
*~
|
3
|
+
\#*\#
|
4
|
+
|
5
|
+
# Bundler artifacts
|
6
|
+
.bundle
|
7
|
+
bundler_stubs/
|
8
|
+
vendor/
|
9
|
+
|
10
|
+
# Ruby / Rails operational files
|
11
|
+
.rvmrc
|
12
|
+
.ruby-version
|
13
|
+
|
14
|
+
# Anything temporary
|
15
|
+
*.old
|
16
|
+
*.swp
|
17
|
+
*.swo
|
18
|
+
tmp
|
19
|
+
|
20
|
+
# Reporting directory
|
21
|
+
/reports/
|
22
|
+
|
23
|
+
# Rubygems artifacts
|
24
|
+
*.gem
|
25
|
+
|
26
|
+
# Test artifacts
|
27
|
+
coverage
|
28
|
+
spec/reports
|
29
|
+
test/tmp
|
30
|
+
test/version_tmp
|
31
|
+
|
32
|
+
# Documentation
|
33
|
+
.yardoc
|
34
|
+
_yardoc
|
35
|
+
doc
|
36
|
+
rdoc
|
37
|
+
|
data/.rspec
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
FBO (0.0.1)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
diff-lcs (1.2.2)
|
10
|
+
rake (10.0.4)
|
11
|
+
rspec (2.13.0)
|
12
|
+
rspec-core (~> 2.13.0)
|
13
|
+
rspec-expectations (~> 2.13.0)
|
14
|
+
rspec-mocks (~> 2.13.0)
|
15
|
+
rspec-core (2.13.1)
|
16
|
+
rspec-expectations (2.13.0)
|
17
|
+
diff-lcs (>= 1.1.3, < 2.0)
|
18
|
+
rspec-mocks (2.13.1)
|
19
|
+
|
20
|
+
PLATFORMS
|
21
|
+
ruby
|
22
|
+
|
23
|
+
DEPENDENCIES
|
24
|
+
FBO!
|
25
|
+
bundler (~> 1.3)
|
26
|
+
rake
|
27
|
+
rspec
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Chris Kottom
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# FBO
|
2
|
+
|
3
|
+
Fetch a dated dump file from [FedBizOpps](https://www.fbo.gov) and parse its
|
4
|
+
contents into structures suitable for use by other programs.
|
5
|
+
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
gem 'fbo'
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install fbo
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
The gem provides a number of
|
24
|
+
|
25
|
+
|
26
|
+
|
27
|
+
## Contributing
|
28
|
+
|
29
|
+
Code contributions gratefully accepted.
|
30
|
+
|
31
|
+
1. Fork it
|
32
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
33
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
34
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
35
|
+
5. Create new Pull Request
|
36
|
+
|
37
|
+
Logging any issues you might notice at the
|
38
|
+
[Issues tracker](https://github.com/chriskottom/fbo/issues)
|
39
|
+
on GitHub is also a great way to help out.
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/fbo.gemspec
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'fbo/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "fbo"
|
8
|
+
spec.version = FBO::VERSION
|
9
|
+
spec.authors = ["Chris Kottom"]
|
10
|
+
spec.email = ["chris@chriskottom.com"]
|
11
|
+
spec.summary = "Parse and process FBO bulk listings"
|
12
|
+
spec.description = <<-EOF
|
13
|
+
The FBO gem manages the process of downloading and parsing file-based notice
|
14
|
+
information from the Federal Business Opportunities (https://www.fbo.gov/)
|
15
|
+
application / database. The FBO feed files include new and updated
|
16
|
+
opportunities, information about awarded contracts, and other details
|
17
|
+
concerning the offer and disposition of federal government contracts and
|
18
|
+
tenders.
|
19
|
+
EOF
|
20
|
+
spec.homepage = ""
|
21
|
+
spec.license = "MIT"
|
22
|
+
|
23
|
+
spec.files = `git ls-files`.split($/)
|
24
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
25
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
26
|
+
spec.require_paths = ["lib"]
|
27
|
+
|
28
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
29
|
+
spec.add_development_dependency "rake"
|
30
|
+
end
|
data/lib/fbo.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
require_relative "fbo/version"
|
2
|
+
|
3
|
+
require_relative "fbo/file"
|
4
|
+
require_relative "fbo/remote_file"
|
5
|
+
require_relative "fbo/notice"
|
6
|
+
require_relative "fbo/notices/amendment"
|
7
|
+
require_relative "fbo/notices/award"
|
8
|
+
require_relative "fbo/notices/combined_solicitation"
|
9
|
+
require_relative "fbo/notices/modification"
|
10
|
+
require_relative "fbo/notices/presolicitation"
|
11
|
+
require_relative "fbo/notices/sources_sought"
|
12
|
+
require_relative "fbo/notices/unknown"
|
13
|
+
require_relative "fbo/parser"
|
14
|
+
require_relative "fbo/parser/notice_handler"
|
15
|
+
require_relative "fbo/parser/parser_helper"
|
16
|
+
require_relative "fbo/parser/amendment_handler"
|
17
|
+
require_relative "fbo/parser/award_handler"
|
18
|
+
require_relative "fbo/parser/combined_solicitation_handler"
|
19
|
+
require_relative "fbo/parser/modification_handler"
|
20
|
+
require_relative "fbo/parser/presolicitation_handler"
|
21
|
+
require_relative "fbo/parser/sources_sought_handler"
|
22
|
+
require_relative "fbo/parser/unknown_handler"
|
23
|
+
|
24
|
+
module FBO
|
25
|
+
end
|
data/lib/fbo/file.rb
ADDED
data/lib/fbo/notice.rb
ADDED
data/lib/fbo/notices.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
module FBO
|
2
|
+
module Notices
|
3
|
+
class Amendment < Struct.new(:date, :year, :zip, :class_code, :naics_code,
|
4
|
+
:agency, :office, :location, :office_address,
|
5
|
+
:subject, :solicitation_number, :notice_type,
|
6
|
+
:response_date, :archive_date, :contact_info,
|
7
|
+
:description, :link_url, :link_description,
|
8
|
+
:email_address, :email_description, :setaside,
|
9
|
+
:pop_address, :pop_zip, :pop_country)
|
10
|
+
include FBO::Notice
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module FBO
|
2
|
+
module Notices
|
3
|
+
class Award < Struct.new(:date, :year, :zip, :class_code, :naics_code,
|
4
|
+
:agency, :office, :location, :office_address,
|
5
|
+
:subject, :solicitation_number, :notice_type,
|
6
|
+
:description, :contact_info, :award_number,
|
7
|
+
:award_amount, :line_number, :award_date,
|
8
|
+
:archive_date, :awardee, :awardee_duns, :link_url,
|
9
|
+
:link_description, :email_address, :email_description,
|
10
|
+
:setaside, :correction)
|
11
|
+
include FBO::Notice
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module FBO
|
2
|
+
module Notices
|
3
|
+
class CombinedSolicitation < Struct.new(:date, :year, :zip, :class_code,
|
4
|
+
:naics_code, :agency, :office, :location,
|
5
|
+
:office_address, :subject, :solicitation_number,
|
6
|
+
:response_date, :archive_date,
|
7
|
+
:contact_info, :description, :link_url,
|
8
|
+
:link_description, :email_address,
|
9
|
+
:email_description, :setaside, :pop_address,
|
10
|
+
:pop_zip, :pop_country)
|
11
|
+
include FBO::Notice
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module FBO
|
2
|
+
module Notices
|
3
|
+
class Modification < Struct.new(:date, :year, :zip, :class_code,
|
4
|
+
:naics_code, :agency, :office, :location,
|
5
|
+
:office_address, :subject, :solicitation_number,
|
6
|
+
:notice_type, :response_date, :archive_date,
|
7
|
+
:contact_info, :description, :link_url,
|
8
|
+
:link_description, :email_address,
|
9
|
+
:email_description, :setaside, :pop_address,
|
10
|
+
:pop_zip, :pop_country)
|
11
|
+
include FBO::Notice
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module FBO
|
2
|
+
module Notices
|
3
|
+
class Presolicitation < Struct.new(:date, :year, :zip, :class_code,
|
4
|
+
:naics_code, :agency, :office, :location,
|
5
|
+
:office_address, :subject, :solicitation_number,
|
6
|
+
:response_date, :archive_date,
|
7
|
+
:contact_info, :description, :link_url,
|
8
|
+
:link_description, :email_address,
|
9
|
+
:email_description, :setaside, :pop_address,
|
10
|
+
:pop_zip, :pop_country)
|
11
|
+
include FBO::Notice
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module FBO
|
2
|
+
module Notices
|
3
|
+
class SourcesSought < Struct.new(:date, :year, :zip, :class_code, :naics_code,
|
4
|
+
:agency, :office, :location, :office_address,
|
5
|
+
:subject, :solicitation_number,
|
6
|
+
:response_date, :archive_date, :contact_info,
|
7
|
+
:description, :link_url, :link_description,
|
8
|
+
:email_address, :email_description, :setaside,
|
9
|
+
:pop_address, :pop_zip, :pop_country)
|
10
|
+
include FBO::Notice
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
data/lib/fbo/parser.rb
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
require "strscan"
|
2
|
+
|
3
|
+
module FBO
|
4
|
+
class Parser
|
5
|
+
TAG_PATTERN = /<\w+>/m
|
6
|
+
PATTERN = /<([A-Z]+)>.*<\/\1>/m
|
7
|
+
|
8
|
+
def initialize(file, tag_pattern: TAG_PATTERN)
|
9
|
+
@file = file
|
10
|
+
@tag_pattern = tag_pattern
|
11
|
+
end
|
12
|
+
|
13
|
+
def notices
|
14
|
+
if @notices.nil?
|
15
|
+
parse_notices(split_file_contents)
|
16
|
+
end
|
17
|
+
@notices
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def split_file_contents
|
23
|
+
contents = @file.read
|
24
|
+
contents = cleanup_contents(contents)
|
25
|
+
scanner = StringScanner.new(contents)
|
26
|
+
text_notices = []
|
27
|
+
|
28
|
+
while !scanner.eos?
|
29
|
+
initial_tag = scanner.check_until(@tag_pattern)
|
30
|
+
if initial_tag
|
31
|
+
initial_tag.strip!
|
32
|
+
else
|
33
|
+
break
|
34
|
+
end
|
35
|
+
|
36
|
+
text_notice = scanner.scan_until(/#{ closing_tag_for(initial_tag) }\s*/)
|
37
|
+
if text_notice
|
38
|
+
text_notices << text_notice.strip
|
39
|
+
else
|
40
|
+
break
|
41
|
+
end
|
42
|
+
end
|
43
|
+
text_notices
|
44
|
+
end
|
45
|
+
|
46
|
+
def parse_notices(text_notices)
|
47
|
+
@notices = []
|
48
|
+
text_notices.each do |text|
|
49
|
+
if FBO::Parser::PresolicitationHandler.is_presolicitation?(text)
|
50
|
+
@notices << FBO::Parser::PresolicitationHandler.parse(text)
|
51
|
+
elsif FBO::Parser::CombinedSolicitationHandler.is_combined_solicitation?(text)
|
52
|
+
@notices << FBO::Parser::CombinedSolicitationHandler.parse(text)
|
53
|
+
elsif FBO::Parser::SourcesSoughtHandler.is_sources_sought?(text)
|
54
|
+
@notices << FBO::Parser::SourcesSoughtHandler.parse(text)
|
55
|
+
elsif FBO::Parser::AmendmentHandler.is_amendment?(text)
|
56
|
+
@notices << FBO::Parser::AmendmentHandler.parse(text)
|
57
|
+
elsif FBO::Parser::ModificationHandler.is_modification?(text)
|
58
|
+
@notices << FBO::Parser::ModificationHandler.parse(text)
|
59
|
+
elsif FBO::Parser::AwardHandler.is_award?(text)
|
60
|
+
@notices << FBO::Parser::AwardHandler.parse(text)
|
61
|
+
else
|
62
|
+
@notices << FBO::Parser::UnknownHandler.parse(text)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def cleanup_contents(contents)
|
68
|
+
contents
|
69
|
+
.encode('UTF-16le', :invalid => :replace, :replace => '')
|
70
|
+
.encode('UTF-8')
|
71
|
+
.gsub(/\r\n/, "\n")
|
72
|
+
.gsub(/^M/, "")
|
73
|
+
end
|
74
|
+
|
75
|
+
def closing_tag_for(tag)
|
76
|
+
tag.sub(/</, "</")
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module FBO::Parser::AmendmentHandler
|
2
|
+
TAGS = %w( AMDCSS DATE YEAR CBAC PASSWORD ZIP CLASSCOD NAICS OFFADD NTYPE) +
|
3
|
+
%w( AGENCY OFFICE LOCATION SUBJECT SOLNBR RESPDATE ARCHDATE CONTACT DESC ) +
|
4
|
+
%w( LINK URL EMAIL ADDRESS SETASIDE POPADDRESS POPZIP POPCOUNTRY \/AMDCSS )
|
5
|
+
|
6
|
+
ANY_AMDCSS_TAG = TAGS.join("|")
|
7
|
+
AMENDMENT_PATTERN = /^<AMDCSS>/
|
8
|
+
|
9
|
+
|
10
|
+
def is_amendment?(text)
|
11
|
+
text =~ AMENDMENT_PATTERN
|
12
|
+
end
|
13
|
+
|
14
|
+
def parse(text)
|
15
|
+
params = {
|
16
|
+
date: date(text),
|
17
|
+
year: year(text),
|
18
|
+
zip: zip(text),
|
19
|
+
class_code: classification_code(text),
|
20
|
+
naics_code: naics_code(text),
|
21
|
+
agency: agency(text),
|
22
|
+
office: office(text),
|
23
|
+
location: location(text),
|
24
|
+
office_address: office_address(text),
|
25
|
+
subject: subject(text),
|
26
|
+
solicitation_number: solicitation_number(text),
|
27
|
+
notice_type: notice_type(text),
|
28
|
+
response_date: response_date(text),
|
29
|
+
archive_date: archive_date(text),
|
30
|
+
contact_info: contact(text),
|
31
|
+
description: description(text),
|
32
|
+
link_url: link_url(text),
|
33
|
+
link_description: link_description(text),
|
34
|
+
email_address: email_address(text),
|
35
|
+
email_description: email_description(text),
|
36
|
+
setaside: set_aside(text),
|
37
|
+
pop_address: pop_address(text),
|
38
|
+
pop_zip: pop_zip_code(text),
|
39
|
+
pop_country: pop_country(text)
|
40
|
+
}
|
41
|
+
FBO::Notices::Amendment.new(params)
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
protected
|
46
|
+
|
47
|
+
# Returns a concatenated list of all tags for the notice
|
48
|
+
#
|
49
|
+
# @return [String]
|
50
|
+
def any_notice_tag
|
51
|
+
ANY_AMDCSS_TAG
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
extend FBO::Parser::NoticeHandler
|
56
|
+
extend FBO::Parser::ParserHelper
|
57
|
+
extend self
|
58
|
+
end
|