fbo 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 341885ee905b5466fe116ddfbef59dc1ceb9fc1b
+  data.tar.gz: 0f9954197c0c2cf290b860756a0e13bbf482f415
+SHA512:
+  metadata.gz: df079c191ef00e41a23e3ce5b881131db4ef43ad786f39a4094a6b3b9e329b2d3f96a28406e608dc89cdb9947f13e5f8b5d5bf065787f6e802165a7be423f73d
+  data.tar.gz: 102f3ecbb75bb265890745ff7da8ae0a491977b1c0d9be81779d315bcef655ff5bef5c5adc651beb0cfed5dfeae50e316a1e538303895811a85b5e7422b1fd51

data/.gitignore

@@ -0,0 +1,37 @@
+# Emacs temporary files
+*~
+\#*\#
+
+# Bundler artifacts
+.bundle
+bundler_stubs/
+vendor/
+
+# Ruby / Rails operational files
+.rvmrc
+.ruby-version
+
+# Anything temporary
+*.old
+*.swp
+*.swo
+tmp
+
+# Reporting directory
+/reports/
+
+# Rubygems artifacts
+*.gem
+
+# Test artifacts
+coverage
+spec/reports
+test/tmp
+test/version_tmp
+
+# Documentation
+.yardoc
+_yardoc
+doc
+rdoc
+

data/.rspec

@@ -0,0 +1,2 @@
+--format doc
+--color

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fbo.gemspec
+gemspec
+
+gem 'rspec'

data/Gemfile.lock

@@ -0,0 +1,27 @@
+PATH
+  remote: .
+  specs:
+    FBO (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.2)
+    rake (10.0.4)
+    rspec (2.13.0)
+      rspec-core (~> 2.13.0)
+      rspec-expectations (~> 2.13.0)
+      rspec-mocks (~> 2.13.0)
+    rspec-core (2.13.1)
+    rspec-expectations (2.13.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.13.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  FBO!
+  bundler (~> 1.3)
+  rake
+  rspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Chris Kottom
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# FBO
+
+Fetch a dated dump file from [FedBizOpps](https://www.fbo.gov) and parse its
+contents into structures suitable for use by other programs.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fbo'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fbo
+
+## Usage
+
+The gem provides a number of 
+
+
+
+## Contributing
+
+Code contributions gratefully accepted.
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+Logging any issues you might notice at the
+[Issues tracker](https://github.com/chriskottom/fbo/issues)
+on GitHub is also a great way to help out.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/fbo.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fbo/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fbo"
+  spec.version       = FBO::VERSION
+  spec.authors       = ["Chris Kottom"]
+  spec.email         = ["chris@chriskottom.com"]
+  spec.summary       = "Parse and process FBO bulk listings"
+  spec.description   = <<-EOF
+    The FBO gem manages the process of downloading and parsing file-based notice
+    information from the Federal Business Opportunities (https://www.fbo.gov/)
+    application / database. The FBO feed files include new and updated
+    opportunities, information about awarded contracts, and other details
+    concerning the offer and disposition of federal government contracts and
+    tenders.
+  EOF
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/fbo.rb

@@ -0,0 +1,25 @@
+require_relative "fbo/version"
+
+require_relative "fbo/file"
+require_relative "fbo/remote_file"
+require_relative "fbo/notice"
+require_relative "fbo/notices/amendment"
+require_relative "fbo/notices/award"
+require_relative "fbo/notices/combined_solicitation"
+require_relative "fbo/notices/modification"
+require_relative "fbo/notices/presolicitation"
+require_relative "fbo/notices/sources_sought"
+require_relative "fbo/notices/unknown"
+require_relative "fbo/parser"
+require_relative "fbo/parser/notice_handler"
+require_relative "fbo/parser/parser_helper"
+require_relative "fbo/parser/amendment_handler"
+require_relative "fbo/parser/award_handler"
+require_relative "fbo/parser/combined_solicitation_handler"
+require_relative "fbo/parser/modification_handler"
+require_relative "fbo/parser/presolicitation_handler"
+require_relative "fbo/parser/sources_sought_handler"
+require_relative "fbo/parser/unknown_handler"
+
+module FBO
+end

data/lib/fbo/file.rb

@@ -0,0 +1,14 @@
+require 'forwardable'
+
+module FBO
+  class File
+    extend Forwardable
+
+    attr_reader     :file
+    def_delegators  :@file, :open, :readline, :read
+
+    def initialize(filename)
+      @file = ::File.new(filename)
+    end
+  end
+end

data/lib/fbo/notice.rb

@@ -0,0 +1,9 @@
+module FBO
+  module Notice
+    def initialize(params = {})
+      params.each do |key, value|
+        self[key] = value if members.include?(key)
+      end
+    end
+  end
+end

data/lib/fbo/notices.rb

@@ -0,0 +1,6 @@
+module FBO
+  module Notices
+    NOTICE_TYPES = %i{ PRESOL COMBINE AMDCSS MOD AWARD JA ITB FAIROPP } +
+      %i{ SRCSGT FSTD SNOTE SSALE EPSUPLOAD DELETE ARCHIVE UNARCHIVE }
+  end
+end

data/lib/fbo/notices/amendment.rb

@@ -0,0 +1,13 @@
+module FBO
+  module Notices
+    class Amendment < Struct.new(:date, :year, :zip, :class_code, :naics_code,
+                                 :agency, :office, :location, :office_address,
+                                 :subject, :solicitation_number, :notice_type,
+                                 :response_date, :archive_date, :contact_info,
+                                 :description, :link_url, :link_description,
+                                 :email_address, :email_description, :setaside,
+                                 :pop_address, :pop_zip, :pop_country)
+      include FBO::Notice
+    end
+  end
+end

data/lib/fbo/notices/award.rb

@@ -0,0 +1,14 @@
+module FBO
+  module Notices
+    class Award < Struct.new(:date, :year, :zip, :class_code, :naics_code,
+                             :agency, :office, :location, :office_address,
+                             :subject, :solicitation_number, :notice_type,
+                             :description, :contact_info, :award_number,
+                             :award_amount, :line_number, :award_date, 
+                             :archive_date, :awardee, :awardee_duns, :link_url,
+                             :link_description, :email_address, :email_description,
+                             :setaside, :correction)
+      include FBO::Notice
+    end
+  end
+end

data/lib/fbo/notices/combined_solicitation.rb

@@ -0,0 +1,14 @@
+module FBO
+  module Notices
+    class CombinedSolicitation < Struct.new(:date, :year, :zip, :class_code,
+                                            :naics_code, :agency, :office, :location,
+                                            :office_address, :subject, :solicitation_number,
+                                            :response_date, :archive_date,
+                                            :contact_info, :description, :link_url,
+                                            :link_description, :email_address,
+                                            :email_description, :setaside, :pop_address,
+                                            :pop_zip, :pop_country)
+      include FBO::Notice
+    end
+  end
+end

data/lib/fbo/notices/modification.rb

@@ -0,0 +1,14 @@
+module FBO
+  module Notices
+    class Modification < Struct.new(:date, :year, :zip, :class_code,
+                                    :naics_code, :agency, :office, :location,
+                                    :office_address, :subject, :solicitation_number,
+                                    :notice_type, :response_date, :archive_date,
+                                    :contact_info, :description, :link_url,
+                                    :link_description, :email_address,
+                                    :email_description, :setaside, :pop_address,
+                                    :pop_zip, :pop_country)
+      include FBO::Notice
+    end
+  end
+end

data/lib/fbo/notices/presolicitation.rb

@@ -0,0 +1,14 @@
+module FBO
+  module Notices
+    class Presolicitation < Struct.new(:date, :year, :zip, :class_code,
+                                       :naics_code, :agency, :office, :location,
+                                       :office_address, :subject, :solicitation_number,
+                                       :response_date, :archive_date,
+                                       :contact_info, :description, :link_url,
+                                       :link_description, :email_address,
+                                       :email_description, :setaside, :pop_address,
+                                       :pop_zip, :pop_country)
+      include FBO::Notice
+    end
+  end
+end

data/lib/fbo/notices/sources_sought.rb

@@ -0,0 +1,13 @@
+module FBO
+  module Notices
+    class SourcesSought < Struct.new(:date, :year, :zip, :class_code, :naics_code,
+                                     :agency, :office, :location, :office_address,
+                                     :subject, :solicitation_number,
+                                     :response_date, :archive_date, :contact_info,
+                                     :description, :link_url, :link_description,
+                                     :email_address, :email_description, :setaside,
+                                     :pop_address, :pop_zip, :pop_country)
+      include FBO::Notice
+    end
+  end
+end

data/lib/fbo/notices/unknown.rb

@@ -0,0 +1,7 @@
+module FBO
+  module Notices
+    class Unknown < Struct.new(:type, :text)
+      include FBO::Notice
+    end
+  end
+end

data/lib/fbo/parser.rb

@@ -0,0 +1,79 @@
+require "strscan"
+
+module FBO
+  class Parser
+    TAG_PATTERN = /<\w+>/m
+    PATTERN = /<([A-Z]+)>.*<\/\1>/m
+
+    def initialize(file, tag_pattern: TAG_PATTERN)
+      @file = file
+      @tag_pattern = tag_pattern
+    end
+
+    def notices
+      if @notices.nil?
+        parse_notices(split_file_contents)
+      end
+      @notices
+    end
+
+    private
+
+    def split_file_contents
+      contents = @file.read
+      contents = cleanup_contents(contents)
+      scanner = StringScanner.new(contents)
+      text_notices = []
+
+      while !scanner.eos?
+        initial_tag = scanner.check_until(@tag_pattern)
+        if initial_tag
+          initial_tag.strip!
+        else
+          break
+        end
+
+        text_notice = scanner.scan_until(/#{ closing_tag_for(initial_tag) }\s*/)
+        if text_notice
+          text_notices << text_notice.strip
+        else
+          break
+        end
+      end
+      text_notices
+    end
+
+    def parse_notices(text_notices)
+      @notices = []
+      text_notices.each do |text|
+        if FBO::Parser::PresolicitationHandler.is_presolicitation?(text)
+          @notices << FBO::Parser::PresolicitationHandler.parse(text)
+        elsif FBO::Parser::CombinedSolicitationHandler.is_combined_solicitation?(text)
+          @notices << FBO::Parser::CombinedSolicitationHandler.parse(text)
+        elsif FBO::Parser::SourcesSoughtHandler.is_sources_sought?(text)
+          @notices << FBO::Parser::SourcesSoughtHandler.parse(text)
+        elsif FBO::Parser::AmendmentHandler.is_amendment?(text)
+          @notices << FBO::Parser::AmendmentHandler.parse(text)
+        elsif FBO::Parser::ModificationHandler.is_modification?(text)
+          @notices << FBO::Parser::ModificationHandler.parse(text)
+        elsif FBO::Parser::AwardHandler.is_award?(text)
+          @notices << FBO::Parser::AwardHandler.parse(text)
+        else
+          @notices << FBO::Parser::UnknownHandler.parse(text)
+        end
+      end
+    end
+
+    def cleanup_contents(contents)
+      contents
+        .encode('UTF-16le', :invalid => :replace, :replace => '')
+        .encode('UTF-8')
+        .gsub(/\r\n/, "\n")
+        .gsub(/^M/, "")
+    end
+
+    def closing_tag_for(tag)
+      tag.sub(/</, "</")
+    end
+  end
+end

data/lib/fbo/parser/amendment_handler.rb

@@ -0,0 +1,58 @@
+module FBO::Parser::AmendmentHandler
+  TAGS = %w( AMDCSS DATE YEAR CBAC PASSWORD ZIP CLASSCOD NAICS OFFADD NTYPE) +
+    %w( AGENCY OFFICE LOCATION SUBJECT SOLNBR RESPDATE ARCHDATE CONTACT DESC ) +
+    %w( LINK URL EMAIL ADDRESS SETASIDE POPADDRESS POPZIP POPCOUNTRY \/AMDCSS )
+
+  ANY_AMDCSS_TAG     = TAGS.join("|")
+  AMENDMENT_PATTERN  = /^<AMDCSS>/
+
+
+  def is_amendment?(text)
+    text =~ AMENDMENT_PATTERN
+  end
+
+  def parse(text)
+    params = {
+      date: date(text),
+      year: year(text),
+      zip:  zip(text),
+      class_code: classification_code(text),
+      naics_code: naics_code(text),
+      agency: agency(text),
+      office: office(text),
+      location: location(text),
+      office_address: office_address(text),
+      subject: subject(text),
+      solicitation_number: solicitation_number(text),
+      notice_type: notice_type(text),
+      response_date: response_date(text),
+      archive_date: archive_date(text),
+      contact_info: contact(text),
+      description: description(text),
+      link_url: link_url(text),
+      link_description: link_description(text),
+      email_address: email_address(text),
+      email_description: email_description(text),
+      setaside: set_aside(text),
+      pop_address: pop_address(text),
+      pop_zip: pop_zip_code(text),
+      pop_country: pop_country(text)
+    }
+    FBO::Notices::Amendment.new(params)
+  end
+
+
+  protected
+
+  # Returns a concatenated list of all tags for the notice
+  #
+  # @return [String]
+  def any_notice_tag
+    ANY_AMDCSS_TAG
+  end
+
+
+  extend FBO::Parser::NoticeHandler
+  extend FBO::Parser::ParserHelper
+  extend self
+end