fbo 0.0.3 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0c4e53ddeb9ea9df943ef8af9ca33a4e1f99bbc
-  data.tar.gz: bc42c21126580a453a02996fa1799375d7057339
+  metadata.gz: f7da92f700f183764c8e4964b1b3f8c68f2cea39
+  data.tar.gz: cc5fb5d1eeef566f49ca7481aba104f977ead804
 SHA512:
-  metadata.gz: 02ea8f7e4d11ba748b01abbe212952c958cb8e20f95b690dfd4b1da657e7bbdb6f744728395b8c2a3c523b8208d5f2560ff57811e3cc92dd2f27b2654d6dada7
-  data.tar.gz: 02ec6b226bbf7c58391fb0073c28bedb6635ad96dc73676f4c353511e585d6554cf27f3226a05c7e6cf6ac7f7c748c8e70703a4728e4e03d4b9fe3c30600e468
+  metadata.gz: 81cefd851decaefb1e611a0ca7509e9ced5e942eaab3797fa268117d23f5d4eb2b08682a27811699b9ed3fbc4786e4deec0a5c67d445f2a7a7108204c42a139d
+  data.tar.gz: 1ebca32362436a4b6737c00eb05c04c3f1cba34d777d45f5dc1458c36ea8acbba3469afc524d28e3b0a116528c75e8e1326b7132a6796d381d2bc5f45c140fb3

data/.gitignore

@@ -5,8 +5,9 @@
 
 # Bundler artifacts
 .bundle
-bundler_stubs/
-vendor/
+bin/
+Gemfile.lock
+vendor/bundle
 
 # Ruby / Rails operational files
 .rvmrc
@@ -18,9 +19,6 @@ vendor/
 *.swo
 tmp
 
-# Reporting directory
-/reports/
-
 # Rubygems artifacts
 *.gem
 

data/Gemfile

@@ -2,5 +2,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fbo.gemspec
 gemspec
-
-gem 'rspec'

data/Rakefile

@@ -1 +1,8 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs += %w( spec test )
+  t.pattern = '{spec,test}/**/*_{spec,test}.rb'
+end

data/fbo.gemspec

@@ -4,11 +4,11 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'fbo/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "fbo"
+  spec.name          = 'fbo'
   spec.version       = FBO::VERSION
-  spec.authors       = ["Chris Kottom"]
-  spec.email         = ["chris@chriskottom.com"]
-  spec.summary       = "Parse and process FBO bulk listings"
+  spec.authors       = ['Chris Kottom']
+  spec.email         = ['chris@chriskottom.com']
+  spec.summary       = 'Parse and process FBO bulk listings'
   spec.description   = <<-EOF
     The FBO gem manages the process of downloading and parsing file-based notice
     information from the Federal Business Opportunities (https://www.fbo.gov/)
@@ -17,14 +17,19 @@ Gem::Specification.new do |spec|
     concerning the offer and disposition of federal government contracts and
     tenders.
   EOF
-  spec.homepage      = ""
-  spec.license       = "MIT"
+  spec.homepage      = ''
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files`.split($/)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency 'treetop'
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'minitest', '~> 4.7.5'
+  spec.add_development_dependency 'mocha'
+  spec.add_development_dependency 'turn'
 end

data/lib/fbo.rb

@@ -1,42 +1,11 @@
-require_relative "fbo/version"
+require 'fbo/version'
 
-require_relative "fbo/file"
-require_relative "fbo/remote_file"
-require_relative "fbo/notice"
-require_relative "fbo/notices/amendment"
-require_relative "fbo/notices/archive"
-require_relative "fbo/notices/award"
-require_relative "fbo/notices/combined_solicitation"
-require_relative "fbo/notices/fair_opportunity"
-require_relative "fbo/notices/foreign_standard"
-require_relative "fbo/notices/intent_to_bundle"
-require_relative "fbo/notices/justification_approval"
-require_relative "fbo/notices/modification"
-require_relative "fbo/notices/presolicitation"
-require_relative "fbo/notices/sale_of_surplus"
-require_relative "fbo/notices/sources_sought"
-require_relative "fbo/notices/special_notice"
-require_relative "fbo/notices/unarchive"
-require_relative "fbo/notices/unknown"
-require_relative "fbo/parser"
-require_relative "fbo/parser/handler_selector"
-require_relative "fbo/parser/notice_handler"
-require_relative "fbo/parser/parser_helper"
-require_relative "fbo/parser/amendment_handler"
-require_relative "fbo/parser/archive_handler"
-require_relative "fbo/parser/award_handler"
-require_relative "fbo/parser/combined_solicitation_handler"
-require_relative "fbo/parser/fair_opportunity_handler"
-require_relative "fbo/parser/foreign_standard_handler"
-require_relative "fbo/parser/intent_to_bundle_handler"
-require_relative "fbo/parser/justification_approval_handler"
-require_relative "fbo/parser/modification_handler"
-require_relative "fbo/parser/presolicitation_handler"
-require_relative "fbo/parser/sale_of_surplus_handler"
-require_relative "fbo/parser/sources_sought_handler"
-require_relative "fbo/parser/special_notice_handler"
-require_relative "fbo/parser/unarchive_handler"
-require_relative "fbo/parser/unknown_handler"
+require 'fbo/file'
+require 'fbo/chunked_file'
+require 'fbo/remote_file'
+require 'fbo/parser'
+require 'fbo/node_extensions'
+require 'fbo/interpreter'
 
 module FBO
 end

data/lib/fbo/chunked_file.rb

@@ -0,0 +1,74 @@
+require 'forwardable'
+
+module FBO
+  class ChunkedFile
+    extend Forwardable
+
+    attr_reader     :file, :chunk_size
+    def_delegators  :@file, :readline, :read, :eof?
+
+
+    DEFAULT_CHUNK_SIZE    = 250 * 1024   # 250KB
+    NOTICE_TAG_NAMES      = %w( PRESOL COMBINE AMDCSS MOD AWARD JA ITB FAIROPP FSTD) +
+                            %w( SRCSGT SNOTE SSALE EPSUPLOAD DELETE ARCHIVE UNARCHIVE )
+    NOTICE_CLOSE_REGEXP   = /<\/(#{ NOTICE_TAG_NAMES.join('|') })>$/
+
+
+    def initialize(file, chunk_size = DEFAULT_CHUNK_SIZE)
+      @file = file
+      @chunk_size = chunk_size
+    end
+
+    def contents
+      if @contents.nil?
+        @contents = []
+        while !eof?
+          @contents << next_chunk
+        end
+        @contents.compact!
+      end
+      @contents
+    end
+
+
+    private
+
+    def next_chunk
+      return nil if eof?
+
+      chunk, line = "", ""
+
+      # Run up the chunk to about #chunk_size.
+      begin
+        line = gets
+        break unless line
+        chunk += line
+      end while (chunk.bytesize < @chunk_size)
+
+      # Add lines up to the end of a notice.
+      if line && line !~ NOTICE_CLOSE_REGEXP
+        begin
+          line = gets
+          break unless line
+          chunk += line
+          break if line =~ NOTICE_CLOSE_REGEXP
+        end while (true)
+      end
+
+      return chunk.strip
+    end
+
+    def gets
+      line = file.gets
+      return unless line
+      cleanup_data(line)
+    end
+ 
+    def cleanup_data(data)
+      data.encode('UTF-16le', :invalid => :replace, :replace => '')
+          .encode('UTF-8')
+          .gsub(/\r\n/, "\n")
+          .gsub(/^M/, "")
+    end
+  end
+end

data/lib/fbo/dump.treetop

@@ -0,0 +1,324 @@
+module FBO
+  grammar Dump
+
+    # Non-domain structural elements
+    #
+    rule dumpfile
+      notice* <DumpNode>
+    end
+
+    rule notice
+      presol / combine / amdcss / mod / award / ja / itb / fairopp / srcsgt /
+      fstd / snote / ssale / epsupload / delete / archive / unarchive
+    end
+
+    rule body
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      body_tag? body_tag? body_tag? body_tag? body_tag? body_tag? body_tag?
+      <BodyNode>
+    end
+
+    rule tag_open
+      '<' tag_name '>'
+    end
+
+    rule tag_close
+      '</' ( notice_tag_name / 'FILELIST' / 'FILE' ) '>'
+    end
+
+    rule end_of_tag
+      space? &( tag_open / tag_close / end_of_buffer )
+    end
+
+    rule tag_content
+      ( !end_of_tag ( . / "\n" ) )* <TagContentNode>
+    end
+
+    rule space
+      [ \t\n\r]+
+    end
+
+    rule end_of_buffer
+      !.
+    end
+
+
+    # Domain constants
+    #
+    rule notice_tag_name
+      'PRESOL' / 'COMBINE' / 'AMDCSS' / 'MOD' / 'AWARD' / 'JA' / 'ITB' /
+      'FAIROPP' / 'SRCSGT' / 'FSTD' / 'SNOTE' / 'SSALE' / 'EPSUPLOAD' /
+      'DELETE' / 'ARCHIVE' / 'UNARCHIVE'
+    end
+
+    rule body_tag_name
+      'DATE' / 'YEAR' / 'CBAC' / 'PASSWORD' / 'ZIP' / 'CLASSCOD' / 'NAICS' /
+      'OFFADD' / 'CONTACT' / 'SUBJECT' / 'DESC' / 'SOLNBR' / 'NTYPE' /
+      'RESPDATE' / 'ARCHDATE' / 'LINK' / 'URL' / 'EMAIL' / 'ADDRESS' /
+      'FILELIST' / 'FILE' / 'URL' / 'MIMETYPE' / 'SETASIDE' / 'POPADDRESS' /
+      'POPZIP' / 'POPCOUNTRY' / 'CORRECTION' / 'AWDNBR' / 'AWDAMT' / 'LINENBR' /
+      'AWDDATE' / 'AWARDEEDUNS' / 'AWARDEE_DUNS' / 'AWARDEE' / 'STAUTH' /
+      'MODNBR' / 'USERID' / 'PROJID' / 'UPLOADTYPE' / 'DONBR' / 'FOJA' /
+      'RESPONSEDATE' / 'AGENCY' / 'LOCATION' / 'OFFICE'
+    end
+
+    rule tag_name
+      body_tag_name / notice_tag_name
+    end
+
+
+    # Notice types
+    #
+    rule presol
+      '<PRESOL>' space? body '</PRESOL>' space? <PresolicitationNode>
+    end
+
+    rule combine
+      '<COMBINE>' space? body '</COMBINE>' space? <CombinedSolicitationNode>
+    end
+
+    rule amdcss
+      '<AMDCSS>' space? body '</AMDCSS>' space? <AmendmentNode>
+    end
+
+    rule mod
+      '<MOD>' space? body '</MOD>' space? <ModificationNode>
+    end
+
+    rule award
+      '<AWARD>' space? body '</AWARD>' space? <AwardNode>
+    end
+
+    rule ja
+      '<JA>' space? body '</JA>' space? <JustificationAndApprovalNode>
+    end
+
+    rule itb
+      '<ITB>' space? body '</ITB>' space? <IntentToBundleNode>
+    end
+
+    rule fairopp
+      '<FAIROPP>' space? body '</FAIROPP>' space? <FairOpportunityNode>
+    end
+
+    rule srcsgt
+      '<SRCSGT>' space? body '</SRCSGT>' space? <SourcesSoughtNode>
+    end
+
+    rule fstd
+      '<FSTD>' space? body '</FSTD>' space? <ForeignStandardNode>
+    end
+
+    rule snote
+      '<SNOTE>' space? body '</SNOTE>' space? <SpecialNoticeNode>
+    end
+
+    rule ssale
+      '<SSALE>' space? body '</SSALE>' space? <SaleOfSurplusNode>
+    end
+
+    rule epsupload
+      '<EPSUPLOAD>' space? body '</EPSUPLOAD>' space? <DocumentUploadNode>
+    end
+
+    rule delete
+      '<DELETE>' space? body '</DELETE>' space? <DocumentDeletingNode>
+    end
+
+    rule archive
+      '<ARCHIVE>' space? body '</ARCHIVE>' space? <DocumentArchivalNode>
+    end
+
+    rule unarchive
+      '<UNARCHIVE>' space? body '</UNARCHIVE>' space? <DocumentUnarchivalNode>
+    end
+
+
+    # Body tags
+    #
+    rule body_tag
+      date / year / zip / agency / office / location / classcod / naics /
+      offadd / subject / solnbr / respdate / contact / desc / link / url /
+      setaside / popcountry / popzip / popaddress / archdate / ntype / awdnbr /
+      awdamt / linenbr / awardee / awardee_duns / awddate / email /
+      email_address / stauth / foja / donbr / cbac / password / projid /
+      uploadtype / filelist / file / mimetype / correction
+    end
+
+    rule date
+      '<DATE>' space? tag_content end_of_tag <DateNode>
+    end
+
+    rule year
+      '<YEAR>' space? tag_content end_of_tag <YearNode>
+    end
+
+    rule cbac
+      ( '<CBAC>' / '<USERID>' ) space? tag_content end_of_tag <CBACNode>
+    end
+
+    rule password
+      '<PASSWORD>' space? tag_content end_of_tag <PasswordNode>
+    end
+
+    rule zip
+      '<ZIP>' space? tag_content end_of_tag <ZipNode>
+    end
+
+    rule agency
+      '<AGENCY>' space? tag_content end_of_tag <AgencyNode>
+    end
+
+    rule office
+      '<OFFICE>' space? tag_content end_of_tag <OfficeNode>
+    end
+
+    rule location
+      '<LOCATION>' space? tag_content end_of_tag <LocationNode>
+    end
+
+    rule classcod
+      '<CLASSCOD>' space? tag_content end_of_tag <ClassificationCodeNode>
+    end
+
+    rule naics
+      '<NAICS>' space? tag_content end_of_tag <NaicsCodeNode>
+    end
+
+    rule offadd
+      '<OFFADD>' space? tag_content end_of_tag <OfficeAddressNode>
+    end
+
+    rule subject
+      '<SUBJECT>' space? tag_content end_of_tag <SubjectNode>
+    end
+
+    rule solnbr
+      '<SOLNBR>' space? tag_content end_of_tag <SolicitationNumberNode>
+    end
+
+    rule respdate
+      ( '<RESPDATE>' / '<RESPONSEDATE>' ) space? tag_content end_of_tag <ResponseDateNode>
+    end
+
+    rule archdate
+      '<ARCHDATE>' space? tag_content end_of_tag <ArchiveDateNode>
+    end
+
+    rule contact
+      '<CONTACT>' space? tag_content end_of_tag <ContactNode>
+    end
+
+    rule desc
+      '<DESC>' space? tag_content end_of_tag <DescriptionNode>
+    end
+
+    rule link
+      '<LINK>' space? url? desc? url? <LinkNode> 
+    end
+
+    rule url
+      '<URL>' space? tag_content end_of_tag <UrlNode>
+    end
+
+    rule setaside
+      '<SETASIDE>' space? tag_content end_of_tag <SetAsideNode>
+    end
+
+    rule popcountry
+      '<POPCOUNTRY>' space? tag_content end_of_tag <PopCountryNode>
+    end
+
+    rule popzip
+      '<POPZIP>' space? tag_content end_of_tag <PopZipNode>
+    end
+
+    rule popaddress
+      '<POPADDRESS>' space? tag_content end_of_tag <PopAddressNode>
+    end
+
+    rule ntype
+      '<NTYPE>' space? tag_content end_of_tag <NoticeTypeNode>
+    end
+
+    rule awdnbr
+      '<AWDNBR>' space? tag_content end_of_tag <AwardNumberNode>
+    end
+
+    rule awdamt
+      '<AWDAMT>' space? tag_content end_of_tag <AwardAmountNode>
+    end
+
+    rule awddate
+      '<AWDDATE>' space? tag_content end_of_tag <AwardDateNode>
+    end
+
+    rule linenbr
+      '<LINENBR>' space? tag_content end_of_tag <LineNumberNode>
+    end
+
+    rule awardee
+      '<AWARDEE>' space? tag_content end_of_tag <AwardeeNode>
+    end
+
+    rule awardee_duns
+      ( '<AWARDEEDUNS>' / '<AWARDEE_DUNS>' ) space? tag_content end_of_tag <AwardeeDunsNode>
+    end
+
+    rule email_address
+      ( '<EMAIL>' / '<ADDRESS>' ) space? tag_content end_of_tag <EmailAddressNode>
+    end
+
+    rule email
+      '<EMAIL>' space? email_address? desc? email_address? <EmailNode>
+    end
+
+    rule stauth
+      '<STAUTH>' space? tag_content end_of_tag <StatutoryAuthorityNode>
+    end
+
+    rule foja
+      '<FOJA>' space? tag_content end_of_tag <JustificationAuthorityNode>
+    end
+
+    rule donbr
+      '<DONBR>' space? tag_content end_of_tag <DeliveryOrderNumberNode>
+    end
+
+    rule projid
+      '<PROJID>' space? tag_content end_of_tag <ProjectIDNode>
+    end
+
+    rule uploadtype
+      '<UPLOADTYPE>' space? tag_content end_of_tag <UploadTypeNode>
+    end
+
+    rule filelist
+      '<FILELIST>' space? file? space? file? space? '</FILELIST>' space? <FileListNode>
+    end
+
+    rule file
+      '<FILE>' space? url? desc? mimetype? url? desc? url? mimetype? desc? space? '</FILE>' <FileNode>
+    end
+
+    rule mimetype
+      '<MIMETYPE>' space? tag_content end_of_tag <MimeTypeNode>
+    end
+
+    rule correction
+      '<CORRECTION>' space? tag_content end_of_tag <CorrectionNode>
+    end
+
+    rule modnbr
+      '<MODNBR>' space? tag_content end_of_tag <ModificationNumberNode>
+    end
+  end
+end