fbo 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 038791a71715415069e8fef85ee55570b4c604a7
-  data.tar.gz: ec28f0ae4378dd0d7066bada202ce433d8f5ef32
+  metadata.gz: cb42052147996e0d44fdbaee04e245b538affecb
+  data.tar.gz: 4fb9642b21ee11a0840110cd21ed1488ba779662
 SHA512:
-  metadata.gz: 29f5094d45d2a9014447e79ffd852da117b4040005713420ec186e2adf1491223e3b910f7f0e5c577e8d4b1c431d960b59783646ff9678dd92746c3b394e9268
-  data.tar.gz: 4499d2b12c10327761a83c834db340aa515b44bfdb822f1e0b423596fa1a166cd6a8f401600f5fc2d5934cdec9ecf9edafd0b9cbb3ec4345ddc78f30afd2cd12
+  metadata.gz: 5cdded369892b66f85d27ee09eacff5dcbca57f96508eed2d119b01d93e7c66b1f4ed86c8185dadbf4797ab211d1b59c8dad59040343b88db4ec7fd515a7b71f
+  data.tar.gz: 0c496f7dc170fcb6bd98eef097fcc74a422049c6ee10ab3e3b0950881bff49f31dd900b77a4f437d025795bcd2eca6b0067d427e218e95ab227d9fa77af9a050

data/lib/fbo/chunked_file.rb

@@ -1,11 +1,11 @@
 require 'forwardable'
 
 module FBO
-  class ChunkedFile
+  class ChunkedFile < File
     extend Forwardable
 
     attr_reader     :file, :chunk_size
-    def_delegators  :@file, :readline, :read, :eof?
+    def_delegators  :@file, :eof?, :eof
 
 
     DEFAULT_CHUNK_SIZE    = 250 * 1024   # 250KB
@@ -19,7 +19,7 @@ module FBO
     def contents
       if @contents.nil?
         @contents = []
-        while !eof?
+        while !eof
           @contents << next_chunk
         end
         @contents.compact!
@@ -31,7 +31,7 @@ module FBO
     private
 
     def next_chunk
-      return nil if eof?
+      return nil if eof
 
       chunk, line = "", ""
 
@@ -40,7 +40,7 @@ module FBO
         line = gets
         break unless line
         chunk += line
-      end while (chunk.bytesize < @chunk_size)
+      end while (chunk.bytesize < @chunk_size && !eof)
 
       # Add lines up to the end of a notice.
       if line && line !~ FBO::NOTICE_CLOSE_REGEXP
@@ -49,23 +49,10 @@ module FBO
           break unless line
           chunk += line
           break if line =~ FBO::NOTICE_CLOSE_REGEXP
-        end while (true)
+        end while (!eof)
       end
 
       return chunk.strip
     end
-
-    def gets
-      line = file.gets
-      return unless line
-      cleanup_data(line)
-    end
- 
-    def cleanup_data(data)
-      data.encode('UTF-16le', :invalid => :replace, :replace => '')
-          .encode('UTF-8')
-          .gsub(/\r\n/, "\n")
-          .gsub(/^M/, "")
-    end
   end
 end

data/lib/fbo/file.rb

@@ -5,7 +5,7 @@ module FBO
     extend Forwardable
 
     attr_reader     :file
-    def_delegators  :@file, :readline, :read, :eof?, :gets
+    def_delegators  :@file, :eof?, :eof
 
     class << self
       def filename_for_date(date)
@@ -18,6 +18,11 @@ module FBO
       @file = ::File.new(filename)
     end
 
+    def gets
+      str = @file.gets
+      str.nil? ? nil : cleanup_data(str)
+    end
+
     def contents
       if @contents.nil?
         @contents = cleanup_data(@file.read)
@@ -25,7 +30,7 @@ module FBO
       @contents
     end
 
-    private
+    protected
 
     def cleanup_data(data)
       data.encode('UTF-16le', :invalid => :replace, :replace => '')

data/lib/fbo/interpreter.rb

@@ -1,76 +1,47 @@
 module FBO
   class Interpreter
-    class << self
-      def notice_enumeration(name, type)
-        define_method(name) do |&block|
-          selected = notice_nodes(type)
-          return unless selected
-          selected.each do |node|
-            block.call node.to_hash.merge({ type: node.type })
-          end
-        end
-      end
-    end
-
-    notice_enumeration :each_notice, :all
-    notice_enumeration :each_presolicitation, :presol
-    notice_enumeration :each_combined_solicitation, :combine
-    notice_enumeration :each_amendment, :amdcss
-    notice_enumeration :each_modification, :mod
-    notice_enumeration :each_award, :award
-    notice_enumeration :each_justification_and_approval, :ja
-    notice_enumeration :each_intent_to_bundle, :itb
-    notice_enumeration :each_fair_opportunity, :fairopp
-    notice_enumeration :each_sources_sought, :srcsgt
-    notice_enumeration :each_foreign_standard, :fstd
-    notice_enumeration :each_special_notice, :snote
-    notice_enumeration :each_sale_of_surplus, :ssale
-    notice_enumeration :each_document_upload, :epsupload
-    notice_enumeration :each_document_delete, :delete
-    notice_enumeration :each_document_archival, :archive
-    notice_enumeration :each_document_unarchival, :unarchive
-
     def initialize(file)
       @file = file
+      @parser = Parser.new
     end
 
-    private
+    # Walk through the file returning each notice text block and structure.
+    # If a block is given, yield passing the text block and structure as args.
+    #
+    def next_notice(&block)
+      return unless notice_text = next_notice_string
+      return if notice_text.empty?
 
-    def notice_nodes(type = :all)
-      if @notice_nodes.nil?
-        @notice_nodes = {}
-      end
-
-      if @file.is_a?(FBO::SegmentedFile) && type != :all
-        @notice_nodes[type] = parse_segment(type)
-      else
-        @notice_nodes[:all] = parse_file
-        if type == :all
-          @notice_nodes[:all]
-        else
-          @notice_nodes[type] = @notice_nodes[:all].select { |n| n.type == type }
-        end
-      end
+      notice_node = parse_notice(notice_text)
+      yield notice_text, notice_node if block_given?
+      return notice_text, notice_node
     end
 
-    def parse_file
-      tree = FBO::Parser.new(@file).parse
-      tree.elements
-    end
 
-    def parse_segment(type = :all)
-      return unless @file.is_a? FBO::SegmentedFile
+    private
 
-      content = ""
-      if type == :all
-        content = @file.contents
-      else
-        content = @file.contents_for_type(type)
+    # Get the next whole notice from the FBO dump file.
+    # Return an empty string if no more exist
+    #
+    def next_notice_string
+      return if @file.eof?
+
+      line, entry_string = '', ''
+      while !@file.eof
+        line = @file.gets
+        break unless line
+        next unless line =~ /\S/
+        entry_string += line
+        break if line =~ FBO::NOTICE_CLOSE_REGEXP
       end
-      return unless content
+      return entry_string.strip
+    end
 
-      tree = FBO::Parser.new.parse(content)
-      tree.elements
+    # Parse the text to extract a structure of data for the notice.
+    #
+    def parse_notice(text)
+      tree = @parser.parse(text)
+      tree.elements.first
     end
   end
 end

data/lib/fbo/segmented_file.rb

@@ -1,11 +1,11 @@
 require 'forwardable'
 
 module FBO
-  class SegmentedFile
+  class SegmentedFile < File
     extend Forwardable
 
     attr_reader     :file
-    def_delegators  :@file, :readline, :read, :eof?
+    def_delegators  :@file, :eof?, :eof
 
 
     def initialize(file)

data/lib/fbo/version.rb

@@ -1,3 +1,3 @@
 module FBO
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/spec/fbo/chunked_file_spec.rb

@@ -1,9 +1,9 @@
 require 'spec_helper'
 
 describe FBO::ChunkedFile do
-  let(:filename)   { File.join(File.dirname(__FILE__), '..', 'fixtures', 'FBOFeed20131003') }
+  let(:filename)   { File.join(File.dirname(__FILE__), '..', 'fixtures', 'FBOFeed20131013') }
   let(:file)       { FBO::File.new(filename) }
-  let(:chunk_size) { 50 * 1024 }   # 50KB
+  let(:chunk_size) { 25 * 1024 }   # 25KB
   subject          { FBO::ChunkedFile.new(file, chunk_size) }
 
   describe '#contents' do

data/spec/fbo/interpreter_spec.rb

@@ -3,102 +3,22 @@ require 'spec_helper'
 describe FBO::Interpreter do
   let(:filename)  { File.join(File.dirname(__FILE__), '..', 'fixtures', 'FBOFeed20130331') }
   let(:file)      { FBO::File.new(filename) }
-  let(:actor)     { stub('Act on ALL the Notices', process: nil) }
+  let(:actor)     { MiniTest::Mock.new }
+
 
   context 'with input FBO::File' do
     subject  { FBO::Interpreter.new(file) }
 
-    describe '#each_notice' do
-      it 'acts on each notice' do
-        actor.expects(:process).with(instance_of(Hash)).times(8)
-        subject.each_notice { |n| actor.process(n) }
-      end
-    end
-
-    describe '#each_presolicitation' do
-      it 'acts on each PRESOL notice' do
-        actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
-        subject.each_presolicitation { |n| actor.process(n) }
-      end
-    end
-
-    describe '#each_sources_sought' do
-      it 'acts on each SRCSGT notice' do
-        actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
-        subject.each_sources_sought { |n| actor.process(n) }
-      end
-    end
-
-    describe '#each_special_notice' do
-      it 'does nothing (no SNOTE notices)' do
-        actor.expects(:process).never
-        subject.each_special_notice { |n| actor.process(n) }
-      end
-    end
-  end
-
-  context 'with input FBO::ChunkedFile' do
-    let(:chunked)   { FBO::ChunkedFile.new(file) }
-    subject         { FBO::Interpreter.new(chunked) }
-
-    describe '#each_notice' do
-      it 'acts on each notice' do
-        actor.expects(:process).with(instance_of(Hash)).times(8)
-        subject.each_notice { |n| actor.process(n) }
-      end
-    end
-
-    describe '#each_presolicitation' do
-      it 'acts on each PRESOL notice' do
-        actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
-        subject.each_presolicitation { |n| actor.process(n) }
-      end
-    end
-
-    describe '#each_sources_sought' do
-      it 'acts on each SRCSGT notice' do
-        actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
-        subject.each_sources_sought { |n| actor.process(n) }
-      end
-    end
-
-    describe '#each_special_notice' do
-      it 'does nothing (no SNOTE notices)' do
-        actor.expects(:process).never
-        subject.each_special_notice { |n| actor.process(n) }
-      end
-    end
-  end
-
-  context 'with input FBO::SegmentedFile' do
-    let(:segmented) { FBO::SegmentedFile.new(file) }
-    subject         { FBO::Interpreter.new(segmented) }
-
-    describe '#each_notice' do
-      it 'acts on each notice' do
-        actor.expects(:process).with(instance_of(Hash)).times(8)
-        subject.each_notice { |n| actor.process(n) }
-      end
-    end
-
-    describe '#each_presolicitation' do
-      it 'acts on each PRESOL notice' do
-        actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
-        subject.each_presolicitation { |n| actor.process(n) }
-      end
-    end
-
-    describe '#each_sources_sought' do
-      it 'acts on each SRCSGT notice' do
-        actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
-        subject.each_sources_sought { |n| actor.process(n) }
-      end
-    end
+    describe '#next_notice' do
+      it 'acts on the first notice' do
+        actor.expect(:process, nil, [String, Treetop::Runtime::SyntaxNode ])
+        string, structure = subject.next_notice { |str, struct| actor.process(str, struct) }
+        actor.verify
 
-    describe '#each_special_notice' do
-      it 'does nothing (no SNOTE notices)' do
-        actor.expects(:process).never
-        subject.each_special_notice { |n| actor.process(n) }
+        string.must_match /^\<PRESOL\>/
+        string.must_match /\<SOLNBR\>SPE4A713R0795/
+        structure.type.must_equal :presol
+        structure.to_hash[:solicitation_number].must_equal 'SPE4A713R0795'
       end
     end
   end

data/spec/fbo/remote_file_spec.rb

@@ -38,7 +38,7 @@ describe FBO::RemoteFile do
     end
 
     it 'should to open the file and read its contents' do
-      subject.readline.wont_be_nil
+      subject.gets.wont_be_nil
     end
   end
 

metadata

@@ -1,97 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: fbo
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Chris Kottom
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-02 00:00:00.000000000 Z
+date: 2014-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: treetop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 4.7.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 4.7.5
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: turn
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: |2
@@ -107,7 +107,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -143,17 +143,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Parse and process FBO bulk listings