fbo 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 938f95ed22c71da91bf9582a430fd0598e7d5aba
-  data.tar.gz: b653b7d2c6212a955154c956c08ceeb1ddeb1f41
+  metadata.gz: 038791a71715415069e8fef85ee55570b4c604a7
+  data.tar.gz: ec28f0ae4378dd0d7066bada202ce433d8f5ef32
 SHA512:
-  metadata.gz: 7366ea869f51591cdf264c3412d9d85725356d94c4aab7a58852f6a65be08b4fe78974bcc19984070dfbac0abfd2a4e120279baa612fa091c7527a427c325328
-  data.tar.gz: 719b8a8624b7922fe93299a8439da06392524f066c98a15f39b2b6c80bd10d1fb32a41955d6ed9ffaa24585982a87a12be106ac5a34e00396d7104f07493ad4b
+  metadata.gz: 29f5094d45d2a9014447e79ffd852da117b4040005713420ec186e2adf1491223e3b910f7f0e5c577e8d4b1c431d960b59783646ff9678dd92746c3b394e9268
+  data.tar.gz: 4499d2b12c10327761a83c834db340aa515b44bfdb822f1e0b423596fa1a166cd6a8f401600f5fc2d5934cdec9ecf9edafd0b9cbb3ec4345ddc78f30afd2cd12

data/lib/fbo.rb

@@ -1,11 +1,18 @@
 require 'fbo/version'
 
+module FBO
+  NOTICE_TYPES        = %w( presol combine amdcss mod award ja itb fairopp fstd ) +
+                        %w( srcsgt snote ssale epsupload delete archive unarchive )
+  NOTICE_TAG_NAMES    = NOTICE_TYPES.map { |name| name.upcase }
+  NOTICE_CLOSE_REGEXP = /<\/(#{ NOTICE_TAG_NAMES.join('|') })>$/
+
+end
+
 require 'fbo/file'
 require 'fbo/chunked_file'
+require 'fbo/segmented_file'
 require 'fbo/remote_file'
 require 'fbo/parser'
 require 'fbo/node_extensions'
 require 'fbo/interpreter'
 
-module FBO
-end

data/lib/fbo/chunked_file.rb

@@ -9,9 +9,6 @@ module FBO
 
 
     DEFAULT_CHUNK_SIZE    = 250 * 1024   # 250KB
-    NOTICE_TAG_NAMES      = %w( PRESOL COMBINE AMDCSS MOD AWARD JA ITB FAIROPP FSTD) +
-                            %w( SRCSGT SNOTE SSALE EPSUPLOAD DELETE ARCHIVE UNARCHIVE )
-    NOTICE_CLOSE_REGEXP   = /<\/(#{ NOTICE_TAG_NAMES.join('|') })>$/
 
 
     def initialize(file, chunk_size = DEFAULT_CHUNK_SIZE)
@@ -46,12 +43,12 @@ module FBO
       end while (chunk.bytesize < @chunk_size)
 
       # Add lines up to the end of a notice.
-      if line && line !~ NOTICE_CLOSE_REGEXP
+      if line && line !~ FBO::NOTICE_CLOSE_REGEXP
         begin
           line = gets
           break unless line
           chunk += line
-          break if line =~ NOTICE_CLOSE_REGEXP
+          break if line =~ FBO::NOTICE_CLOSE_REGEXP
         end while (true)
       end
 

data/lib/fbo/interpreter.rb

@@ -1,87 +1,76 @@
 module FBO
   class Interpreter
-    def initialize(tree)
-      @notice_nodes = tree.elements
-    end
-
-    def each_notice
-      each_node(@notice_nodes, &Proc.new)
-    end
-
-    def each_presolicitation
-      each_node(nodes_by_type(FBO::Dump::PresolicitationNode), &Proc.new)
-    end
-
-    def each_combined_solicitation
-      each_node(nodes_by_type(FBO::Dump::CombinedSolicitationNode), &Proc.new)
-    end
-
-    def each_amendment
-      each_node(nodes_by_type(FBO::Dump::AmendmentNode), &Proc.new)
-    end
-
-    def each_modification
-      each_node(nodes_by_type(FBO::Dump::ModificationNode), &Proc.new)
-    end
-
-    def each_award
-      each_node(nodes_by_type(FBO::Dump::AwardNode), &Proc.new)
-    end
-
-    def each_justification_and_approval
-      each_node(nodes_by_type(FBO::Dump::JustificationAndApprovalNode), &Proc.new)
-    end
-
-    def each_intent_to_bundle
-      each_node(nodes_by_type(FBO::Dump::IntentToBundleNode), &Proc.new)
-    end
-
-    def each_fair_opportunity
-      each_node(nodes_by_type(FBO::Dump::FairOpportunityNode), &Proc.new)
-    end
-
-    def each_sources_sought
-      each_node(nodes_by_type(FBO::Dump::SourcesSoughtNode), &Proc.new)
-    end
-
-    def each_foreign_standard
-      each_node(nodes_by_type(FBO::Dump::ForeignStandardNode), &Proc.new)
-    end
-
-    def each_special_notice
-      each_node(nodes_by_type(FBO::Dump::SpecialNoticeNode), &Proc.new)
+    class << self
+      def notice_enumeration(name, type)
+        define_method(name) do |&block|
+          selected = notice_nodes(type)
+          return unless selected
+          selected.each do |node|
+            block.call node.to_hash.merge({ type: node.type })
+          end
+        end
+      end
     end
 
-    def each_sale_of_surplus
-      each_node(nodes_by_type(FBO::Dump::SaleOfSurplusNode), &Proc.new)
+    notice_enumeration :each_notice, :all
+    notice_enumeration :each_presolicitation, :presol
+    notice_enumeration :each_combined_solicitation, :combine
+    notice_enumeration :each_amendment, :amdcss
+    notice_enumeration :each_modification, :mod
+    notice_enumeration :each_award, :award
+    notice_enumeration :each_justification_and_approval, :ja
+    notice_enumeration :each_intent_to_bundle, :itb
+    notice_enumeration :each_fair_opportunity, :fairopp
+    notice_enumeration :each_sources_sought, :srcsgt
+    notice_enumeration :each_foreign_standard, :fstd
+    notice_enumeration :each_special_notice, :snote
+    notice_enumeration :each_sale_of_surplus, :ssale
+    notice_enumeration :each_document_upload, :epsupload
+    notice_enumeration :each_document_delete, :delete
+    notice_enumeration :each_document_archival, :archive
+    notice_enumeration :each_document_unarchival, :unarchive
+
+    def initialize(file)
+      @file = file
     end
 
-    def each_document_upload
-      each_node(nodes_by_type(FBO::Dump::DocumentUploadNode), &Proc.new)
-    end
+    private
 
-    def each_document_delete
-      each_node(nodes_by_type(FBO::Dump::DocumentDeletingNode), &Proc.new)
-    end
+    def notice_nodes(type = :all)
+      if @notice_nodes.nil?
+        @notice_nodes = {}
+      end
 
-    def each_document_archival
-      each_node(nodes_by_type(FBO::Dump::DocumentArchivalNode), &Proc.new)
+      if @file.is_a?(FBO::SegmentedFile) && type != :all
+        @notice_nodes[type] = parse_segment(type)
+      else
+        @notice_nodes[:all] = parse_file
+        if type == :all
+          @notice_nodes[:all]
+        else
+          @notice_nodes[type] = @notice_nodes[:all].select { |n| n.type == type }
+        end
+      end
     end
 
-    def each_document_unarchival
-      each_node(nodes_by_type(FBO::Dump::DocumentUnarchivalNode), &Proc.new)
+    def parse_file
+      tree = FBO::Parser.new(@file).parse
+      tree.elements
     end
 
-    private
-
-    def nodes_by_type(type)
-      @notice_nodes.select { |n| n.is_a? type }
-    end
+    def parse_segment(type = :all)
+      return unless @file.is_a? FBO::SegmentedFile
 
-    def each_node(nodes)
-      nodes.each do |node|
-        yield node.to_hash.merge({ type: node.type })
+      content = ""
+      if type == :all
+        content = @file.contents
+      else
+        content = @file.contents_for_type(type)
       end
+      return unless content
+
+      tree = FBO::Parser.new.parse(content)
+      tree.elements
     end
   end
 end

data/lib/fbo/node_extensions.rb

@@ -14,84 +14,88 @@ module FBO
         body_node = elements.first
         Hash[ body_node.elements.map { |e| [ e.to_sym, e.value ] } ]
       end
+
+      def type
+        self.class.type
+      end
     end
     class PresolicitationNode < NoticeNode
-      def type
+      def self.type
         :presol
       end
     end
     class CombinedSolicitationNode < NoticeNode
-      def type
+      def self.type
         :combine
       end
     end
     class AmendmentNode < NoticeNode
-      def type
+      def self.type
         :amdcss
       end
     end
     class ModificationNode < NoticeNode
-      def type
+      def self.type
         :mod
       end
     end
     class AwardNode < NoticeNode
-      def type
+      def self.type
         :award
       end
     end
     class JustificationAndApprovalNode < NoticeNode
-      def type
+      def self.type
         :ja
       end
     end
     class IntentToBundleNode < NoticeNode
-      def type
+      def self.type
         :itb
       end
     end
     class FairOpportunityNode < NoticeNode
-      def type
+      def self.type
         :fairopp
       end
     end
     class SourcesSoughtNode < NoticeNode
-      def type
+      def self.type
         :srcsgt
       end
     end
     class ForeignStandardNode < NoticeNode
-      def type
+      def self.type
         :fstd
       end
     end
     class SpecialNoticeNode < NoticeNode
-      def type
+      def self.type
         :snote
       end
     end
     class SaleOfSurplusNode < NoticeNode
-      def type
+      def self.type
         :ssale
       end
     end
     class DocumentUploadNode < NoticeNode
-      def type
+      def self.type
         :epsupload
       end
     end
     class DocumentDeletingNode < NoticeNode
-      def type
+      def self.type
         :delete
       end
     end
     class DocumentArchivalNode < NoticeNode
-      def type
+      def self.type
         :archive
       end
     end
     class DocumentUnarchivalNode < NoticeNode
-      def type
+      def self.type
         :unarchive
       end
     end

data/lib/fbo/parser.rb

@@ -6,7 +6,7 @@ Treetop.load(::File.join(base_path, './dump.treetop'))
 
 module FBO
   class Parser
-    def initialize(file)
+    def initialize(file = nil)
       @file = file
     end
 

data/lib/fbo/segmented_file.rb

@@ -0,0 +1,64 @@
+require 'forwardable'
+
+module FBO
+  class SegmentedFile
+    extend Forwardable
+
+    attr_reader     :file
+    def_delegators  :@file, :readline, :read, :eof?
+
+
+    def initialize(file)
+      @file = file
+    end
+
+    FBO::NOTICE_TYPES.each do |type|
+      # Define RegExp constants for partitioning the file
+      regexp_constant_name = "#{ type.upcase }_REGEXP"
+      regexp               = /<#{ type.upcase }>.*<\/#{ type.upcase }>/m
+      FBO::SegmentedFile.const_set(regexp_constant_name, regexp)
+
+      # Define methods for accessing content segments
+      method_name   = "#{ type }_contents"
+      variable_name = "@#{ type }_contents"
+      define_method(method_name) do
+        if !instance_variable_defined?(variable_name)
+          regexp = FBO::SegmentedFile.const_get(regexp_constant_name)
+          match_data = @file.contents.match(regexp)
+          matched_text = cleanup_data( match_data ? match_data.to_s : nil)
+          instance_variable_set(variable_name, matched_text)
+       end
+        instance_variable_get(variable_name)
+      end
+    end
+
+    def contents
+      if @contents.nil?
+        @contents = [ presol_contents, combine_contents, amdcss_contents,
+                      mod_contents, award_contents, ja_contents, itb_contents,
+                      fairopp_contents, srcsgt_contents, fstd_contents,
+                      snote_contents, ssale_contents, epsupload_contents,
+                      delete_contents, archive_contents, unarchive_contents ]
+        @contents.compact!
+        #@contents = @file.contents
+      end
+      @contents
+    end
+
+    def contents_for_type(type)
+      return unless type
+      method_name = "#{ type }_contents"
+      self.respond_to?(method_name) ? self.send(method_name) : nil 
+    end
+
+    private
+ 
+    def cleanup_data(data)
+      return if data.nil?
+      data.encode('UTF-16le', :invalid => :replace, :replace => '')
+          .encode('UTF-8')
+          .gsub(/\r\n/, "\n")
+          .gsub(/^M/, "")
+    end
+  end
+end

data/lib/fbo/version.rb

@@ -1,3 +1,3 @@
 module FBO
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/spec/fbo/chunked_file_spec.rb

@@ -6,16 +6,16 @@ describe FBO::ChunkedFile do
   let(:chunk_size) { 50 * 1024 }   # 50KB
   subject          { FBO::ChunkedFile.new(file, chunk_size) }
 
-  describe "#contents" do
-    it "returns an Array of String" do
+  describe '#contents' do
+    it 'returns an Array of String' do
       subject.contents.must_be_instance_of Array
       subject.contents.each { |x| x.must_be_instance_of String }
     end
 
-    it "returns chunks that include whole notices" do
+    it 'returns chunks that include whole notices' do
       contents = subject.contents
       contents.each do |chunk|
-        chunk.must_match FBO::ChunkedFile::NOTICE_CLOSE_REGEXP
+        chunk.must_match FBO::NOTICE_CLOSE_REGEXP
       end
     end
   end

data/spec/fbo/interpreter_spec.rb

@@ -3,71 +3,103 @@ require 'spec_helper'
 describe FBO::Interpreter do
   let(:filename)  { File.join(File.dirname(__FILE__), '..', 'fixtures', 'FBOFeed20130331') }
   let(:file)      { FBO::File.new(filename) }
-  let(:tree)      { FBO::Parser.new(file).parse }
   let(:actor)     { stub('Act on ALL the Notices', process: nil) }
 
-  subject         { FBO::Interpreter.new(tree) }
+  context 'with input FBO::File' do
+    subject  { FBO::Interpreter.new(file) }
 
-  describe '#each_notice' do
-    it 'acts on each notice' do
-      actor.expects(:process).with(instance_of(Hash)).times(8)
-      subject.each_notice { |n| actor.process(n) }
+    describe '#each_notice' do
+      it 'acts on each notice' do
+        actor.expects(:process).with(instance_of(Hash)).times(8)
+        subject.each_notice { |n| actor.process(n) }
+      end
     end
-  end
 
-  describe '#each_presolicitation' do
-    it 'acts on each PRESOL notice' do
-      actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
-      subject.each_presolicitation { |n| actor.process(n) }
+    describe '#each_presolicitation' do
+      it 'acts on each PRESOL notice' do
+        actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
+        subject.each_presolicitation { |n| actor.process(n) }
+      end
     end
-  end
 
-  describe '#each_sources_sought' do
-    it 'acts on each SRCSGT notice' do
-      actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
-      subject.each_sources_sought { |n| actor.process(n) }
+    describe '#each_sources_sought' do
+      it 'acts on each SRCSGT notice' do
+        actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
+        subject.each_sources_sought { |n| actor.process(n) }
+      end
     end
-  end
 
-  describe '#each_combined_solicitation' do
-    it 'acts on each COMBINE notice' do
-      actor.expects(:process).with(has_entry(solicitation_number: 'SPM7L213TA359'))
-      subject.each_combined_solicitation { |n| actor.process(n) }
+    describe '#each_special_notice' do
+      it 'does nothing (no SNOTE notices)' do
+        actor.expects(:process).never
+        subject.each_special_notice { |n| actor.process(n) }
+      end
     end
   end
 
-  describe '#each_amendment' do
-    it 'acts on each AMDCSS notice' do
-      actor.expects(:process).with(has_entry(solicitation_number: 'SEG30013C0020'))
-      subject.each_amendment { |n| actor.process(n) }
+  context 'with input FBO::ChunkedFile' do
+    let(:chunked)   { FBO::ChunkedFile.new(file) }
+    subject         { FBO::Interpreter.new(chunked) }
+
+    describe '#each_notice' do
+      it 'acts on each notice' do
+        actor.expects(:process).with(instance_of(Hash)).times(8)
+        subject.each_notice { |n| actor.process(n) }
+      end
     end
-  end
 
-  describe '#each_modification' do
-    it 'acts on each MOD notice' do
-      actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0497'))
-      subject.each_modification { |n| actor.process(n) }
+    describe '#each_presolicitation' do
+      it 'acts on each PRESOL notice' do
+        actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
+        subject.each_presolicitation { |n| actor.process(n) }
+      end
     end
-  end
 
-  describe '#each_award' do
-    it 'acts on each AWARD notice' do
-      actor.expects(:process).with(has_entry(solicitation_number: 'SPM4A612R0732'))
-      subject.each_award { |n| actor.process(n) }
+    describe '#each_sources_sought' do
+      it 'acts on each SRCSGT notice' do
+        actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
+        subject.each_sources_sought { |n| actor.process(n) }
+      end
     end
-  end
 
-  describe '#each_document_archival' do
-    it 'acts on each ARCHIVE notice' do
-      actor.expects(:process).with(has_entry(solicitation_number: 'FA4690-13-B-0001'))
-      subject.each_document_archival { |n| actor.process(n) }
+    describe '#each_special_notice' do
+      it 'does nothing (no SNOTE notices)' do
+        actor.expects(:process).never
+        subject.each_special_notice { |n| actor.process(n) }
+      end
     end
   end
 
-  describe '#each_document_unarchival' do
-    it 'acts on each UNARCHIVE notice' do
-      actor.expects(:process).with(has_entry(solicitation_number: 'SEG30013C0020'))
-      subject.each_document_unarchival { |n| actor.process(n) }
+  context 'with input FBO::SegmentedFile' do
+    let(:segmented) { FBO::SegmentedFile.new(file) }
+    subject         { FBO::Interpreter.new(segmented) }
+
+    describe '#each_notice' do
+      it 'acts on each notice' do
+        actor.expects(:process).with(instance_of(Hash)).times(8)
+        subject.each_notice { |n| actor.process(n) }
+      end
+    end
+
+    describe '#each_presolicitation' do
+      it 'acts on each PRESOL notice' do
+        actor.expects(:process).with(has_entry(solicitation_number: 'SPE4A713R0795'))
+        subject.each_presolicitation { |n| actor.process(n) }
+      end
+    end
+
+    describe '#each_sources_sought' do
+      it 'acts on each SRCSGT notice' do
+        actor.expects(:process).with(has_entry(solicitation_number: 'W5J9JE-13-S-0002'))
+        subject.each_sources_sought { |n| actor.process(n) }
+      end
+    end
+
+    describe '#each_special_notice' do
+      it 'does nothing (no SNOTE notices)' do
+        actor.expects(:process).never
+        subject.each_special_notice { |n| actor.process(n) }
+      end
     end
   end
 end

data/spec/fbo/segmented_file_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+describe FBO::SegmentedFile do
+  let(:filename)   { File.join(File.dirname(__FILE__), '..', 'fixtures', 'FBOFeed20131003') }
+  let(:file)       { FBO::File.new(filename) }
+  subject          { FBO::SegmentedFile.new(file) }
+
+  describe "#contents" do
+    it "returns an Array of String" do
+      subject.contents.must_be_instance_of Array
+      subject.contents.each { |x| x.must_be_instance_of String }
+    end
+
+    it "returns chunks that include whole notices" do
+      contents = subject.contents
+      contents.each do |segment|
+        segment.must_match FBO::NOTICE_CLOSE_REGEXP
+      end
+    end
+  end
+
+  describe '#presol_contents' do
+    let(:presol_contents)  { subject.presol_contents }
+
+    it 'returns a single big String' do
+      presol_contents.must_be_instance_of String
+    end
+
+    it 'must start and end with a PRESOL' do
+      presol_contents.must_match /^<PRESOL>/m
+      presol_contents.must_match /<\/PRESOL>$/m
+    end
+
+    it 'should not contain other notice types' do
+      non_presols = FBO::NOTICE_TAG_NAMES - [ 'PRESOL' ]
+      regexp_constants = non_presols.map { |name| FBO::SegmentedFile.const_get("#{ name }_REGEXP") }
+      regexp_constants.each do |regexp|
+        presol_contents.wont_match regexp
+      end
+    end
+  end
+
+  describe '#fstd_contents' do
+    it 'returns nil (no content)' do
+      subject.fstd_contents.must_be_nil
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fbo
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Chris Kottom
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-22 00:00:00.000000000 Z
+date: 2013-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: treetop
@@ -121,12 +121,14 @@ files:
 - lib/fbo/node_extensions.rb
 - lib/fbo/parser.rb
 - lib/fbo/remote_file.rb
+- lib/fbo/segmented_file.rb
 - lib/fbo/version.rb
 - spec/fbo/chunked_file_spec.rb
 - spec/fbo/file_spec.rb
 - spec/fbo/interpreter_spec.rb
 - spec/fbo/parser_spec.rb
 - spec/fbo/remote_file_spec.rb
+- spec/fbo/segmented_file_spec.rb
 - spec/fixtures/.keep
 - spec/fixtures/FBOFeed20130331
 - spec/fixtures/FBOFeed20131003
@@ -161,6 +163,7 @@ test_files:
 - spec/fbo/interpreter_spec.rb
 - spec/fbo/parser_spec.rb
 - spec/fbo/remote_file_spec.rb
+- spec/fbo/segmented_file_spec.rb
 - spec/fixtures/.keep
 - spec/fixtures/FBOFeed20130331
 - spec/fixtures/FBOFeed20131003