fbo 0.0.3 → 0.1.0

data/lib/fbo/file.rb

@@ -5,7 +5,7 @@ module FBO
     extend Forwardable
 
     attr_reader     :file
-    def_delegators  :@file, :open, :readline, :read, :path, :to_path
+    def_delegators  :@file, :readline, :read, :eof?, :gets
 
     class << self
       def filename_for_date(date)
@@ -17,5 +17,21 @@ module FBO
     def initialize(filename)
       @file = ::File.new(filename)
     end
+
+    def contents
+      if @contents.nil?
+        @contents = cleanup_data(@file.read)
+      end
+      @contents
+    end
+
+    private
+
+    def cleanup_data(data)
+      data.encode('UTF-16le', :invalid => :replace, :replace => '')
+          .encode('UTF-8')
+          .gsub(/\r\n/, "\n")
+          .gsub(/^M/, "")
+    end
   end
 end

data/lib/fbo/interpreter.rb

@@ -0,0 +1,87 @@
+module FBO
+  class Interpreter
+    def initialize(tree)
+      @notice_nodes = tree.elements
+    end
+
+    def each_notice
+      each_node(@notice_nodes, &Proc.new)
+    end
+
+    def each_presolicitation
+      each_node(nodes_by_type(FBO::Dump::PresolicitationNode), &Proc.new)
+    end
+
+    def each_combined_solicitation
+      each_node(nodes_by_type(FBO::Dump::CombinedSolicitationNode), &Proc.new)
+    end
+
+    def each_amendment
+      each_node(nodes_by_type(FBO::Dump::AmendmentNode), &Proc.new)
+    end
+
+    def each_modification
+      each_node(nodes_by_type(FBO::Dump::ModificationNode), &Proc.new)
+    end
+
+    def each_award
+      each_node(nodes_by_type(FBO::Dump::AwardNode), &Proc.new)
+    end
+
+    def each_justification_and_approval
+      each_node(nodes_by_type(FBO::Dump::JustificationAndApprovalNode), &Proc.new)
+    end
+
+    def each_intent_to_bundle
+      each_node(nodes_by_type(FBO::Dump::IntentToBundleNode), &Proc.new)
+    end
+
+    def each_fair_opportunity
+      each_node(nodes_by_type(FBO::Dump::FairOpportunityNode), &Proc.new)
+    end
+
+    def each_sources_sought
+      each_node(nodes_by_type(FBO::Dump::SourcesSoughtNode), &Proc.new)
+    end
+
+    def each_foreign_standard
+      each_node(nodes_by_type(FBO::Dump::ForeignStandardNode), &Proc.new)
+    end
+
+    def each_special_notice
+      each_node(nodes_by_type(FBO::Dump::SpecialNoticeNode), &Proc.new)
+    end
+
+    def each_sale_of_surplus
+      each_node(nodes_by_type(FBO::Dump::SaleOfSurplusNode), &Proc.new)
+    end
+
+    def each_document_upload
+      each_node(nodes_by_type(FBO::Dump::DocumentUploadNode), &Proc.new)
+    end
+
+    def each_document_delete
+      each_node(nodes_by_type(FBO::Dump::DocumentDeletingNode), &Proc.new)
+    end
+
+    def each_document_archival
+      each_node(nodes_by_type(FBO::Dump::DocumentArchivalNode), &Proc.new)
+    end
+
+    def each_document_unarchival
+      each_node(nodes_by_type(FBO::Dump::DocumentUnarchivalNode), &Proc.new)
+    end
+
+    private
+
+    def nodes_by_type(type)
+      @notice_nodes.select { |n| n.is_a? type }
+    end
+
+    def each_node(nodes)
+      nodes.each do |node|
+        yield node.to_hash.merge({ type: node.type })
+      end
+    end
+  end
+end

data/lib/fbo/node_extensions.rb

@@ -0,0 +1,173 @@
+module FBO
+  module Dump
+
+    # Structural node types
+    #
+    class DumpNode < Treetop::Runtime::SyntaxNode; end
+    class BodyNode < Treetop::Runtime::SyntaxNode; end
+    class TagContentNode < Treetop::Runtime::SyntaxNode; end
+
+    # Node types corresponding to 
+    #
+    class NoticeNode < Treetop::Runtime::SyntaxNode
+      def to_hash
+        body_node = elements.first
+        Hash[ body_node.elements.map { |e| [ e.to_sym, e.value ] } ]
+      end
+    end
+    class PresolicitationNode < NoticeNode
+      def type
+        :presol
+      end
+    end
+    class CombinedSolicitationNode < NoticeNode
+      def type
+        :combine
+      end
+    end
+    class AmendmentNode < NoticeNode
+      def type
+        :amdcss
+      end
+    end
+    class ModificationNode < NoticeNode
+      def type
+        :mod
+      end
+    end
+    class AwardNode < NoticeNode
+      def type
+        :award
+      end
+    end
+    class JustificationAndApprovalNode < NoticeNode
+      def type
+        :ja
+      end
+    end
+    class IntentToBundleNode < NoticeNode
+      def type
+        :itb
+      end
+    end
+    class FairOpportunityNode < NoticeNode
+      def type
+        :fairopp
+      end
+    end
+    class SourcesSoughtNode < NoticeNode
+      def type
+        :srcsgt
+      end
+    end
+    class ForeignStandardNode < NoticeNode
+      def type
+        :fstd
+      end
+    end
+    class SpecialNoticeNode < NoticeNode
+      def type
+        :snote
+      end
+    end
+    class SaleOfSurplusNode < NoticeNode
+      def type
+        :ssale
+      end
+    end
+    class DocumentUploadNode < NoticeNode
+      def type
+        :epsupload
+      end
+    end
+    class DocumentDeletingNode < NoticeNode
+      def type
+        :delete
+      end
+    end
+    class DocumentArchivalNode < NoticeNode
+      def type
+        :archive
+      end
+    end
+    class DocumentUnarchivalNode < NoticeNode
+      def type
+        :unarchive
+      end
+    end
+
+    # Simple property nodes have a name/symbol and a value
+    #
+    class SimplePropertyNode < Treetop::Runtime::SyntaxNode
+      def to_sym
+        class_name = self.class.name
+        base_name = class_name.split('::').last
+        base_name.sub!(/Node$/, '')
+        base_name.gsub!(/([^A-Z])([A-Z])/, '\1_\2')
+        base_name.tr!('A-Z', 'a-z')
+        base_name.to_sym
+      end
+
+      def value
+        elements[0].text_value
+      end
+
+      def to_hash
+        { self.to_sym => self.value }
+      end
+    end
+    class DateNode < SimplePropertyNode; end
+    class YearNode < SimplePropertyNode; end
+    class AgencyNode < SimplePropertyNode; end
+    class OfficeNode < SimplePropertyNode; end
+    class LocationNode < SimplePropertyNode; end
+    class ZipNode < SimplePropertyNode; end
+    class ClassificationCodeNode < SimplePropertyNode; end
+    class NaicsCodeNode < SimplePropertyNode; end
+    class OfficeAddressNode < SimplePropertyNode; end
+    class SubjectNode < SimplePropertyNode; end
+    class SolicitationNumberNode < SimplePropertyNode; end
+    class NoticeTypeNode < SimplePropertyNode; end
+    class ResponseDateNode < SimplePropertyNode; end
+    class ArchiveDateNode < SimplePropertyNode; end
+    class ContactNode < SimplePropertyNode; end
+    class DescriptionNode < SimplePropertyNode; end
+    class UrlNode < SimplePropertyNode; end
+    class EmailAddressNode < SimplePropertyNode; end
+    class SetAsideNode < SimplePropertyNode; end
+    class PopAddressNode < SimplePropertyNode; end
+    class PopZipNode < SimplePropertyNode; end
+    class PopCountryNode < SimplePropertyNode; end
+    class AwardNumberNode < SimplePropertyNode; end
+    class AwardAmountNode < SimplePropertyNode; end
+    class LineNumberNode < SimplePropertyNode; end
+    class AwardDateNode < SimplePropertyNode; end
+    class AwardeeNode < SimplePropertyNode; end
+    class AwardeeDunsNode < SimplePropertyNode; end
+    class CorrectionNode < SimplePropertyNode; end
+    class FileNode < SimplePropertyNode; end
+    class MimeTypeNode < SimplePropertyNode; end
+    class StatutoryAuthorityNode < SimplePropertyNode; end
+    class ModificationNumberNode < SimplePropertyNode; end
+    class DeliveryOrderNumberNode < SimplePropertyNode; end
+    class JustificationAuthorityNode < SimplePropertyNode; end
+    class CBACNode < SimplePropertyNode; end
+    class PasswordNode < SimplePropertyNode; end
+    class ProjectIDNode < SimplePropertyNode; end
+    class UploadTypeNode < SimplePropertyNode; end
+    class CorrectionNode < SimplePropertyNode; end
+
+    # Complex properties may contain other simple properties
+    #
+    class ComplexPropertyNode < SimplePropertyNode
+      def value
+        value_hash = {}
+        elements.each { |e| value_hash.merge!(e.to_hash) }
+        value_hash
+      end
+    end
+    class LinkNode < ComplexPropertyNode; end
+    class EmailNode < ComplexPropertyNode; end
+    class FileListNode < ComplexPropertyNode; end
+  end
+end

data/lib/fbo/parser.rb

@@ -1,66 +1,63 @@
-require "strscan"
+require 'treetop'
+require 'fbo/node_extensions'
+
+base_path = ::File.expand_path(::File.dirname(__FILE__))
+Treetop.load(::File.join(base_path, './dump.treetop'))
 
 module FBO
   class Parser
-    TAG_PATTERN = /<\w+>/m
-    PATTERN = /<([A-Z]+)>.*<\/\1>/m
-
-    def initialize(file, tag_pattern: TAG_PATTERN)
+    def initialize(file)
       @file = file
-      @tag_pattern = tag_pattern
     end
 
-    def notices
-      if @notices.nil?
-        parse_notices(split_file_contents)
+    def parse(data = nil)
+      data ||= @file.contents
+
+      puts "Class = #{ data.class.name }"
+      if data.respond_to? :each
+        @tree = parse_collection(data)
+      else
+        @tree = parse_string(data)
       end
-      @notices
+      @tree
     end
 
     private
 
-    def split_file_contents
-      contents = @file.read
-      contents = cleanup_contents(contents)
-      scanner = StringScanner.new(contents)
-      text_notices = []
-
-      while !scanner.eos?
-        initial_tag = scanner.check_until(@tag_pattern)
-        if initial_tag
-          initial_tag.strip!
-        else
-          break
-        end
-
-        text_notice = scanner.scan_until(/#{ closing_tag_for(initial_tag) }\s*/)
-        if text_notice
-          text_notices << text_notice.strip
-        else
-          break
-        end
-      end
-      text_notices
+    def parser
+      @parser ||= FBO::DumpParser.new
     end
 
-    def parse_notices(text_notices)
-      @notices = []
-      text_notices.each do |text|
-        handler = FBO::Parser::HandlerSelector.select(text)
-        @notices << handler.parse(text)
+    def parse_string(data)
+      tree = parser.parse(data)
+      if tree.nil?
+        line = parser.failure_line
+        column = parser.failure_column
+        reason = parser.failure_reason
+        raise Exception, "Parse error at line #{ line }, column #{ column }: #{ reason }"
       end
+      clean_tree(tree)
     end
 
-    def cleanup_contents(contents)
-      contents
-        .encode('UTF-16le', :invalid => :replace, :replace => '')
-        .encode('UTF-8')
-        .gsub(/\r\n/, "\n")
-        .gsub(/^M/, "")
+    def parse_collection(data)
+      super_tree = nil
+      data.each do |string|
+        tree = parse_string(string)
+        if super_tree
+          super_tree = FBO::Dump::DumpNode.new(super_tree.input, super_tree.interval,
+                                               super_tree.elements + tree.elements)
+        else
+          super_tree = tree
+        end
+      end
+      super_tree
     end
-
-    def closing_tag_for(tag)
-      tag.sub(/</, "</")
+    
+    def clean_tree(node)
+      return if node.elements.nil?
+      node.elements.delete_if { |node| node.class.name == "Treetop::Runtime::SyntaxNode" }
+      node.elements.each { |node| clean_tree(node) }
+      node
     end
   end
 end

data/lib/fbo/remote_file.rb

@@ -1,10 +1,10 @@
-require "date"
-require "net/ftp"
+require 'date'
+require 'net/ftp'
 
 module FBO
   class RemoteFile < File
-    FTP_SERVER = "ftp.fbo.gov"
-    TMP_DIR    = "/tmp/fbo"
+    FTP_SERVER = 'ftp.fbo.gov'
+    TMP_DIR    = '/tmp/fbo'
 
     class << self
       def for_date(date, options = {})

data/lib/fbo/version.rb

@@ -1,3 +1,3 @@
 module FBO
-  VERSION = "0.0.3"
+  VERSION = "0.1.0"
 end

data/spec/fbo/chunked_file_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+
+describe FBO::ChunkedFile do
+  let(:filename)   { File.join(File.dirname(__FILE__), '..', 'fixtures', 'FBOFeed20131003') }
+  let(:file)       { FBO::File.new(filename) }
+  let(:chunk_size) { 50 * 1024 }   # 50KB
+  subject          { FBO::ChunkedFile.new(file, chunk_size) }
+
+  describe "#contents" do
+    it "returns an Array of String" do
+      subject.contents.must_be_instance_of Array
+      subject.contents.each { |x| x.must_be_instance_of String }
+    end
+
+    it "returns chunks that include whole notices" do
+      contents = subject.contents
+      contents.each do |chunk|
+        chunk.must_match FBO::ChunkedFile::NOTICE_CLOSE_REGEXP
+      end
+    end
+  end
+end