quandl_operation 0.1.18 → 0.1.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87874fab0a0c7090d8a8d659e7492ff91fe7e98b
-  data.tar.gz: 0ea2c0c04aa165ba162eeeea6473da15dc6565c6
+  metadata.gz: 21f4883192fd7504a13a091e6215b64d839359f8
+  data.tar.gz: 542095e6c669ebb97ac8c34543e40011a95847d2
 SHA512:
-  metadata.gz: 597a729a3f03e538c4876bfa0ab3fdbb56fe309bae13f3348c8cd1312c3548ca77a051981aaca5858b9bcf566114768792195457aff332e251258fceb4baf9a4
-  data.tar.gz: f7ccf8150ee7f1ee3ad1161f6f3c18b9cb40276d093bb4d117b9db059081d4052454611f4d049404266c449912bb913a4818f3b9ece52b663cfa29a2137ce572
+  metadata.gz: b3163d4f4952ddded01c8d8bbeab2c2680fadd9ef451728397ce90e0e49d71734a82153ca0b8e72e977ddf2536fe9969620ca9ba4092aae2ed46023220f8e331
+  data.tar.gz: 10e238077f7a8e08010ccc28561709451cd319b4cbd7e2a871c208b959350e0f92227de42d87d2dba5c2e6d11d469bd32a57f344c7adf04b014c5ef9f385d7bd

data/lib/quandl/operation/qdformat.rb

@@ -1,126 +1,24 @@
+require 'csv'
+require 'yaml'
+
+require 'quandl/operation/qdformat/dump'
+require 'quandl/operation/qdformat/load'
+require 'quandl/operation/qdformat/node'
+
 class Quandl::Operation::QDFormat
-  
-  attr_accessor :source_code, :code, :name, :description, :data, :headers
-  
   class << self
     
-    def read(path)
-      parse(File.read(path).strip)
-    end
-  
-    def parse(output)
-      datasets = []
-      attributes = {}
-      # for each line
-      output.each_line do |line|
-        # strip whitespace
-        line = line.strip.rstrip
-        
-        # skip blank
-        if line.blank?
-          next
-          
-        # - denotes new dataset
-        elsif line[0] == '-'
-          # append current dataset
-          datasets << attributes unless attributes.blank?
-          # reset attributes for next dataset
-          attributes = {}
-        
-        # without the data key we're still adding metadata
-        elsif !attributes.has_key?(:data)
-          # for each rule
-          matched = false
-          rules.each do |name, rule|
-            # test line
-            m = line.match(rule)
-            # if the rule matches
-            if !matched && m.present?
-              matched = true
-              case name
-              when :full_code then attributes[:full_code] = line
-              when :attribute then attributes[m[1].to_sym] = m[2]
-              end
-            end
-          end
-          # if no match was found for this line it must be data
-          attributes[:data] = line if !matched
-        # otherwise we're adding data
-        else
-          attributes[:data] += "\n" + line
-        end
-      end
-      datasets << attributes unless attributes.blank?
-      datasets.collect{|attrs| self.new(attrs) }
+    def load(input)
+      Quandl::Operation::QDFormat::Load.from_string(input)
     end
     
-    def rules
-      {
-        full_code:      /^([A-Z0-9_]+)\/([A-Z0-9_]+)$/,
-        attribute:      /^([a-z0-9_]+): (.+)/
-      }
-    end
-  
-  end
-  
-  def initialize(attrs)
-    assign_attributes(attrs)
-  end
-  
-  def assign_attributes(attrs)
-    attrs.each do |key, value|
-      self.send("#{key}=", value) if self.respond_to?(key)
+    def load_file(file_path)
+      Quandl::Operation::QDFormat::Load.from_file(input)
     end
-  end
-  
-  def attributes
-    { name: name, source_code: source_code, code: code, description: description, column_names: headers, data: data }
-  end
-  
-  def inspect
-    "<##{self.class.name}" + 
-    [:full_code, :name, :description, :headers].inject({}){|m,k| m[k] = self.send(k); m }.to_s +
-    " >"
-  end
-  
-  def headers=(names)
-    names = names.split(",").collect(&:strip) if names.is_a?(String)
-    @headers = Array(names).flatten
-  end
-  def headers_as_qdf
-    headers.join(', ') if headers.is_a?(Array)
-  end
-  
-  def full_code=(value)
-    value = value.split('/')
-    self.source_code = value[0]
-    self.code = value[1]
-  end
-  
-  def full_code
-    [source_code, code].join('/')
-  end
-  
-  def data_as_qdf
-    o = data
-    o = o.to_a if o.respond_to?(:to_a)
-    o = o.collect(&:to_csv).join if o.respond_to?(:to_csv) && o.first.is_a?(Array)
-    o = o.to_csv if o.respond_to?(:to_csv)
-    o
-  end
-  
-  def data=(rows)
-    @data = rows
-  end
-  
-  def to_qdf
-    output = [full_code]
-    [:name, :description].each do |attr_name|
-      output << "#{attr_name}: #{self.send(attr_name)}" if self.send(attr_name).present?
+    
+    def dump(nodes)
+      Quandl::Operation::QDFormat::Dump.nodes(nodes)
     end
-    output << "headers: #{headers_as_qdf}" if headers_as_qdf.present?
-    output << data_as_qdf
-    output.join("\n")
+    
   end
-  
 end

data/lib/quandl/operation/qdformat/dump.rb

@@ -0,0 +1,49 @@
+module Quandl
+module Operation
+class QDFormat
+
+class Dump
+
+  class << self
+
+    def nodes(*args)
+      Array(args).flatten.collect{|r| node(r) }.join("\n")
+    end
+
+    def node(node)
+      self.new(node).to_qdf
+    end
+
+  end
+
+  attr_accessor :node
+
+  def initialize(r)
+    self.node = r
+  end
+
+  def to_qdf
+    [attributes, column_names, data].compact.join
+  end
+
+  def attributes
+    [:source_code, :code, :name, :description].
+    inject({}){|m,k| m[k.to_s] = node.send(k) unless node.send(k).blank?; m }.
+    to_yaml[4..-1]
+  end
+
+  def data
+    data = node.data.is_a?(Array) ? node.data.collect(&:to_csv).join : node.data
+    data = data.to_csv if data.respond_to?(:to_csv)
+    data
+  end
+
+  def column_names
+    node.column_names.to_csv if node.column_names.present?
+  end
+
+end
+
+end
+end
+end

data/lib/quandl/operation/qdformat/load.rb

@@ -0,0 +1,53 @@
+class Quandl::Operation::QDFormat::Load
+  
+  class << self
+  
+    def from_file(path)
+      from_string(File.read(path).strip)
+    end
+  
+    def from_string(input)
+      nodes = []
+      input.each_line do |line|
+        # strip whitespace
+        line = line.strip.rstrip
+        # ignore comments and blank lines
+        next if line[0] == '#' || line.blank?
+        # code_format denotes the start of a new node
+        nodes << { attributes: '', data: '' } if line =~ code_format
+        # attribute_format denotes an attribute
+        if line =~ attribute_format
+          # add the attribute to attributes
+          nodes[-1][:attributes] += "#{line}\n"
+          # otherwise it must be data
+        else
+          nodes[-1][:data] += "#{line}\n"
+        end
+      end
+      # append the current node
+      nodes = parse_nodes(nodes)
+      nodes
+    end
+    
+    
+    protected
+    
+    def parse_nodes(nodes)
+      nodes.collect do |node|
+        node[:attributes] = YAML.load( node[:attributes] )
+        node[:attributes][:data] = CSV.parse(node[:data])
+        Quandl::Operation::QDFormat::Node.new(node[:attributes])
+      end
+    end
+  
+    def code_format
+      /^code: (.+)/
+    end
+  
+    def attribute_format
+      /^([a-z0-9_]+): (.+)/
+    end
+  
+  end
+
+end

data/lib/quandl/operation/qdformat/node.rb

@@ -0,0 +1,49 @@
+class Quandl::Operation::QDFormat::Node
+  
+  ATTRIBUTES = :source_code, :code, :name, :description, :column_names
+  attr_accessor *ATTRIBUTES
+
+  attr_accessor :data
+
+  def initialize(attrs)
+    assign_attributes(attrs)
+  end
+
+  def assign_attributes(attrs)
+    attrs.each do |key, value|
+      self.send("#{key}=", value) if self.respond_to?(key)
+    end
+  end
+
+  def attributes
+    ATTRIBUTES.inject({}){|m,k| m[k] = self.send(k); m }
+  end
+
+  def inspect
+    "<##{self.class.name} #{attributes.to_s} >"
+  end
+
+  def full_code=(value)
+    value = value.split('/')
+    self.source_code = value[0]
+    self.code = value[1]
+  end
+
+  def full_code
+    [source_code, code].join('/')
+  end
+
+  def data=(rows)
+    self.column_names = rows.shift unless rows.first.collect{|r| r.to_s.numeric? }.include?(true)
+    @data = rows
+  end
+
+  def column_names=(names)
+    @column_names = Array(names).flatten.collect{|n| n.strip.rstrip }
+  end
+
+  def to_qdf
+    Quandl::Operation::QDFormat::Dump.node(self)
+  end
+  
+end

data/lib/quandl/operation/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Operation
-  VERSION = "0.1.18"
+  VERSION = "0.1.19"
 end
 end

data/spec/lib/quandl/operation/qdformat_spec.rb

@@ -4,61 +4,71 @@ require 'spec_helper'
 
 describe Quandl::Operation::QDFormat do
   
-  let(:full_code){ "SOURCE_CODE/DATASET_CODE" }
-  let(:name){ 'name: Test dataset name' }
-  let(:description){ 'description: Dataset description' }
-  let(:headers){ 'headers: Date, value, high, low' }
-  let(:data){ "2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353\n2013-11-19,10.039388096885814,,14.09718770934256\n2013-11-18,10.03702588792184,11.040801329322205,14.148600982164867\n2013-11-16,10.019903902583621,10.99988541851354,14.186053161235304\n2013-11-15,9.98453953586862,10.922239168500502,\n2013-11-14,10.004508614940358,10.894612328250766,\n2013-11-13,,10.877309120435308,14.187437960548612\n2013-11-12,,10.838918617657301,14.22499294338536\n2013-11-11,9.965116185761039,10.827442115591547,14.178970907392053\n2013-11-09,9.881291973139637,10.924889094631869" }
-  
-  let(:output){
+  let(:qdf_dataset){
     %Q{
-      ---
-      #{full_code}
-      #{name}
-      #{description}
-      #{headers}
-      #{data}
-      -
-      - I am a comment.
-      -
-      #{full_code}
-      #{name}
-      #{description}
-      #{headers}
-      #{data}
-      -----
+      # YAML metadata
+      code:           DATASET_CODE
+      source_code:    SOURCE_CODE
+      name:           Test Dataset Name 1
+      description:    Here is a description with multiple lines.\\n This is the second line.
+      # CSV data
+      Date, Value, High, Low
+      2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
+      2013-11-19,10.039388096885814,,14.09718770934256
+      
+      # Second dataset
+      code:           DATASET_CODE_2
+      source_code:    SOURCE_CODE_2
+      name:           Test Dataset Name 2
+      description:    Here is a description with multiple lines.
+      # CSV data
+      Date, Value, High, Low
+      2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
+      2013-11-19,10.039388096885814,,14.09718770934256
+      2013-11-18,11.039388096885814,,15.09718770934256
     }
   }
-  let(:collection){ Quandl::Operation::QDFormat.parse(output) }
   
-  describe ".parse" do
+  let(:collection){ Quandl::Operation::QDFormat.load(qdf_dataset) }
+  
+  describe ".load" do
     
     subject{ collection }
-      
+    
     its(:count){ should eq 2 }
     
     describe "#first" do
       subject{ collection.first }
 
-      it{ should be_a Quandl::Operation::QDFormat }
+      it{ should be_a Quandl::Operation::QDFormat::Node }
       its(:source_code){ should eq 'SOURCE_CODE' }
       its(:code){ should eq 'DATASET_CODE' }
-      its(:name){ should eq 'Test dataset name' }
-      its(:description){ should eq 'Dataset description' }
-      its(:headers){ should eq ['Date', 'value', 'high', 'low'] }
-      its(:data){ should eq data }
+      its(:name){ should eq 'Test Dataset Name 1' }
+      its(:description){ should eq 'Here is a description with multiple lines.\n This is the second line.' }
+      its(:column_names){ should eq ['Date', 'Value', 'High', 'Low'] }
+      its(:data){ should eq [
+        ["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"], 
+        ["2013-11-19", "10.039388096885814", nil, "14.09718770934256"]] }
     end
   end
   
   describe "#to_qdf" do
     subject{ collection.first }
-    its(:to_qdf){ should eq [full_code, name, description, headers, data].join("\n")}
+    
+    its(:to_qdf){ should eq %Q{source_code: SOURCE_CODE
+code: DATASET_CODE
+name: Test Dataset Name 1
+description: Here is a description with multiple lines.\\n This is the second line.
+Date,Value,High,Low
+2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
+2013-11-19,10.039388096885814,,14.09718770934256
+}}
     
     context "data Array" do
       let(:data){ [['2013-11-20',9.94,11.2],['2013-11-19',9.94,11.2],['2013-11-18',9.94,11.2]] }
-      subject{ Quandl::Operation::QDFormat.new( full_code: "TEST/OIL", data: data ) }
+      subject{ Quandl::Operation::QDFormat::Node.new( full_code: "TEST/OIL", data: data ) }
       
-      its(:to_qdf){ should eq ["TEST/OIL", data.collect(&:to_csv).join].join("\n") }
+      its(:to_qdf){ should eq "source_code: TEST\ncode: OIL\n2013-11-20,9.94,11.2\n2013-11-19,9.94,11.2\n2013-11-18,9.94,11.2\n" }
       
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: quandl_operation
 version: !ruby/object:Gem::Version
-  version: 0.1.18
+  version: 0.1.19
 platform: ruby
 authors:
 - Blake Hilscher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-21 00:00:00.000000000 Z
+date: 2013-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -149,6 +149,9 @@ files:
 - lib/quandl/operation/core_ext/time.rb
 - lib/quandl/operation/parse.rb
 - lib/quandl/operation/qdformat.rb
+- lib/quandl/operation/qdformat/dump.rb
+- lib/quandl/operation/qdformat/load.rb
+- lib/quandl/operation/qdformat/node.rb
 - lib/quandl/operation/transform.rb
 - lib/quandl/operation/version.rb
 - quandl_operation.gemspec