quandl_operation 0.1.21 → 0.1.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7d2fcd944ad2dc1d1b251472b4fd21246f3d689
-  data.tar.gz: e0bc3a24a4069a5ec6b7ba42f2a6e9788f3d99cb
+  metadata.gz: 0ae58461349d4ba8c510ee5a0361b2502fb38867
+  data.tar.gz: a683b6d46cfa49cfb783fc8de468f22612f9e3e8
 SHA512:
-  metadata.gz: f9a93903052832e5567b45a42467e6a0d89d9b7a76800a65d972cfa6cf88a3e973aff49d6bbb7b2eaef50a554312be1349c4e12e30a99c9177cd707c53472f68
-  data.tar.gz: 8a4c52d84b8a2bd1701f0436312d4fc87a57c65e7a4bdcdc73acbe5979c3c16d0cc1d0e3738a1ceb6389026c477aac2e94fb8eb6d6adc3c2c75df7b9763743f2
+  metadata.gz: e83c29041365745773cffadbe0f913bcbbe828ed26bba60a71cd9b6a04dbdae620d4029e055ed74557e3bcbfcb37a150963d885cdd4cc97ba0fb338d6228d4c8
+  data.tar.gz: 61b2c18bdbcfd7efd021ae4e98029a2f890055f9794c3903cfaf425072622231131e0b88113723552f7e4f14af5986abe0f1267f9fe468efcd84078c12a8a91c

data/UPGRADE.md

@@ -1,3 +1,7 @@
+## 0.1.22
+
+* refactor QDFormat to support full yaml. metadata and data are seperated by a
+
 ## 0.1.21
 
 * QDFormat should handle new lines in attributes

data/lib/quandl/operation/qdformat.rb

@@ -13,7 +13,7 @@ class Quandl::Operation::QDFormat
     end
     
     def load_file(file_path)
-      Quandl::Operation::QDFormat::Load.from_file(input)
+      Quandl::Operation::QDFormat::Load.from_file(file_path)
     end
     
     def dump(nodes)

data/lib/quandl/operation/qdformat/dump.rb

@@ -4,6 +4,8 @@ class QDFormat
 
 class Dump
 
+  ATTRIBUTES = [:source_code, :code, :name, :description]
+
   class << self
 
     def nodes(*args)
@@ -23,23 +25,22 @@ class Dump
   end
 
   def to_qdf
-    [attributes, column_names, data].compact.join
+    [ attributes, 
+      "-\n",
+      column_names,
+      data
+    ].compact.join
   end
 
   def attributes
-    attrs = [:source_code, :code, :name, :description].inject({}) do |memo, name|
+    attrs = ATTRIBUTES.inject({}) do |memo, name|
       name = name.to_s
       memo[name] = node.send(name) if node.respond_to?(name) && node.send(name).present?
-      memo[name] = memo[name].gsub("\n", '\n') if memo[name].present?
       memo
     end
     attrs.to_yaml[4..-1]
   end
   
-  def description
-    record.description
-  end
-
   def data
     data = node.data.is_a?(Array) ? node.data.collect(&:to_csv).join : node.data
     data = data.to_csv if data.respond_to?(:to_csv)

data/lib/quandl/operation/qdformat/load.rb

@@ -1,5 +1,7 @@
 class Quandl::Operation::QDFormat::Load
   
+  SECTION_DELIMITER = '-'
+  
   class << self
   
     def from_file(path)
@@ -7,22 +9,32 @@ class Quandl::Operation::QDFormat::Load
     end
   
     def from_string(input)
-      nodes = [{ attributes: '', data: '' }]
-      input.each_line do |line|
+      nodes = []
+      section_type = :data
+      input.each_line do |rline|
         # strip whitespace
-        line = line.strip.rstrip
+        line = rline.strip.rstrip
         # ignore comments and blank lines
         next if line[0] == '#' || line.blank?
-        # code_format denotes the start of a new node
-        nodes << { attributes: '', data: '' } if line[0..2] == node_delimiter
-        # attribute_format denotes an attribute
+        
+        # are we looking at an attribute?
         if line =~ attribute_format
-          # add the attribute to attributes
-          nodes[-1][:attributes] += "#{line}\n"
-          # otherwise it must be data
-        else
-          nodes[-1][:data] += "#{line}\n"
+          # if we are leaving the data section
+          # then this is the start of a new node
+          nodes << { attributes: '', data: '' } if section_type == :data
+          # update the section to attributes
+          section_type = :attributes
+          
+        # have we reached the end of the attributes?
+        elsif line[0] == '-'
+          # update the section to data
+          section_type = :data
+          # skip to the next line
+          next
         end
+        # add the line to it's section in the current node.
+        # YAML must include whitespace
+        nodes[-1][section_type] += (section_type == :data) ? "#{line}\n" : rline
       end
       # append the current node
       nodes = parse_nodes(nodes)
@@ -37,10 +49,6 @@ class Quandl::Operation::QDFormat::Load
       nodes.collect do |node|
         # parse attrs as yaml
         node[:attributes] = YAML.load( node[:attributes] )
-        # replace new line characters
-        node[:attributes].each do |key, value|
-          node[:attributes][key].gsub!('\n', "\n") if node[:attributes][key].respond_to?(:gsub!)
-        end
         # parse data as csv
         node[:attributes][:data] = CSV.parse(node[:data])
         node
@@ -53,10 +61,6 @@ class Quandl::Operation::QDFormat::Load
       end
     end
     
-    def node_delimiter
-      '---'
-    end
-  
     def attribute_format
       /^([a-z0-9_]+): (.+)/
     end

data/lib/quandl/operation/qdformat/node.rb

@@ -30,7 +30,11 @@ class Quandl::Operation::QDFormat::Node
   def full_code
     [source_code, code].join('/')
   end
-
+  
+  def description=(value)
+    @description = value.to_s.gsub('\n', "\n")
+  end
+  
   def data=(rows)
     self.column_names = rows.shift unless rows.first.collect{|r| r.to_s.numeric? }.include?(true)
     @data = rows

data/lib/quandl/operation/version.rb

@@ -1,5 +1,5 @@
 module Quandl
 module Operation
-  VERSION = "0.1.21"
+  VERSION = "0.1.22"
 end
 end

data/spec/lib/quandl/operation/qdformat/load_spec.rb

@@ -6,22 +6,24 @@ describe Quandl::Operation::QDFormat::Load do
   
   let(:qdf_dataset){
     %Q{
-      # YAML metadata
+      # first dataset
       source_code: NSE
       code: OIL
       name: Oil India Limited
-      description: 'Historical prices for Oil India Limited (OIL), (ISIN: INE274J01014), National \\nStock Exchange of India.'
-      # CSV data
+      description: |-
+        Here is a description with multiple lines.
+        This is the second line.
+      -
       Date, Value, High, Low
       2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
       2013-11-19,10.039388096885814,,14.09718770934256
-      ---
       
+      # Second dataset
       code:           DATASET_CODE_2
       source_code:    SOURCE_CODE_2
       name:           Test Dataset Name 2
       description:    Here is a description with multiple lines.
-      
+      -
       Date, Value, High, Low
       2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
       2013-11-19,10.039388096885814,,14.09718770934256
@@ -43,7 +45,7 @@ describe Quandl::Operation::QDFormat::Load do
       its(:source_code){ should eq 'NSE' }
       its(:code){ should eq 'OIL' }
       its(:name){ should eq 'Oil India Limited' }
-      its(:description){ should eq "Historical prices for Oil India Limited (OIL), (ISIN: INE274J01014), National \nStock Exchange of India." }
+      its(:description){ should eq "Here is a description with multiple lines.\nThis is the second line." }
       its(:column_names){ should eq ['Date', 'Value', 'High', 'Low'] }
       its(:data){ should eq [["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"], 
         ["2013-11-19", "10.039388096885814", nil, "14.09718770934256"]] }

data/spec/lib/quandl/operation/qdformat/node_spec.rb

@@ -5,9 +5,9 @@ require 'spec_helper'
 describe Quandl::Operation::QDFormat::Node do
   
   let(:attributes) { {
-    code:           'DATASET_CODE',
+    code:           'DATASET_CODE_2',
     source_code:    'SOURCE_CODE',
-    name:           'Test Dataset Name 1',
+    name:           'Test Dataset Name 2',
     description:    "Here is a description with multiple lines.\n This is the second line.",
     column_names:   ['Date', 'Value', 'High', 'Low'],
     data:           [["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"], 
@@ -16,9 +16,9 @@ describe Quandl::Operation::QDFormat::Node do
   
   subject{ Quandl::Operation::QDFormat::Node.new(attributes) }
   
-  its(:code){ should eq 'DATASET_CODE' }
+  its(:code){ should eq 'DATASET_CODE_2' }
   its(:source_code){ should eq 'SOURCE_CODE' }
-  its(:name){ should eq 'Test Dataset Name 1' }
+  its(:name){ should eq 'Test Dataset Name 2' }
   its(:description){ should eq "Here is a description with multiple lines.\n This is the second line." }
   its(:column_names){ should eq ['Date', 'Value', 'High', 'Low'] }
   its(:data){ should eq [["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"], 
@@ -27,9 +27,12 @@ describe Quandl::Operation::QDFormat::Node do
   its(:attributes){ should eq attributes }
   
   its(:to_qdf){ should eq %Q{source_code: SOURCE_CODE
-code: DATASET_CODE
-name: Test Dataset Name 1
-description: Here is a description with multiple lines.\\n This is the second line.
+code: DATASET_CODE_2
+name: Test Dataset Name 2
+description: |-
+  Here is a description with multiple lines.
+   This is the second line.
+-
 Date,Value,High,Low
 2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
 2013-11-19,10.039388096885814,,14.09718770934256

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: quandl_operation
 version: !ruby/object:Gem::Version
-  version: 0.1.21
+  version: 0.1.22
 platform: ruby
 authors:
 - Blake Hilscher