quandl_format 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0a70e0e436b6880e25c8bfc3643d70b2440bd4d8
+  data.tar.gz: e1fe743d521464cac0e2202e8c78ad1f2b09858f
+SHA512:
+  metadata.gz: 4429a0b2cbd4c1a99c452182bbb621edb06029ceabdaa7188a3838efa0d4ba288a6ef3ffef3a11dcd20975a1365680285375ab4688b896728acef386924ee915
+  data.tar.gz: 6c38e3a0d254ee664b5ea5d1c041d5c49a2642ad89ac3f223b727100113884f735233daa88abb66927e38289f7d3a1b7b2a14e1cbb8b82463374eab74ef7903a

data/.gitignore

@@ -0,0 +1,5 @@
+/Gemfile.lock
+/pkg
+/tmp
+*.gem
+coverage/*

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2012-2013 Blake Hilscher
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,2 @@
+# Quandl::Format
+

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler"
+require "rake"
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+task :default => :spec
+
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.pattern = "spec/**/*_spec.rb"
+end

data/UPGRADE.md

@@ -0,0 +1,3 @@
+## 0.0.1
+
+* init

data/lib/quandl/format/dump.rb

@@ -0,0 +1,56 @@
+module Quandl
+module Format
+
+class Dump
+
+  ATTRIBUTES = [ :source_code, :code, :name, :description, :private, :display_url ]
+
+  class << self
+
+    def nodes(*args)
+      Array(args).flatten.collect{|r| node(r) }.join("\n")
+    end
+
+    def node(node)
+      self.new(node).to_qdf
+    end
+
+  end
+
+  attr_accessor :node
+
+  def initialize(r)
+    self.node = r
+  end
+
+  def to_qdf
+    [ attributes, 
+      "-\n",
+      column_names,
+      data
+    ].compact.join
+  end
+
+  def attributes
+    attrs = ATTRIBUTES.inject({}) do |memo, name|
+      name = name.to_s
+      memo[name] = node.send(name) if node.respond_to?(name) && node.send(name).present?
+      memo
+    end
+    attrs.to_yaml[4..-1]
+  end
+  
+  def data
+    data = node.data.is_a?(Array) ? node.data.collect(&:to_csv).join : node.data
+    data = data.to_csv if data.respond_to?(:to_csv)
+    data
+  end
+
+  def column_names
+    node.column_names.to_csv if node.column_names.present?
+  end
+
+end
+
+end
+end

data/lib/quandl/format/load.rb

@@ -0,0 +1,70 @@
+class Quandl::Format::Load
+  
+  SECTION_DELIMITER = '-'
+  
+  class << self
+  
+    def from_file(path)
+      from_string(File.read(path).strip)
+    end
+  
+    def from_string(input)
+      nodes = []
+      section_type = :data
+      input.each_line do |rline|
+        # strip whitespace
+        line = rline.strip.rstrip
+        # ignore comments and blank lines
+        next if line[0] == '#' || line.blank?
+        
+        # are we looking at an attribute?
+        if line =~ attribute_format
+          # if we are leaving the data section
+          # then this is the start of a new node
+          nodes << { attributes: '', data: '' } if section_type == :data
+          # update the section to attributes
+          section_type = :attributes
+          
+        # have we reached the end of the attributes?
+        elsif line[0] == '-'
+          # update the section to data
+          section_type = :data
+          # skip to the next line
+          next
+        end
+        # add the line to it's section in the current node.
+        # YAML must include whitespace
+        nodes[-1][section_type] += (section_type == :data) ? "#{line}\n" : rline
+      end
+      # append the current node
+      nodes = parse_nodes(nodes)
+      nodes = initialize_nodes(nodes)
+      nodes
+    end
+    
+    
+    protected
+    
+    def parse_nodes(nodes)
+      nodes.collect do |node|
+        # parse attrs as yaml
+        node[:attributes] = YAML.load( node[:attributes] )
+        # parse data as csv
+        node[:attributes][:data] = CSV.parse(node[:data])
+        node
+      end
+    end
+    
+    def initialize_nodes(nodes)
+      nodes.collect do |node|
+        Quandl::Format::Node.new(node[:attributes])
+      end
+    end
+    
+    def attribute_format
+      /^([a-z0-9_]+): (.+)/
+    end
+  
+  end
+
+end

data/lib/quandl/format/node.rb

@@ -0,0 +1,51 @@
+class Quandl::Format::Node
+  
+  ATTRIBUTES = :source_code, :code, :name, :description, :column_names, :data
+  attr_accessor *ATTRIBUTES
+
+  def initialize(attrs)
+    assign_attributes(attrs)
+  end
+
+  def assign_attributes(attrs)
+    attrs.each do |key, value|
+      self.send("#{key}=", value) if self.respond_to?(key)
+    end
+  end
+
+  def attributes
+    ATTRIBUTES.inject({}){|m,k| m[k] = self.send(k); m }
+  end
+
+  def inspect
+    "<##{self.class.name} #{attributes.to_s} >"
+  end
+
+  def full_code=(value)
+    value = value.split('/')
+    self.source_code = value[0]
+    self.code = value[1]
+  end
+
+  def full_code
+    [source_code, code].join('/')
+  end
+  
+  def description=(value)
+    @description = value.to_s.gsub('\n', "\n")
+  end
+  
+  def data=(rows)
+    self.column_names = rows.shift unless rows.first.collect{|r| r.to_s.numeric? }.include?(true)
+    @data = rows
+  end
+
+  def column_names=(names)
+    @column_names = Array(names).flatten.collect{|n| n.strip.rstrip }
+  end
+
+  def to_qdf
+    Quandl::Format::Dump.node(self)
+  end
+  
+end

data/lib/quandl/format/version.rb

@@ -0,0 +1,5 @@
+module Quandl
+module Format
+  VERSION = "0.0.1"
+end
+end

data/lib/quandl/format.rb

@@ -0,0 +1,38 @@
+require 'quandl/logger'
+
+require "quandl/format/version"
+
+require 'csv'
+
+require 'quandl/operation'
+
+require "active_support"
+require "active_support/inflector"
+require "active_support/core_ext/hash"
+require "active_support/core_ext/object"
+
+require 'quandl/format/dump'
+require 'quandl/format/load'
+require 'quandl/format/node'
+
+module Quandl
+  module Format
+    
+    class << self
+    
+      def load(input)
+        Quandl::Format::Load.from_string(input)
+      end
+    
+      def load_file(file_path)
+        Quandl::Format::Load.from_file(file_path)
+      end
+    
+      def dump(nodes)
+        Quandl::Format::Dump.nodes(nodes)
+      end
+    
+    end
+    
+  end
+end

data/quandl_format.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "quandl/format/version"
+
+Gem::Specification.new do |s|
+  s.name        = "quandl_format"
+  s.version     = Quandl::Format::VERSION
+  s.authors     = ["Blake Hilscher"]
+  s.email       = ["blake@hilscher.ca"]
+  s.homepage    = "http://blake.hilscher.ca/"
+  s.license     = "MIT"
+  s.summary     = "Quandl Data Format"
+  s.description = "Data will be loaded and dumped."
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency "rspec", "~> 2.13"
+  s.add_development_dependency "fivemat", "~> 1.2"
+  s.add_development_dependency "pry"
+  s.add_development_dependency "simplecov"
+  
+  s.add_runtime_dependency "quandl_operation", "~> 0.1"
+  
+end

data/spec/lib/quandl/format/load_spec.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Format::Load do
+  
+  let(:qdf_dataset){
+    %Q{
+      # first dataset
+      source_code: NSE
+      code: OIL
+      name: Oil India Limited
+      description: |-
+        Here is a description with multiple lines.
+        This is the second line.
+      -
+      Date, Value, High, Low
+      2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
+      2013-11-19,10.039388096885814,,14.09718770934256
+      
+      # Second dataset
+      code:           DATASET_CODE_2
+      source_code:    SOURCE_CODE_2
+      name:           Test Dataset Name 2
+      description:    Here is a description with multiple lines.
+      -
+      Date, Value, High, Low
+      2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
+      2013-11-19,10.039388096885814,,14.09718770934256
+      2013-11-18,11.039388096885814,,15.09718770934256
+    }
+  }
+  
+  describe ".from_string" do
+    
+    let(:collection){ Quandl::Format::Load.from_string(qdf_dataset) }
+    subject{ collection }
+    
+    its(:count){ should eq 2 }
+    
+    describe "#first" do
+      subject{ collection.first }
+
+      it{ should be_a Quandl::Format::Node }
+      its(:source_code){ should eq 'NSE' }
+      its(:code){ should eq 'OIL' }
+      its(:name){ should eq 'Oil India Limited' }
+      its(:description){ should eq "Here is a description with multiple lines.\nThis is the second line." }
+      its(:column_names){ should eq ['Date', 'Value', 'High', 'Low'] }
+      its(:data){ should eq [["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"], 
+        ["2013-11-19", "10.039388096885814", nil, "14.09718770934256"]] }
+    end
+  end
+  
+end

data/spec/lib/quandl/format/node_spec.rb

@@ -0,0 +1,40 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe Quandl::Format::Node do
+  
+  let(:attributes) { {
+    code:           'DATASET_CODE_2',
+    source_code:    'SOURCE_CODE',
+    name:           'Test Dataset Name 2',
+    description:    "Here is a description with multiple lines.\n This is the second line.",
+    column_names:   ['Date', 'Value', 'High', 'Low'],
+    data:           [["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"], 
+                    ["2013-11-19", "10.039388096885814", nil, "14.09718770934256"]], 
+  }}
+  
+  subject{ Quandl::Format::Node.new(attributes) }
+  
+  its(:code){ should eq 'DATASET_CODE_2' }
+  its(:source_code){ should eq 'SOURCE_CODE' }
+  its(:name){ should eq 'Test Dataset Name 2' }
+  its(:description){ should eq "Here is a description with multiple lines.\n This is the second line." }
+  its(:column_names){ should eq ['Date', 'Value', 'High', 'Low'] }
+  its(:data){ should eq [["2013-11-20", "9.99470588235294", "11.003235294117646", "14.00164705882353"], 
+                    ["2013-11-19", "10.039388096885814", nil, "14.09718770934256"]] }
+                    
+  its(:attributes){ should eq attributes }
+  
+  its(:to_qdf){ should eq %Q{source_code: SOURCE_CODE
+code: DATASET_CODE_2
+name: Test Dataset Name 2
+description: |-
+  Here is a description with multiple lines.
+   This is the second line.
+-
+Date,Value,High,Low
+2013-11-20,9.99470588235294,11.003235294117646,14.00164705882353
+2013-11-19,10.039388096885814,,14.09718770934256
+}}
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+if ENV['COVERAGE']
+  require 'simplecov'
+  SimpleCov.start
+end
+
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require "rspec"
+require "quandl/format"
+require 'pry'

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: quandl_format
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Blake Hilscher
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.13'
+- !ruby/object:Gem::Dependency
+  name: fivemat
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: quandl_operation
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+description: Data will be loaded and dumped.
+email:
+- blake@hilscher.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- UPGRADE.md
+- lib/quandl/format.rb
+- lib/quandl/format/dump.rb
+- lib/quandl/format/load.rb
+- lib/quandl/format/node.rb
+- lib/quandl/format/version.rb
+- quandl_format.gemspec
+- spec/lib/quandl/format/load_spec.rb
+- spec/lib/quandl/format/node_spec.rb
+- spec/spec_helper.rb
+homepage: http://blake.hilscher.ca/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.10
+signing_key: 
+specification_version: 4
+summary: Quandl Data Format
+test_files:
+- spec/lib/quandl/format/load_spec.rb
+- spec/lib/quandl/format/node_spec.rb
+- spec/spec_helper.rb