quandl_cassandra_models 0.3.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d22ebaef57fca90a9ffab18635ffb33f6cf4d85b
+  data.tar.gz: f9e5c8ab3b033b3478551e6833568c9e4b8c1bcf
+SHA512:
+  metadata.gz: 9a59a536e6e8d8ca42f090a12606c30eb1758c1bc64018103aa1434af78f40b66dda50c3a21ced85212c71d03ac20606def5841844815880a5aefeed8ff3e037
+  data.tar.gz: b0a5d4a870bcbc01e157c0e56371ea4373a1bde000276f67ed43bc55b6f181097992bacf8fc0617dfa41072377dd913111a5048e8cb52c18a88f9aedba673c67

data/.gitignore

@@ -0,0 +1,7 @@
+/Gemfile.lock
+/pkg
+/tmp
+.rvmrc
+*.gem
+*.log
+log/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+gemspec
+
+gem 'cassandra_migrations','~> 0.3', git: 'git@github.com:blakehilscher/cassandra_migrations.git'

data/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2012-2013 Blake Hilscher
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,7 @@
+# Installation
+
+```ruby
+
+gem 'quandl_cassandra_models', git: 'https://github.com/quandl/quandl_cassandra_models.git'
+
+```

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler"
+require "rake"
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.pattern = "spec/**/*_spec.rb"
+end
+
+load "tasks/migrations.rake"

data/UPGRADE.md

@@ -0,0 +1,34 @@
+## 0.3.5
+
+* fixes quandl/wikiposit#417
+* refactor column trim_start and trim_end to use Column.find_max_time_by_ids
+* move error handling to strategy.perform
+
+
+## 0.3.3
+
+* should handle column that is outside the range of available columns
+
+
+## 0.3.1
+
+* given data with empty columns it should save the dataset
+
+
+## 0.3.0
+
+* multiset given dataset with columns that do not have column_attributes saved, should not raise error. should guess at frequency.
+
+## 0.2.1
+
+* Dataset#data= accepts array data with dates
+
+
+## 0.2.0
+
+* include Quandl::Cassandra::Models in Quandl::Cassandra
+
+
+## 0.0.1
+
+* init

data/config/cassandra.yml

@@ -0,0 +1,41 @@
+production:
+  host: 10.235.22.174
+  port: 9042
+  keyspace: wikiposit
+  consistency: one
+  replication:
+    class: SimpleStrategy
+    replication_factor: 1
+
+staging:
+  host: 10.235.36.67
+  port: 9042
+  keyspace: wikiposit
+  consistency: one
+  replication:
+    class: SimpleStrategy
+    replication_factor: 1
+
+development: &development
+  host: 127.0.0.1
+  port: 9042
+  keyspace: wikiposit
+  consistency: one
+  replication:
+    class: SimpleStrategy
+    replication_factor: 1
+
+test:
+  <<: *development
+  keyspace: wikiposit_test
+    
+vagrant: &vagrant
+  <<: *development
+  host: 192.168.33.10
+     
+vagrant_staging:
+  <<: *vagrant
+
+vagrant_test:
+  <<: *vagrant
+  keyspace: wikiposit_test

data/lib/quandl/cassandra/models.rb

@@ -0,0 +1,16 @@
+require 'quandl/cassandra/models/version'
+
+require 'quandl/cassandra'
+
+require 'quandl/cassandra/models/column'
+require 'quandl/cassandra/models/column_attribute'
+require 'quandl/cassandra/models/data'
+require 'quandl/cassandra/models/dataset'
+require 'quandl/cassandra/models/dataset_attribute'
+require 'quandl/cassandra/models/multiset'
+
+module Quandl
+module Cassandra
+  include Quandl::Cassandra::Models
+end
+end

data/lib/quandl/cassandra/models/column.rb

@@ -0,0 +1,42 @@
+class Quandl::Cassandra::Models::Column < Quandl::Cassandra::Base
+
+  table_name :columns
+
+  require_relative 'column/read'
+  require_relative 'column/write'
+  
+  class << self
+    
+    def read(*args)
+      Quandl::Cassandra::Models::Column::Read.perform(*args)[:data]
+    end
+    
+    def write(*args)
+      Quandl::Cassandra::Models::Column::Write.perform(*args)
+    end
+    
+    def find_max_time_by_ids(ids, collapses, order)
+      unique = collapses.uniq.count == 1
+      time = find_max_time_by_ids_and_collapse(ids, collapses.first, "DESC") if unique
+      time = find_max_time_by_ids_and_collapses(ids, collapses, "DESC") unless unique
+      Date.jd(time) if time.is_a?(Integer)
+    end
+    
+    def find_max_time_by_ids_and_collapses(ids, collapses, order)
+      # otherwise lookup each column seperately
+      times = []
+      column_ids.each_with_index do |id, index|
+        column_collapse = collapses[index]
+        next if column_collapse.blank?
+        times << limit(1).order("type #{order}, time #{order}").where( id: id, type: collapse ).select(:time, :type).to_a[0]['time']
+      end
+      times.flatten.max
+    end
+    
+    def find_max_time_by_ids_and_collapse(ids, collapse, order)
+      limit(1).order("type #{order}, time #{order}").where( id: ids, type: collapse ).select(:time, :type).to_a[0]['time']
+    end
+    
+  end
+  
+end

data/lib/quandl/cassandra/models/column/read.rb

@@ -0,0 +1,49 @@
+class Quandl::Cassandra::Models::Column::Read < Quandl::Strategy::Strategize
+
+  require_relative 'read/collapse'
+  require_relative 'read/column'
+  require_relative 'read/data'
+  require_relative 'read/offset'
+  require_relative 'read/select_columns'
+  require_relative 'read/row'
+  require_relative 'read/transform'
+  require_relative 'read/trim'
+  require_relative 'read/type'
+
+  define_attributes :id
+
+  def self.perform(attributes)
+    strategy = Quandl::Strategy.new( attributes ) do |c|
+      c.use Quandl::Cassandra::Models::Column::Read::Type
+      c.use Quandl::Cassandra::Models::Column::Read::Row
+      c.use Quandl::Cassandra::Models::Column::Read::Column
+      c.use Quandl::Cassandra::Models::Column::Read::Collapse
+      c.use Quandl::Cassandra::Models::Column::Read::Transform
+      c.use Quandl::Cassandra::Models::Column::Read::Offset
+      c.use Quandl::Cassandra::Models::Column::Read::Trim
+      c.use Quandl::Cassandra::Models::Column::Read::SelectColumns
+      c.use Quandl::Cassandra::Models::Column::Read::Data
+    end
+    strategy.perform
+  rescue => e
+    Quandl::Logger.error("#{e} #{strategy.attributes}")
+    raise
+  end
+  
+  def count?
+    attributes[:count] == true
+  end
+  
+  def column_ids=(value)
+    attributes[:column_ids] = Array(value).compact.flatten
+  end
+  
+  def column_collapses
+    attributes[:column_collapses]
+  end
+  
+  def column_ids
+    attributes[:column_ids]
+  end
+  
+end

data/lib/quandl/cassandra/models/column/read/collapse.rb

@@ -0,0 +1,41 @@
+class Quandl::Cassandra::Models::Column::Read::Collapse < Quandl::Cassandra::Models::Column::Read
+  
+  def perform
+    attributes[:collapse] ||= :source
+    attributes[:column_frequencies] ||= select_column_frequencies
+    attributes[:column_collapses] = build_column_collapses_using_frequencies( attributes[:column_frequencies], attributes[:collapse] )
+    attributes[:frequency] = find_largest_frequency( attributes[:column_frequencies] )
+  end
+  
+  def select_column_frequencies
+    freqs = Quandl::Cassandra::Models::ColumnAttribute.where( id: column_ids ).select(:id, :frequency).to_a
+    column_ids.collect{|id| freqs.detect{|f| f['id'] == id }.try(:[], 'frequency') }
+  end
+  
+  def build_column_collapses_using_frequencies(freqs, collapse)
+    freqs.collect do |frequency|
+      # is frequency less than collapse?
+      if Quandl::Operation::Collapse.collapses_greater_than( frequency ).include?( collapse )
+        # this column needs to be collapsed
+        collapse
+      # otherwise frequency is greater than or equal to collapse
+      else
+        # so collapse is not needed
+        :source
+      end
+    end
+  end
+  
+  # [:daily, :weekly, :monthly]
+  def find_largest_frequency(freqs)
+    attributes[:column_frequencies].inject(:daily) do |freq,column_freq|
+      # index of both frequencies
+      freq_index = Quandl::Operation::Collapse.valid_collapses.index(freq)
+      column_freq_index = Quandl::Operation::Collapse.valid_collapses.index(column_freq).to_i
+      # is column_freq greater than freq?
+      freq = column_freq if column_freq_index > freq_index
+      freq
+    end
+  end
+  
+end

data/lib/quandl/cassandra/models/column/read/column.rb

@@ -0,0 +1,19 @@
+class Quandl::Cassandra::Models::Column::Read::Column < Quandl::Cassandra::Models::Column::Read
+  
+  def perform
+    # given an id, this is a dataset
+    self.column_ids ||= Quandl::Cassandra::Models::Dataset.find_column_ids_by_id(id) if id.present?
+    self.column_ids = [] if self.column_ids.blank?
+    # given a column, this is requesting a specific column
+    self.column_ids = pluck_column_id if column.present? && self.column_ids.present?
+  end
+  
+  def pluck_column_id
+    [ self.column_ids[ column ] ]
+  end
+  
+  def column
+    attributes[:column].present? ? attributes[:column] - 1 : nil
+  end
+  
+end

data/lib/quandl/cassandra/models/column/read/data.rb

@@ -0,0 +1,59 @@
+class Quandl::Cassandra::Models::Column::Read::Data < Quandl::Cassandra::Models::Column::Read
+  
+  def perform
+    attributes[:data] = quandl_data( attributes[:data] ) unless count?
+  end
+  
+  def quandl_data(data)
+    # init
+    data = Quandl::Cassandra::Models::Data.new( data )
+    data.dataset_id = attributes[:id]
+    data.column_ids = column_ids unless attributes[:column].present?
+    data.column_frequencies = attributes[:column_frequencies] unless attributes[:column].present?
+    # post process
+    data = collapse(data)
+    data = transform(data)
+    data = trim(data)
+    data = sort( data )
+    data = row( data )
+    data = limit(data)
+    data
+  end
+  
+  def trim(data)
+    return data unless attributes[:data_table].present?
+    data = data.sort_order( :asc ) if attributes[:data_table][:trim_start] || attributes[:data_table][:trim_end]
+    data = data.trim_start( attributes[:data_table][:trim_start] ) if attributes[:data_table][:trim_start]
+    data = data.trim_end( attributes[:data_table][:trim_end] ) if attributes[:data_table][:trim_end]
+    data
+  end
+  
+  def sort(data)
+    data = data.sort_order( attributes[:order] )
+  end
+  
+  def collapse(data)
+    data = data.collapse( attributes[:collapse] ) if attributes[:collapse]
+    data
+  end
+  
+  def transform(data)
+    data = data.transform( attributes[:transform] ) if attributes[:transform]
+    data
+  end
+  
+  def limit(data)
+    data = data.limit( attributes[:pristine][:limit] ) if attributes[:pristine][:limit]
+    data
+  end
+  
+  def row(data)
+    # if a row was requested and the data count is greater than a single row
+    if attributes[:row].present? && data.count > 1
+      # grab the last row
+      data.data_array = [data.data_array[-1]]
+    end
+    data
+  end
+  
+end

data/lib/quandl/cassandra/models/column/read/offset.rb

@@ -0,0 +1,104 @@
+class Quandl::Cassandra::Models::Column::Read::Offset < Quandl::Cassandra::Models::Column::Read
+  
+  def perform
+    # apply offset
+    return unless column_ids.present?
+    apply_offset_with_transform
+    apply_offset
+  end
+  
+  def apply_offset_with_transform
+    # rdiff_from needs the data from the current to offset
+    if attributes[:row].present? && attributes[:transform] == :rdiff_from
+      # limit the results by trim 
+      attributes[:trim_start] = trim_start.occurrences_of_frequency_ago( attributes[:row], collapse_with_frequency ).end_of_frequency(collapse_with_frequency).jd if order == :desc
+      attributes[:trim_end] = trim_end.occurrences_of_frequency_ahead( attributes[:row], collapse_with_frequency ).end_of_frequency(collapse_with_frequency).jd if order == :asc
+      # the query should not limit or offset the data
+      attributes[:limit] = nil
+      attributes[:offset] = nil
+    end
+  end
+  
+  def apply_offset
+    return unless attributes[:offset].present?
+    order == :asc ? apply_offset_asc : apply_offset_desc
+  end
+  
+  def apply_offset_asc
+    # calculate ranges
+    offset_start  = trim_end.occurrences_of_frequency_ahead( offset, collapse_with_frequency ).start_of_frequency(collapse_with_frequency)
+    offset_end    = offset_start.occurrences_of_frequency_ahead( accuracy_with_limit, collapse_with_frequency ).end_of_frequency( collapse_with_frequency ) if limit
+    # set trims
+    attributes[:trim_start]  = offset_start.jd
+    attributes[:trim_end]    = offset_end.jd  if limit
+  end
+  
+  def apply_offset_desc
+    # calculate ranges
+    offset_start  = trim_start.occurrences_of_frequency_ago( offset, collapse_with_frequency ).end_of_frequency(collapse_with_frequency)
+    offset_end    = offset_start.occurrences_of_frequency_ago( accuracy_with_limit, collapse_with_frequency ).start_of_frequency( collapse_with_frequency ) if limit
+    # set trims
+    attributes[:trim_start] = offset_end.jd  if limit
+    attributes[:trim_end]   = offset_start.jd
+  end
+  
+  def trim_start
+    @trim_start ||= Quandl::Cassandra::Column.find_max_time_by_ids(column_ids, column_collapses, "DESC")
+  end
+  
+  def trim_end
+    @trim_end ||= Quandl::Cassandra::Column.find_max_time_by_ids(column_ids, column_collapses, "ASC")
+  end
+  
+  def accuracy_with_limit
+    # # short circuit for now
+    # return 0
+    # revist this at a later date:
+    awl = accuracy
+    awl = awl + limit - 1 if limit && limit > 0
+    awl
+  end
+  
+  def accuracy
+    # did the query include an accuracy?
+    accuracy = attributes[:accuracy]
+    # otherwise guess at the accuracy by collapse
+    if accuracy.blank?
+      accuracy = case frequency
+      when :daily     then 0
+      when :weekly    then 1
+      when :monthly   then 1
+      when :quarterly then 1
+      when :annual    then 0
+      else
+        3
+      end
+    end
+    accuracy
+  end
+  
+  def limit
+    @limit ||= attributes[:limit]
+  end
+  
+  def offset
+    attributes[:offset]
+  end
+  
+  def order
+    @order ||= attributes[:order] == :asc ? :asc : :desc
+  end
+  
+  def collapse_with_frequency
+    collapse == :source ? frequency : collapse
+  end
+  
+  def frequency
+    attributes[:frequency]
+  end
+  
+  def collapse
+    attributes[:collapse]
+  end
+  
+end