DatamapperRackStreamer 0.1

data/LICENSE

@@ -0,0 +1,28 @@
+== DatamapperRackStreamer
+BSD Licence
+
+Copyright (c) 2009, crealytics GmbH Germany
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, 
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the crealytics GmbH nor the names of its contributors
+      may be used to endorse or promote products derived from this software without
+      specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.

data/README

@@ -0,0 +1,7 @@
+== DatamapperRackStreamer
+
+This project streams data from a database to a client via a web-interface.
+It uses DataMapper for data retrieval and Sinatra/Rack combo for serving the interface.
+The data is streamed as csv.
+
+For more information see demo project streamer_app and spec.

data/Rakefile

@@ -0,0 +1,37 @@
+
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/testtask'
+
+spec = Gem::Specification.new do |s|
+  s.name = 'DatamapperRackStreamer'
+  s.version = '0.1'
+  s.has_rdoc = true
+  s.extra_rdoc_files = ['README', 'LICENSE']
+  s.summary = 'Fetches data via data_mapper and streams it as csv via rack'
+  s.description = s.summary
+  s.author = 'ugroebner'
+  s.email = 'ugroebner@crealytics.de'
+  # s.executables = ['your_executable_here']
+  s.files = %w(LICENSE README Rakefile) + Dir.glob("{bin,lib,spec}/**/*")
+  s.require_path = "lib"
+  s.bindir = "bin"
+end
+
+Rake::GemPackageTask.new(spec) do |p|
+  p.gem_spec = spec
+  p.need_tar = true
+  p.need_zip = true
+end
+
+Rake::RDocTask.new do |rdoc|
+  files =['README', 'LICENSE', 'lib/**/*.rb']
+  rdoc.rdoc_files.add(files)
+  rdoc.main = "README" # page to start on
+  rdoc.title = "DatamapperRackStreamer Docs"
+  rdoc.rdoc_dir = 'doc/rdoc' # rdoc output folder
+  rdoc.options << '--line-numbers'
+end

data/lib/csv_streamer.rb

@@ -0,0 +1,50 @@
+require 'iconv'
+require 'stringio'
+
+class CsvStreamer
+  include Enumerable
+  BYTE_ARRAY_UTF_BOM = [0xff, 0xfe].collect{|byte| byte.chr}.join
+  UTF_16_LE_ICONV = Iconv.new('UTF-16LE', 'UTF-8')
+  COLUMN_SEPARATOR = "\t"
+  ROW_SEPARATOR = "\n"
+  attr_reader :csv_sequence
+  def initialize(model_class,ids,csv_sequence = nil, per_page = 1000, filter_mappings = {}, &block)
+    @ids = ids
+    @model_class = model_class
+    @csv_sequence = csv_sequence || self.class.csv_sequence(@model_class)
+    @per_page = per_page
+    @filter_mappings = filter_mappings
+    @block = block
+  end
+  
+  def self.csv_sequence(model_class)
+    model_class.properties.collect {|p| p.name} - [:id]
+  end
+
+  def to_csv(array)
+    UTF_16_LE_ICONV.iconv(array.join(COLUMN_SEPARATOR) + ROW_SEPARATOR)
+  end
+
+  def each
+    elements_count = @ids.size
+    page_counter = 0
+    yield BYTE_ARRAY_UTF_BOM
+    yield to_csv(@csv_sequence)
+    while (page_counter * @per_page <= elements_count)  do
+      elements = @model_class.all(:id=>@ids[page_counter*@per_page,@per_page])
+      elements.each do |element|
+        element = @block.call(element) unless @block.nil?
+        csv = to_csv(@csv_sequence.collect do |field|
+            fieldval = element.send(field)
+            if @filter_mappings.has_key?(field)
+              filterblock = eval "Proc.new {#{@filter_mappings[field]}}"
+              fieldval = filterblock.call(fieldval)
+            end
+            fieldval
+          end)
+        yield StringIO.new(csv).read
+      end
+      page_counter = page_counter + 1
+    end
+  end
+end

data/spec/csv_streamer_spec.rb

@@ -0,0 +1,25 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'csv_streamer'
+describe CsvStreamer do
+  describe "with a block" do
+    UTF_8_ICONV = Iconv.new('UTF-8', 'UTF-16LE' )
+    before do
+      @model_class = Object.new
+      stub(@model_class).all {@ids.collect {|id| OpenStruct.new(:value=>id)} }
+      stub(@model_class).properties {[OpenStruct.new(:name=>:value)]}
+      @ids = [1,4,5]
+    end
+    it "should apply the block on each model element" do
+      streamer = CsvStreamer.new(@model_class, @ids){|elem| OpenStruct.new(:value => elem.value + 1)}
+      elements = streamer.collect {|elem| UTF_8_ICONV.iconv(elem).strip.to_i}[2..-1]
+      elements.should == [2,5,6]
+    end
+    it "should apply filtermapping blocks on model elements" do
+      filter_mappings = {:value => '|val| val + 1'}
+      streamer = CsvStreamer.new(@model_class, @ids, nil, 100, filter_mappings)
+      elements = streamer.collect {|elem| UTF_8_ICONV.iconv(elem).strip.to_i}[2..-1]
+      elements.should == [2,5,6]
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,35 @@
+$LOAD_PATH << File.dirname(__FILE__)+'/../lib/'
+$LOAD_PATH << File.dirname(__FILE__)+'/../demo/'
+require "spec" # Satisfies Autotest and anyone else not using the Rake tasks
+require 'rr'
+require 'datamapper'
+require 'ostruct'
+require 'pp'
+require 'demo/model/shop'
+require 'demo/model/product'
+
+DataMapper::Logger.new(STDOUT, :debug)
+DataMapper.setup(:products, 'sqlite3::memory:')
+Shop.auto_migrate!
+Product.auto_migrate!
+
+Spec::Runner.configure do |config|
+ config.mock_with RR::Adapters::Rspec
+  config.after(:each) do
+    repository(:products) do
+      while repository.adapter.current_transaction
+        repository.adapter.current_transaction.rollback
+        repository.adapter.pop_transaction
+      end
+    end
+  end
+
+  config.before(:each) do
+    repository(:products) do
+      transaction = DataMapper::Transaction.new(repository)
+      transaction.begin
+      repository.adapter.push_transaction(transaction)
+    end
+  end
+
+end

data/spec/streamer_app_spec.rb

@@ -0,0 +1,62 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'streamer_app'  # <-- your sinatra app
+require 'spec'
+require 'rack/test'
+require 'rr'
+require 'iconv'
+
+set :environment, :test
+require 'streamer_app'
+
+describe StreamerApp do
+  include Rack::Test::Methods
+
+  def app
+    StreamerApp
+  end
+
+  before do
+    @shops = [Shop.create(:name=>'shop'),Shop.create(:name=>'shop2')]
+    @shop = @shops.first
+    @products = [1,2,5].collect do |num|
+      Product.create(:shop => @shop,
+        :id => num,
+        :title => "title #{num}",
+        :description => "description #{num}",
+        :price => num*100
+      )
+    end
+    csv_sequence = CsvStreamer.csv_sequence(Product)
+    p csv_sequence
+    rows = ([csv_sequence] + @products.collect do |product|
+      csv_sequence.collect do |field|
+        product.send(field)
+      end
+    end)
+    p rows
+    @csv = rows.collect {|values| values.join(CsvStreamer::COLUMN_SEPARATOR)}.join(CsvStreamer::ROW_SEPARATOR)+CsvStreamer::ROW_SEPARATOR
+  utf_16_le_iconv = Iconv.new('UTF-16LE', 'UTF-8')
+  @csv =  [0xff, 0xfe].collect{|byte| byte.chr}.join + utf_16_le_iconv.iconv(@csv)
+  end
+
+  it "should respond to URLs like /demo/$/products.csv" do
+    get '/demo/1/products.csv'
+    last_response.should be_ok
+  end
+
+  it "should return csv for all products of the shop" do
+     get '/demo/1/products.csv'
+     last_response.body.should == @csv
+  end
+
+  ['/demo', '/demo/'].each do |shop_string|
+  it "should list all shops when calling #{shop_string}" do
+    get shop_string
+    last_response.should be_ok
+    @shops.each do |shop|
+    last_response.body.should include "<a href='/demo/#{shop.id}/products.csv'>#{shop.name}</a>"
+    end
+  end
+  end
+end
+

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification 
+name: DatamapperRackStreamer
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- ugroebner
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-01-11 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: Fetches data via data_mapper and streams it as csv via rack
+email: ugroebner@crealytics.de
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+- LICENSE
+files: 
+- LICENSE
+- README
+- Rakefile
+- lib/csv_streamer.rb
+- spec/csv_streamer_spec.rb
+- spec/spec_helper.rb
+- spec/streamer_app_spec.rb
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Fetches data via data_mapper and streams it as csv via rack
+test_files: []
+