bovespa_ingestion 0.1.0

data/Gemfile

@@ -0,0 +1,11 @@
+source :rubygems
+
+gem 'rake'
+gem 'activerecord', ">=3"
+gem 'standalone_migrations'
+
+group :dev do
+  gem 'rspec', "~>2"
+  gem 'sqlite3'
+  gem 'jeweler', "~> 1.6.4"
+end

data/Gemfile.lock

@@ -0,0 +1,49 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.1.3)
+      activesupport (= 3.1.3)
+      builder (~> 3.0.0)
+      i18n (~> 0.6)
+    activerecord (3.1.3)
+      activemodel (= 3.1.3)
+      activesupport (= 3.1.3)
+      arel (~> 2.2.1)
+      tzinfo (~> 0.3.29)
+    activesupport (3.1.3)
+      multi_json (~> 1.0)
+    arel (2.2.1)
+    builder (3.0.0)
+    diff-lcs (1.1.3)
+    git (1.2.5)
+    i18n (0.6.0)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    multi_json (1.0.4)
+    rake (0.9.2.2)
+    rspec (2.7.0)
+      rspec-core (~> 2.7.0)
+      rspec-expectations (~> 2.7.0)
+      rspec-mocks (~> 2.7.0)
+    rspec-core (2.7.1)
+    rspec-expectations (2.7.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.7.0)
+    sqlite3 (1.3.5)
+    standalone_migrations (1.0.5)
+      activerecord (>= 3)
+      rake
+    tzinfo (0.3.31)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (>= 3)
+  jeweler (~> 1.6.4)
+  rake
+  rspec (~> 2)
+  sqlite3
+  standalone_migrations

data/README.md

@@ -0,0 +1,51 @@
+Bovespa Ingestion
+=============
+
+An importer/parser of BM&F Bovespa historical stock quotations files.
+
+Introduction
+------------
+
+Installing
+----------
+
+Install bundler:
+
+	$ gem install bundler
+
+Enter the cloned dir and fetch dependencies:
+
+	$ bundler install
+
+Contributing
+------------
+
+1) Fork this git repository
+
+2) Push your changes to your own repository
+
+3) Send a pull request
+
+Running
+-------
+
+Run the test application:
+
+	$ bin/carregar_historico
+	
+Testing
+-------
+
+We use rspec, so:
+
+	$ rspec spec
+
+or simply:
+
+  $ rake
+
+Prerequisites
+-------------
+
+Tested with Ruby 1.9.2
+

data/Rakefile

@@ -0,0 +1,37 @@
+require 'rake'
+require 'rspec/core/rake_task'
+
+begin
+  require 'tasks/standalone_migrations'
+rescue LoadError => e
+  puts "gem install standalone_migrations to get db:migrate:* tasks! (Error: #{e})"
+end
+
+RSpec::Core::RakeTask.new 'rspec' do |t|
+  t.pattern = 'spec/historico_ativos/*_spec.rb'
+  t.rspec_opts = ['--tty', '--color', '--format documentation']
+end
+
+task :default => [:rspec]
+
+# rake install -> install gem locally (for tests)
+# rake release -> push to github and release to gemcutter
+# rake version:bump:patch -> increase version and add a git-tag
+begin
+  require 'jeweler'
+rescue LoadError => e
+  $stderr.puts "Jeweler, or one of its dependencies, is not available:"
+  $stderr.puts "#{e.class}: #{e.message}"
+  $stderr.puts "Install it with: sudo gem install jeweler"
+else
+  Jeweler::Tasks.new do |gem|
+    gem.name = 'bovespa_ingestion'
+    gem.summary = "An importer of historical quotations from BM&F Bovespa"
+    gem.email = "seixasfelipe@gmail.com"
+    gem.homepage = "http://github.com/seixasfelipe/bovespa-ingestion"
+    gem.authors = ["Felipe Seixas", "Rodrigo Fraga"]
+  end
+
+  Jeweler::GemcutterTasks.new
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/carregar_historico

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'historico_ativos'
+
+module HistoricoAtivos
+
+  sample_file = 'sample/sample_cota_hist_2003.txt'
+  sample_file = ARGV[0] if ARGV.size > 0
+
+  puts "Carregando historico do arquivo: #{sample_file}"
+
+  start_time = Time.now
+  
+  loader = CarregaHistorico.new ParserHeader.new, ParserTrailer.new, ParserAtivo.new
+  historico = loader.load sample_file 
+  loader.persist historico
+
+  end_time = Time.now 
+  elapsed_time = (end_time - start_time) * 1000.0
+
+  puts "Historico carregado em #{elapsed_time} ms!"
+end

data/bovespa_ingestion.gemspec

@@ -0,0 +1,80 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{bovespa_ingestion}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Felipe Seixas", "Rodrigo Fraga"]
+  s.date = %q{2011-12-26}
+  s.default_executable = %q{carregar_historico}
+  s.email = %q{seixasfelipe@gmail.com}
+  s.executables = ["carregar_historico"]
+  s.extra_rdoc_files = [
+    "README.md"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "archives/SeriesHistoricas_Layout.pdf",
+    "bin/carregar_historico",
+    "bovespa_ingestion.gemspec",
+    "config/environment.rb",
+    "db/config.yml",
+    "db/migrate/20111221001450_create_historico.rb",
+    "db/migrate/20111222192119_create_ativos.rb",
+    "db/migrate/20111223004105_change_ativos.rb",
+    "db/migrate/20111223023840_add_quantidade_ativos_to_historico_ativos.rb",
+    "db/migrate/20111223131703_add_prazo_termo_to_ativo.rb",
+    "db/migrate/20111223213342_alter_reference_historico_ativo.rb",
+    "db/migrate/20111223214919_remove_data_importacao.rb",
+    "db/schema.rb",
+    "lib/historico_ativos.rb",
+    "lib/historico_ativos/ativo.rb",
+    "lib/historico_ativos/carrega_historico.rb",
+    "lib/historico_ativos/header.rb",
+    "lib/historico_ativos/historico.rb",
+    "lib/historico_ativos/parser_ativo.rb",
+    "lib/historico_ativos/parser_header.rb",
+    "lib/historico_ativos/parser_trailer.rb",
+    "lib/historico_ativos/trailer.rb",
+    "sample/DemoCotacoesHistoricas12022003.txt",
+    "sample/sample_cota_hist_2003.txt",
+    "spec/historico_ativos/ativo_spec.rb",
+    "spec/historico_ativos/carrega_historico_spec.rb",
+    "spec/historico_ativos/historico_spec.rb",
+    "spec/historico_ativos/parser_ativo_spec.rb",
+    "spec/historico_ativos/parser_header_spec.rb",
+    "spec/historico_ativos/parser_trailer_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/seixasfelipe/bovespa-ingestion}
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.6.2}
+  s.summary = %q{An importer of historical quotations from BM&F Bovespa}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<rake>, [">= 0"])
+      s.add_runtime_dependency(%q<activerecord>, [">= 3"])
+      s.add_runtime_dependency(%q<standalone_migrations>, [">= 0"])
+    else
+      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<activerecord>, [">= 3"])
+      s.add_dependency(%q<standalone_migrations>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<activerecord>, [">= 3"])
+    s.add_dependency(%q<standalone_migrations>, [">= 0"])
+  end
+end
+

data/config/environment.rb

@@ -0,0 +1,10 @@
+require 'active_record'
+require 'yaml'
+
+config_path = ::File.expand_path('../../db/config.yml',  __FILE__)
+db_config = YAML.load_file(config_path)
+
+ActiveRecord::Base.establish_connection(
+  db_config['development']
+  )
+

data/db/config.yml

@@ -0,0 +1,22 @@
+development:
+  adapter: sqlite3
+  database: db/development.sqlite3
+  pool: 5
+  timeout: 5000
+
+production:
+  adapter: mysql
+  encoding: utf8
+  reconnect: false
+  database: somedatabase_dev
+  pool: 5
+  username: root
+  password:
+  socket: /var/run/mysqld/mysqld.sock
+
+test: &test
+  adapter: sqlite3
+  database: db/test.sqlite3
+  pool: 5
+  timeout: 5000
+

data/db/migrate/20111221001450_create_historico.rb

@@ -0,0 +1,15 @@
+class CreateHistorico < ActiveRecord::Migration
+  def self.up
+    create_table :historico_ativos do |t|
+      t.string :nome_arquivo
+      t.string :codigo_origem
+      t.date :data_geracao
+      t.date :data_importacao
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :historico_ativos
+  end
+end

data/db/migrate/20111222192119_create_ativos.rb

@@ -0,0 +1,28 @@
+class CreateAtivos < ActiveRecord::Migration
+  def self.up
+    create_table :ativos do |t|
+      t.date :data
+      t.string :codigo_bdi
+      t.string :codigo
+      t.string :tipo_mercado
+      t.string :nome
+      t.string :especificacao
+      t.string :moeda_referencia
+      t.decimal :preco_abertura
+      t.decimal :preco_maximo
+      t.decimal :preco_minimo
+      t.decimal :preco_medio
+      t.decimal :preco_ultimo
+      t.decimal :preco_melhor_oferta_compra
+      t.decimal :preco_melhor_oferta_venda
+      t.decimal :total_negocios
+      t.decimal :quantidade_titulos_negociados
+      t.decimal :volume_negocios
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :ativos
+  end
+end

data/db/migrate/20111223004105_change_ativos.rb

@@ -0,0 +1,10 @@
+class ChangeAtivos < ActiveRecord::Migration
+  def self.up
+    add_column :ativos, :historico_ativo_id, :integer
+    add_index :ativos, :historico_ativo_id
+  end
+
+  def self.down
+    remove_column :ativos, :historico_ativo_id
+  end
+end

data/db/migrate/20111223023840_add_quantidade_ativos_to_historico_ativos.rb

@@ -0,0 +1,9 @@
+class AddQuantidadeAtivosToHistoricoAtivos < ActiveRecord::Migration
+  def self.up
+    add_column :historico_ativos, :quantidade_ativos, :integer
+  end
+
+  def self.down
+    remove_column :historico_ativos, :quantidade_ativos
+  end
+end

data/db/migrate/20111223131703_add_prazo_termo_to_ativo.rb

@@ -0,0 +1,9 @@
+class AddPrazoTermoToAtivo < ActiveRecord::Migration
+  def self.up
+    add_column :ativos, :prazo_termo, :integer
+  end
+
+  def self.down
+    remove_column :ativos, :prazo_termo
+  end
+end

data/db/migrate/20111223213342_alter_reference_historico_ativo.rb

@@ -0,0 +1,13 @@
+class AlterReferenceHistoricoAtivo < ActiveRecord::Migration
+  def self.up
+    remove_index :ativos, :historico_ativo_id
+    rename_column :ativos, :historico_ativo_id, :historico_id
+    add_index :ativos, :historico_id
+  end
+
+  def self.down
+    remove_index :ativos, :historico_id
+    rename_column :ativos, :historico_id, :historico_ativo_id
+    add_index :ativos, :historico_ativo_id
+  end
+end

data/db/migrate/20111223214919_remove_data_importacao.rb

@@ -0,0 +1,9 @@
+class RemoveDataImportacao < ActiveRecord::Migration
+  def self.up
+    remove_column :historico_ativos, :data_importacao
+  end
+
+  def self.down
+    add_column :historico_ativos, :data_importacao, :date
+  end
+end

data/db/schema.rb

@@ -0,0 +1,51 @@
+# encoding: UTF-8
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended to check this file into your version control system.
+
+ActiveRecord::Schema.define(:version => 20111223214919) do
+
+  create_table "ativos", :force => true do |t|
+    t.date     "data"
+    t.string   "codigo_bdi"
+    t.string   "codigo"
+    t.string   "tipo_mercado"
+    t.string   "nome"
+    t.string   "especificacao"
+    t.string   "moeda_referencia"
+    t.decimal  "preco_abertura"
+    t.decimal  "preco_maximo"
+    t.decimal  "preco_minimo"
+    t.decimal  "preco_medio"
+    t.decimal  "preco_ultimo"
+    t.decimal  "preco_melhor_oferta_compra"
+    t.decimal  "preco_melhor_oferta_venda"
+    t.decimal  "total_negocios"
+    t.decimal  "quantidade_titulos_negociados"
+    t.decimal  "volume_negocios"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.integer  "historico_id"
+    t.integer  "prazo_termo"
+  end
+
+  add_index "ativos", ["historico_id"], :name => "index_ativos_on_historico_id"
+
+  create_table "historico_ativos", :force => true do |t|
+    t.string   "nome_arquivo"
+    t.string   "codigo_origem"
+    t.date     "data_geracao"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+    t.integer  "quantidade_ativos"
+  end
+
+end

data/lib/historico_ativos/ativo.rb

@@ -0,0 +1,6 @@
+module HistoricoAtivos
+  class Ativo < ActiveRecord::Base
+    belongs_to :historico
+
+  end
+end

data/lib/historico_ativos/carrega_historico.rb

@@ -0,0 +1,32 @@
+module HistoricoAtivos
+  class CarregaHistorico
+
+    attr_accessor :parser_header, :parser_trailer, :parser_ativo
+
+    def initialize(parser_header, parser_trailer, parser_ativo)
+      @parser_header = parser_header
+      @parser_trailer = parser_trailer
+      @parser_ativo = parser_ativo
+    end
+
+    def load(filepath)
+      return false unless File.exists?(filepath)
+      
+      file = File.open(filepath, "r")
+
+      historico = Historico.new
+
+      file.each { |line|
+         historico.import_header @parser_header.parse(line) if line.start_with?("00")
+         historico.ativos << @parser_ativo.parse(line) if line.start_with?("01")
+         historico.import_trailer @parser_trailer.parse(line) if line.start_with?("99")
+      }
+
+      historico
+    end
+
+    def persist(historico)
+      historico.save
+    end
+  end
+end

data/lib/historico_ativos/header.rb

@@ -0,0 +1,5 @@
+module HistoricoAtivos
+  class Header
+    attr_accessor :nome_arquivo, :codigo_origem, :data_geracao
+  end
+end

data/lib/historico_ativos/historico.rb

@@ -0,0 +1,17 @@
+module HistoricoAtivos
+  class Historico < ActiveRecord::Base
+    set_table_name "historico_ativos"
+
+    has_many :ativos
+
+    def import_header(header)
+      self.nome_arquivo = header.nome_arquivo
+      self.data_geracao = header.data_geracao
+      self.codigo_origem = header.codigo_origem
+    end
+
+    def import_trailer(trailer)
+      self.quantidade_ativos = trailer.quantidade_ativos
+    end
+  end
+end

data/lib/historico_ativos/parser_ativo.rb

@@ -0,0 +1,114 @@
+require 'date'
+require 'bigdecimal'
+
+module HistoricoAtivos
+  class ParserAtivo
+    def parse(row)
+      read_ativo(row) if row.start_with?("01")
+    end
+    
+    def read_ativo(row)
+      ativo = Ativo.new
+      ativo.data = read_data row
+      ativo.codigo_bdi = read_codigo_bdi row
+      ativo.codigo = read_codigo row
+      ativo.tipo_mercado = read_tipo_mercado row
+      ativo.nome = read_nome row
+      ativo.especificacao = read_especificacao row
+      ativo.prazo_termo = read_prazo_termo row
+      ativo.moeda_referencia = read_moeda_referencia row
+      ativo.preco_abertura = read_preco_abertura row
+      ativo.preco_maximo = read_preco_maximo row
+      ativo.preco_minimo = read_preco_minimo row
+      ativo.preco_medio = read_preco_medio row
+      ativo.preco_ultimo = read_preco_ultimo row
+      ativo.preco_melhor_oferta_compra = read_preco_melhor_oferta_compra row
+      ativo.preco_melhor_oferta_venda = read_preco_melhor_oferta_venda row
+      ativo.total_negocios = read_total_negocios row
+      ativo.quantidade_titulos_negociados= read_quantidade_titulos_negociados row
+      ativo.volume_negocios = read_volume_negocios row
+      ativo
+    end
+
+    def read_data(row)
+      year = row[2..5]
+      month = row[6..7]
+      day = row[8..9]
+      Date.new(year.to_i, month.to_i, day.to_i)
+    end
+
+    def read_codigo_bdi(row)
+      row[10..11].to_i
+    end
+
+    def read_codigo(row)
+      row[12..23].strip
+    end
+
+    def read_tipo_mercado(row)
+      row[24..26].to_i
+    end
+
+    def read_nome(row)
+      row[27..38].strip
+    end
+
+    def read_especificacao(row)
+      row[39..48].strip
+    end
+
+    def read_prazo_termo(row)
+      row[49..51].to_i
+    end
+
+    def read_moeda_referencia(row)
+      row[52..55].strip
+    end
+
+    def read_preco_abertura(row)
+      converts_to_big_decimal row[56..68].to_i
+    end
+
+    def read_preco_maximo(row)
+      converts_to_big_decimal row[69..81].to_i
+    end
+
+    def read_preco_minimo(row)
+      converts_to_big_decimal row[82..94].to_i
+    end
+
+    def read_preco_medio(row)
+      converts_to_big_decimal row[95..107].to_i
+    end
+    
+    def read_preco_ultimo(row)
+      converts_to_big_decimal row[108..120].to_i
+    end
+    
+    def read_preco_melhor_oferta_compra(row)
+      converts_to_big_decimal row[121..133].to_i
+    end
+    
+    def read_preco_melhor_oferta_venda(row)
+      converts_to_big_decimal row[134..146].to_i
+    end
+    
+    def read_total_negocios(row)
+      row[147..151].to_i
+    end
+    
+    def read_quantidade_titulos_negociados(row)
+      BigDecimal.new(row[152..169])
+    end
+    
+    def read_volume_negocios(row)
+      BigDecimal.new(row[170..187])
+    end
+    
+    def converts_to_big_decimal(preco)
+      preco_s = preco.to_s
+      preco_s.insert(preco_s.size - 2, '.')
+      BigDecimal.new(preco_s)
+    end
+  end
+end

data/lib/historico_ativos/parser_header.rb

@@ -0,0 +1,32 @@
+require 'date'
+
+module HistoricoAtivos
+  class ParserHeader
+    def parse(row)
+      read_header row if row.start_with?("00")
+    end
+
+    def read_header(row)
+      header = Header.new
+      header.nome_arquivo = read_nome_arquivo row
+      header.codigo_origem = read_codigo_origem row
+      header.data_geracao = read_data_geracao row
+      header
+    end
+
+    def read_nome_arquivo(row)
+      row[02..14].strip
+    end
+
+    def read_codigo_origem(row)
+      row[15..22].strip
+    end
+
+    def read_data_geracao(row)
+      year = row[23..26]
+      month = row[27..28]
+      day = row[29..30]
+      Date.new(year.to_i, month.to_i, day.to_i)
+    end 
+  end
+end

data/lib/historico_ativos/parser_trailer.rb

@@ -0,0 +1,17 @@
+module HistoricoAtivos
+  class ParserTrailer
+    def parse(row)
+      read_trailer row if row.start_with?("99")
+    end
+
+    def read_trailer(row)
+      trailer = Trailer.new
+      trailer.quantidade_ativos = read_quantidade_ativos row
+      trailer
+    end
+
+    def read_quantidade_ativos(row)
+      total = row[31..41].to_i
+    end
+  end
+end