tin_opener 0.0.1

data/README.md

@@ -0,0 +1,37 @@
+# tin_opener
+[![Build Status](https://travis-ci.org/dgilperez/tin_opener.svg?branch=master)](https://travis-ci.org/dgilperez/tin_opener)
+[![Code Climate](https://codeclimate.com/github/dgilperez/tin_opener/badges/gpa.svg)](https://codeclimate.com/github/dgilperez/tin_opener)
+
+Opening Data, one Tin at a time.
+
+Rails Engine developed in the context of the Ayuntamiento de Madrid's Open Data project [https://twitter.com/hashtag/openmad](#OpenMad).
+
+## Usage
+
+Add gem to `Gemfile`:
+
+    gem 'tin_opener', github: 'dgilperez/tin_opener'
+
+Add a mounting point to your `routes.rb`
+
+    mount TinOpener::Engine => "/tin_opener"
+
+Run migrations:
+
+    rake tin_opener:install:migrations
+    rake db:migrate
+
+This gem provides the following models:
+
+* `TinOpener::DataSet` is any type of data structure you declare in your data domain. It's defined by it's name and headers, which in turn is a hash of key name and data type pairs.
+* `TinOpener::Record` is a instance of data belonging to a DataSet format, extracted from any CSV form.
+
+Data input format support:
+
+* CSV
+
+## TODO
+
+Support for more data formats:
+
+* Excel

data/Rakefile

@@ -0,0 +1,33 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+# require 'rdoc/task'
+#
+# RDoc::Task.new(:rdoc) do |rdoc|
+#   rdoc.rdoc_dir = 'rdoc'
+#   rdoc.title    = 'TinOpener'
+#   rdoc.options << '--line-numbers'
+#   rdoc.rdoc_files.include('README.rdoc')
+#   rdoc.rdoc_files.include('lib/**/*.rb')
+# end
+
+APP_RAKEFILE = File.expand_path("../spec/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'
+
+Bundler::GemHelper.install_tasks
+
+Dir[File.join(File.dirname(__FILE__), 'tasks/**/*.rake')].each {|f| load f }
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+
+desc "Run all specs in spec directory (excluding plugin specs)"
+RSpec::Core::RakeTask.new(spec: 'app:db:test:prepare')
+
+task default: :spec

data/app/assets/javascripts/tin_opener/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/app/assets/stylesheets/tin_opener/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/tin_opener/application_controller.rb

@@ -0,0 +1,4 @@
+module TinOpener
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/controllers/tin_opener/data_sets_controller.rb

@@ -0,0 +1,51 @@
+require_dependency "tin_opener/application_controller"
+
+module TinOpener
+  class DataSetsController < ApplicationController
+    before_action :set_data_set, only: [:show, :edit, :update, :destroy]
+
+    def index
+      @data_sets = DataSet.all
+      @data_set  = DataSet.new
+    end
+
+    def show
+      @records = @data_set.records
+    end
+
+    def edit
+    end
+
+    def create
+      @data_set = DataSet.new(data_set_params)
+
+      if @data_set.save
+        redirect_to @data_set, notice: 'Data set was successfully created.'
+      else
+        render :new
+      end
+    end
+
+    def update
+      if @data_set.update(data_set_params)
+        redirect_to @data_set, notice: 'Data set was successfully updated.'
+      else
+        render :edit
+      end
+    end
+
+    def destroy
+      @data_set.destroy
+      redirect_to data_sets_url, notice: 'Data set was successfully destroyed.'
+    end
+
+    private
+      def set_data_set
+        @data_set = DataSet.find(params[:id])
+      end
+
+      def data_set_params
+        params.require(:data_set).permit(:name, :headers, :data_file)
+      end
+  end
+end

data/app/controllers/tin_opener/records_controller.rb

@@ -0,0 +1,62 @@
+require_dependency "tin_opener/application_controller"
+
+module TinOpener
+  class RecordsController < ApplicationController
+    before_action :set_data_set
+    before_action :set_record, only: [:show, :edit, :update, :destroy]
+
+    def index
+      @records = @data_set.records
+    end
+
+    def show
+    end
+
+    def new
+      @record = @data_set.records.new
+    end
+
+    def edit
+    end
+
+    def create
+      @record = @data_set.records.new(record_params)
+
+      if @record.save
+        redirect_to data_set_record_url(@data_set, @record), notice: 'Record was successfully created.'
+      else
+        render :new
+      end
+    end
+
+    def update
+      if @record.update(record_params)
+        redirect_to data_set_record_url(@data_set, @record), notice: 'Record was successfully updated.'
+      else
+        render :edit
+      end
+    end
+
+    def destroy
+      @record.destroy
+      redirect_to data_set_records_url(@data_set), notice: 'Record was successfully destroyed.'
+    end
+
+    private
+      def set_data_set
+        @data_set = DataSet.find(params[:data_set_id])
+      end
+
+      def set_record
+        @record = @data_set.records.find(params[:id])
+      end
+
+      def record_params
+        row_data = params[:record].delete(:row_data)
+        # ActionController::Parameters.new(record: { row_data: row_data })
+        params[:record].permit!.tap do |whitelisted|
+          whitelisted[:row_data] = row_data
+        end
+      end
+  end
+end

data/app/helpers/tin_opener/application_helper.rb

@@ -0,0 +1,4 @@
+module TinOpener
+  module ApplicationHelper
+  end
+end

data/app/models/tin_opener/data_set.rb

@@ -0,0 +1,39 @@
+module TinOpener
+  class DataSet < ActiveRecord::Base
+    has_many :records, dependent: :destroy
+
+    attr_reader :data_file
+
+    validates :name, presence: true, uniqueness: { scope: :headers }
+    validates :headers, presence: true
+
+    before_validation :process_data_file
+
+    def data_file=(data_file)
+      @data_file = data_file.respond_to?(:tempfile) ? data_file.tempfile : data_file
+    end
+
+    private
+
+    def process_data_file
+      return unless data_file.is_a?(File)
+
+      data_file_processor = DataFileProcessor.new(file: data_file)
+
+      self.headers = data_file_processor.headers
+
+      import_records(data_file_processor.rows)
+      # data_file_processor.rows.each do |row|
+      #   records.new(row_data: row)
+      # end
+    end
+
+    def import_records(record_attrs)
+      to_import = record_attrs.map do |row|
+          records.new(row_data: row)
+        end
+
+      Record.import to_import
+    end
+  end
+end

data/app/models/tin_opener/record.rb

@@ -0,0 +1,8 @@
+module TinOpener
+  class Record < ActiveRecord::Base
+    belongs_to :data_set
+
+    validates :data_set, presence: true
+    validates :row_data, presence: true
+  end
+end

data/app/views/layouts/tin_opener/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>TinOpener</title>
+  <%= stylesheet_link_tag    "tin_opener/application", media: "all" %>
+  <%= javascript_include_tag "tin_opener/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/app/views/tin_opener/data_sets/_form.html.erb

@@ -0,0 +1,25 @@
+<%= form_for(@data_set, multipart: true) do |f| %>
+  <% if @data_set.errors.any? %>
+    <div id="error_explanation">
+      <h2><%= pluralize(@data_set.errors.count, "error") %> prohibited this data_set from being saved:</h2>
+
+      <ul>
+      <% @data_set.errors.full_messages.each do |message| %>
+        <li><%= message %></li>
+      <% end %>
+      </ul>
+    </div>
+  <% end %>
+
+  <div class="actions">
+    <div>
+      <%= f.text_field :name %>
+    </div>
+    <div>
+      <% if @data_set.new_record? %>
+        <%= f.file_field :data_file %>
+      <% end %>
+    </div>
+    <%= f.submit %>
+  </div>
+<% end %>

data/app/views/tin_opener/data_sets/edit.html.erb

@@ -0,0 +1,6 @@
+<h1>Editing Data Set</h1>
+
+<%= render 'form' %>
+
+<%= link_to 'Show', @data_set %> |
+<%= link_to 'Back', data_sets_path %>

data/app/views/tin_opener/data_sets/index.html.erb

@@ -0,0 +1,29 @@
+<p id="notice"><%= notice %></p>
+
+<h1>Listing Data Sets</h1>
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"></th>
+    </tr>
+  </thead>
+
+  <tbody>
+    <% @data_sets.each do |data_set| %>
+      <tr>
+        <td><%= data_set.name %></td>
+        <td><%= link_to 'Show', data_set %></td>
+        <td><%= link_to 'Edit', edit_data_set_path(data_set) %></td>
+        <td><%= link_to 'Add more records', new_data_set_record_path(data_set) %></td>
+        <td><%= button_to 'Destroy', data_set, method: :delete, data: { confirm: 'Are you sure?' } %></td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+
+<br>
+
+New Data Set
+
+<%= render 'form' %>

data/app/views/tin_opener/data_sets/show.html.erb

@@ -0,0 +1,19 @@
+<p id="notice"><%= notice %></p>
+
+<%= link_to 'Edit', edit_data_set_path(@data_set) %> |
+<%= link_to 'Back', data_sets_path %>
+
+<h1><%= @data_set.name %></h1>
+
+<h2>Structure</h2>
+<p>
+  <%= @data_set.headers.keys.join(', ') %>
+</p>
+
+<h2>Records</h2>
+
+<% if @records.present? %>
+  <%= render @records %>
+<% else %>
+  <p>No records added for this data set yet.</p>
+<% end %>

data/app/views/tin_opener/records/_form.html.erb

@@ -0,0 +1,24 @@
+<%= form_for([@data_set, @record]) do |f| %>
+  <% if @record.errors.any? %>
+    <div id="error_explanation">
+      <h2><%= pluralize(@record.errors.count, "error") %> prohibited this record from being saved:</h2>
+
+      <ul>
+      <% @record.errors.full_messages.each do |message| %>
+        <li><%= message %></li>
+      <% end %>
+      </ul>
+    </div>
+  <% end %>
+
+  <div class="actions">
+    <%= f.fields_for :row_data do |header_fields| %>
+      <% @data_set.headers.each do |key, _| %>
+        <div>
+          <%= header_fields.text_field key, value: @record.row_data.try{ |h| h[key] } %>
+        </div>
+      <% end %>
+    <% end %>
+    <%= f.submit %>
+  </div>
+<% end %>

data/app/views/tin_opener/records/_record.html.erb

@@ -0,0 +1,8 @@
+<h3>Record#<%= record.id %></h3>
+
+<dl>
+  <% record.row_data.each do |key, value| %>
+    <dt><%= key %></dt>
+    <dd><%= value %></dd>
+  <% end %>
+</dl>

data/app/views/tin_opener/records/edit.html.erb

@@ -0,0 +1,6 @@
+<h1>Editing Record</h1>
+
+<%= render 'form' %>
+
+<%= link_to 'Show', @record %> |
+<%= link_to 'Back', data_set_records_path(@data_set) %>

data/app/views/tin_opener/records/index.html.erb

@@ -0,0 +1,25 @@
+<p id="notice"><%= notice %></p>
+
+<h1>Listing Records</h1>
+
+<table>
+  <thead>
+    <tr>
+      <th colspan="3"></th>
+    </tr>
+  </thead>
+
+  <tbody>
+    <% @records.each do |record| %>
+      <tr>
+        <td><%= link_to 'Show', data_set_record_path(@data_set, record) %></td>
+        <td><%= link_to 'Edit', edit_data_set_record_path(@data_set, record) %></td>
+        <td><%= button_to 'Destroy', data_set_record_path(@data_set, record), method: :delete, data: { confirm: 'Are you sure?' } %></td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+
+<br>
+
+<%= link_to 'New Record', new_data_set_record_path(@data_set) %>

data/app/views/tin_opener/records/new.html.erb

@@ -0,0 +1,5 @@
+<h1>New Record for DataSet: <%= @data_set.name %></h1>
+
+<%= render 'form' %>
+
+<%= link_to 'Back', data_set_records_path(@data_set) %>

data/app/views/tin_opener/records/show.html.erb

@@ -0,0 +1,11 @@
+<p id="notice"><%= notice %></p>
+
+<%= link_to 'Edit', edit_data_set_record_path(@data_set, @record) %> |
+<%= link_to 'Back', data_set_records_path(@data_set) %>
+
+<h2>Data set</h2>
+<%= @record.data_set.name %>
+
+<h2>Datos</h2>
+
+<%= render @record %>

data/config/routes.rb

@@ -0,0 +1,7 @@
+TinOpener::Engine.routes.draw do
+  resources :data_sets, only: [:index, :show, :create, :edit, :update, :destroy] do
+    resources :records
+  end
+
+  root 'data_sets#index'
+end

data/config/spring.rb

@@ -0,0 +1 @@
+Spring.application_root = './spec/dummy'

data/db/migrate/20151024183112_enable_hstore.rb

@@ -0,0 +1,5 @@
+class EnableHstore < ActiveRecord::Migration
+  def change
+    enable_extension 'hstore' unless extension_enabled?('hstore')
+  end
+end

data/db/migrate/20151024183219_create_tin_opener_data_sets.rb

@@ -0,0 +1,10 @@
+class CreateTinOpenerDataSets < ActiveRecord::Migration
+  def change
+    create_table :tin_opener_data_sets do |t|
+      t.string :name
+      t.hstore :headers
+
+      t.timestamps null: false
+    end
+  end
+end

data/db/migrate/20151024194928_create_tin_opener_records.rb

@@ -0,0 +1,10 @@
+class CreateTinOpenerRecords < ActiveRecord::Migration
+  def change
+    create_table :tin_opener_records do |t|
+      t.hstore :row_data
+      t.belongs_to :data_set
+
+      t.timestamps null: false
+    end
+  end
+end

data/lib/tasks/tin_opener_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :tin_opener do
+#   # Task goes here
+# end

data/lib/tin_opener/data_file_processor.rb

@@ -0,0 +1,22 @@
+module TinOpener
+  class DataFileProcessor
+    def initialize(args = {})
+      @file      = args.fetch(:file)
+      @separator = args.fetch(:separator) { ';' }
+    end
+
+    def headers
+      @headers ||= rows.first.try do |row|
+        row.transform_values do |value|
+          value.class.name
+        end
+      end
+    end
+
+    def rows
+      @csv_data ||= CSV.parse(@file, col_sep: @separator, headers: true).map do |row|
+          row.to_hash.transform_keys{ |a| a.squeeze.strip.gsub(/\s/, '_').underscore.to_sym }
+        end
+    end
+  end
+end

data/lib/tin_opener/engine.rb

@@ -0,0 +1,21 @@
+module TinOpener
+  class Engine < ::Rails::Engine
+    isolate_namespace TinOpener
+
+    config.generators do |g|
+      g.test_framework :rspec, view_specs: false, controller_specs: false,
+        routing_specs: false, helper_specs: false, request_specs: false
+      g.integration_tool :rspec
+      g.fixture_replacement :factory_girl, dir: 'spec/factories'
+
+      g.stylesheets     false
+      g.javascripts     false
+      g.helper          false
+    end
+
+    # config.paths.add "lib", eager_load: true
+    # config.paths.add "app", eager_load: true
+    # config.autoload_paths += Dir["#{config.root}/lib/**/"]
+    # config.autoload_paths += Dir["#{config.root}/app/**/"]
+  end
+end

data/lib/tin_opener/version.rb

@@ -0,0 +1,3 @@
+module TinOpener
+  VERSION = "0.0.1"
+end

data/lib/tin_opener.rb

@@ -0,0 +1,8 @@
+require "tin_opener/engine"
+require "tin_opener/data_file_processor"
+require "csv"
+require "activerecord-import"
+require 'pry-byebug' unless Rails.env.production?
+
+module TinOpener
+end

data/spec/dummy/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Rails.application.load_tasks

data/spec/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/spec/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/spec/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,5 @@
+class ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+end

data/spec/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end