orefine 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+pkg/

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Michael Bianco
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/orefine

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+
+require 'orefine'
+require 'slop'
+
+include Orefine
+
+$opts = Slop.parse do
+  banner 'Usage: refine.rb csv_a csv_b [options]'
+
+  on 'output-columns=', 'List of columns you want in the resulting csv', as: Array
+  on 'delete-columns=', 'What columns to delete from the output', as: Array
+  on 'merge=', 'What column to merge in from csv_b', as: Array
+  on 'add-static-column=', 'Add a column with a static value (input: key, value)', as: Array
+  on 'merge-common=', 'Merge values from common columns with a comma'
+
+  on 'diff', 'only output rows in csv_a whose email does not exist in csv_b'
+  on 'common', 'only output rows common to both csvs'
+  on 'split-full-name', 'split a full name field into first and last'
+
+  on 'open', 'open the document in a web browser'
+  on 'stdout', 'write the resulting csv to stdout'
+end
+
+if ARGV.empty?
+  puts $opts.help
+  exit
+end
+
+
+csv_a_path = ARGV[0]
+csv_b_path = ARGV[1]
+
+# TODO clear out all old CSV a & b or timestamp the new ones
+
+CSVUtil.clear_all_csvs
+csv_a = Refine.new("project_name" => 'csv_a', "file_name" => csv_a_path)
+csv_b = Refine.new("project_name" => 'csv_b', "file_name" => csv_b_path) if !csv_b_path.nil? && File.exists?(csv_b_path)
+
+all_csvs = [csv_a]
+all_csvs << csv_b if !csv_b.nil?
+
+CSVUtil.normalize_column_names(all_csvs)
+CSVUtil.normalize_email_column_content(all_csvs)
+CSVUtil.create_common_flag(csv_a, csv_b) if !csv_b.nil?
+
+CSVUtil.split_full_name(csv_a) if !$opts['split-full-name'].nil?
+
+if !$opts['merge'].nil?
+  $opts['merge'].each do |merge_field|
+    CSVUtil.merge_field(csv_a, csv_b, merge_field)
+  end
+end
+
+if !$opts['merge-common'].nil?
+  $opts['merge-common'].each do |common_merge|
+    CSVUtil.merge_common_field(csv_a, csv_b, common_merge)
+  end
+end
+
+if !$opts['add-static-column'].nil?
+  CSVUtil.add_column(csv_a, $opts['add-static-column'].first, $opts['add-static-column'].last)
+end
+
+output_params = { "format" => "csv" }
+
+if !$opts['output-columns'].nil?
+  output_params["options"] ||= {}
+  output_params["options"]["columns"] ||= []
+
+  $opts['output-columns'].each do |c|
+    output_params["options"]["columns"] << { "name" => c }
+  end
+end
+
+if !$opts['delete-columns'].nil?
+  $opts['delete-columns'].each do |column_name|
+    CSVUtil.delete_column(csv_a, column_name)
+  end
+end
+
+if !csv_b.nil? && ($opts.diff? || $opts.common?)
+  flag = true
+  flag = false if $opts.diff?
+
+  output_params["facets"] = [ CSVUtil.common_facet(flag) ]
+end
+
+puts csv_a.export_rows(output_params) if $opts.stdout?
+
+`open "http://127.0.0.1:3333/project?project=#{csv_a.project_id}"` if $opts.open?

data/lib/orefine.rb

@@ -0,0 +1,234 @@
+require "orefine/version"
+require 'google-refine'
+
+module Orefine
+  class CSVUtil
+    class << self
+      def clear_all_csvs
+        Refine.get_all_project_metadata["projects"]
+              .select { |k, v| v["name"].start_with?('csv_') }
+              .keys
+              .map { |project_id| Refine.new("project_id" => project_id) }
+              .map(&:delete_project)
+      end
+
+      def normalize_column_names(projects)
+        self.normalize_email_column_name(projects)
+        self.normalize_zip_column_name(projects)
+        self.normalize_full_name_column_name(projects)
+      end
+
+      def normalize_email_column_name(projects)
+        self.perform_operation(projects, %q{
+  [
+    {
+      "op": "core/column-rename",
+      "oldColumnName": "E-mail",
+      "newColumnName": "email"
+    },
+    {
+      "op": "core/column-rename",
+      "oldColumnName": "Email Address",
+      "newColumnName": "email"
+    },
+    {
+      "op": "core/column-rename",
+      "oldColumnName": "Email",
+      "newColumnName": "email"
+    },
+    {
+      "op": "core/column-rename",
+      "oldColumnName": "[email]",
+      "newColumnName": "email"
+    },
+    {
+      "op": "core/column-rename",
+      "oldColumnName": "email_stripped",
+      "newColumnName": "email"
+    },  
+  ]
+        })
+      end
+
+      def normalize_zip_column_name(projects)
+        self.perform_operation(projects, %q{
+  [
+    {
+      "op": "core/column-rename",
+      "oldColumnName": "Zip",
+      "newColumnName": "zip"
+    },
+    {
+      "op": "core/column-rename",
+      "oldColumnName": "[zip]",
+      "newColumnName": "zip"
+    }
+  ]
+        })
+      end
+
+      def normalize_full_name_column_name(projects)
+        self.perform_operation(projects, %q{
+  [
+    {
+      "op": "core/column-rename",
+      "oldColumnName": "Name",
+      "newColumnName": "full_name"
+    },
+    {
+      "op": "core/column-rename",
+      "oldColumnName": "Full Name",
+      "newColumnName": "full_name"
+    }
+  ]
+        })
+      end
+
+      def normalize_email_column_content(projects)
+        self.perform_operation(projects, %q{
+  [
+    {
+      "op": "core/column-addition",
+      "engineConfig": {
+        "facets": [],
+        "mode": "record-based"
+      },
+      "newColumnName": "email_stripped",
+      "columnInsertIndex": 0,
+      "baseColumnName": "email",
+      "expression": "grel:strip(value.toLowercase())",
+      "onError": "set-to-blank"
+    }
+  ]
+        })
+      end
+
+      def split_full_name(projects)
+        self.perform_operation(projects, %q{
+  [
+    {
+      "op": "core/column-split",
+      "description": "Split column Name by separator",
+      "engineConfig": {
+        "facets": [],
+        "mode": "row-based"
+      },
+      "columnName": "full_name",
+      "guessCellType": false,
+      "removeOriginalColumn": false,
+      "mode": "separator",
+      "separator": "(?<=[a-z]) ",
+      "regex": true,
+      "maxColumns": 2
+    }
+  ]
+        })
+      end
+
+      def create_common_flag(project_a, project_b)
+        if project_a.get_columns_info.map { |c| c["name"] }.include? 'exists'
+          STDERR.puts "'exists' column already exists in csv_a, deleting"
+          self.delete_column(project_a, "exists")
+        end
+
+        self.perform_operation(project_a, %Q{
+  [
+    {
+      "op": "core/column-addition",
+      "engineConfig": {
+        "facets": [],
+        "mode": "record-based"
+      },
+      "newColumnName": "exists",
+      "columnInsertIndex": 0,
+      "baseColumnName": "email_stripped",
+      "expression": "grel:cell.cross(\\\"#{project_b.project_name}\\\", \\\"email_stripped\\\").cells.length() > 0",
+      "onError": "set-to-blank"
+    }
+  ]
+        })
+      end
+
+      def merge_field(project_a, project_b, field)
+        self.perform_operation(project_a, %Q{
+  [
+    {
+      "op": "core/column-addition",
+      "engineConfig": {
+        "facets": [],
+        "mode": "record-based"
+      },
+      "newColumnName": "#{field}_merged",
+      "columnInsertIndex": 0,
+      "baseColumnName": "email_stripped",
+      "expression": "grel:cell.cross(\\\"#{project_b.project_name}\\\", \\\"email_stripped\\\").cells[\\\"#{field}\\\"].value[0]",
+      "onError": "set-to-blank"
+    }
+  ]
+        })
+      end
+
+      def merge_common_field(csv_a, csv_b, common_field)
+        
+      end
+
+      def delete_column(csv, field)
+        self.perform_operation(csv, %Q{
+  [
+    {
+      "op": "core/column-removal",
+      "columnName": "#{field}"
+    }
+  ]
+        })
+      end
+
+      def add_column(csv, field, value)
+        self.perform_operation(csv, %Q{
+  [
+    {
+      "op": "core/column-addition",
+      "engineConfig": {
+        "facets": [],
+        "mode": "record-based"
+      },
+      "newColumnName": "#{field}",
+      "columnInsertIndex": 0,
+      "baseColumnName": "email_stripped",
+      "expression": "grel:\\\"#{value}\\\"",
+      "onError": "set-to-blank"
+    }
+  ]
+        })
+      end
+
+      def common_facet(flag = true)
+        {
+          "invert" =>  false,
+          "expression" =>  "value",
+          "selectError" =>  false,
+          "omitError" =>  false,
+          "selectBlank" =>  false,
+          "name" =>  "exists",
+          "omitBlank" =>  false,
+          "columnName" =>  "exists",
+          "type" =>  "list",
+          "selection" =>  [
+            {
+              "v" =>  {
+                # string vs boolean matters here... be careful
+                "v" =>  flag,
+                "l" =>  flag,
+              }
+            }
+          ]
+        }
+      end
+
+      def perform_operation(projects, operation)
+        projects = [projects] if !projects.is_a?(Array)
+        projects.each { |p| p.apply_operations(operation) }
+      end
+    end
+  end
+end

data/lib/orefine/version.rb

@@ -0,0 +1,3 @@
+module Orefine
+  VERSION = "0.0.1"
+end

data/orefine.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'orefine/version'
+
+Gem::Specification.new do |s|
+  s.name          = "orefine"
+  s.version       = Orefine::VERSION
+  s.authors       = ["Michael Bianco"]
+  s.email         = ["info@cliffsidedev.com"]
+  s.description   = %q{Easily modify CSVs from the command line using Open Refine}
+  s.summary       = %q{Easily modify CSVs from the command line using Open Refine}
+  s.homepage      = "http://github.com/iloveitaly/orefine"
+  s.license       = "MIT"
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'google-refine', '~> 0.1.1'
+  s.add_dependency 'slop', '~> 3.4.6'
+
+  s.add_development_dependency "bundler", "~> 1.3"
+  s.add_development_dependency "rake"
+end

data/readme.md

@@ -0,0 +1,36 @@
+# ORefine - CLI for Working With Open Refine
+Makes working with CSVs a bit less painful. Tries to automate some common operations. Contributions welcome, this is a rough cut with the minimal features I needed to get a job done.
+
+You'll need [OpenRefine](https://github.com/OpenRefine/OpenRefine) installed & running.
+
+## Examples
+```
+# output a list of stripped emails from a CSV
+orefine a_list_of_emails_and_other_columns.csv --output-columns=email_stripped
+
+# list of emails common to both csvs
+orefine full_list.csv other_set_to_intersect_with.csv --common --output-columns=email
+
+# merge data from another list & tag
+orefine import_list.csv external_list_with_zip_and_state_data.csv --merge=zip,State --add-static-column=source,"LIST-TAG-DATA" --open
+
+# tag a list
+orefine import_list.csv --add-static-column=source,AUG-2013-POSTCARD-IMPORT --output-columns=email_stripped,source --stdout > ~/Desktop/list_import.csv
+```
+
+## Development Resources
+* [OpenRefine API](https://github.com/OpenRefine/OpenRefine/blob/a7273625d7c33af70b6d16db5782c802186b3b99/main/webapp/modules/core/MOD-INF/controller.js)
+* [GRel cross documentation](https://github.com/OpenRefine/OpenRefine/wiki/GREL-Other-Functions)
+* [Google Refine Gem](https://github.com/iloveitaly/refine-ruby)
+* [Tutorial on merging datasets with a common column](http://blog.ouseful.info/2011/05/06/merging-datesets-with-common-columns-in-google-refine/)
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Authors
+* Mike Bianco, @iloveitaly

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: orefine
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Michael Bianco
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: google-refine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.4.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.4.6
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Easily modify CSVs from the command line using Open Refine
+email:
+- info@cliffsidedev.com
+executables:
+- orefine
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- Rakefile
+- bin/orefine
+- lib/orefine.rb
+- lib/orefine/version.rb
+- orefine.gemspec
+- readme.md
+homepage: http://github.com/iloveitaly/orefine
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Easily modify CSVs from the command line using Open Refine
+test_files: []